Merge pull request 'Persona chats from SMS threads (talk about / practice), date_info, photos never sent to OpenRouter' (#186) from feat/sms-anchored-persona-chat into master
Reviewed-on: #186
This commit was merged in pull request #186.
This commit is contained in:
commit
e8aa337442
17 files changed
+2668
-212
No files matched your search
@@ -577,6 +577,21 @@ POST /insights/chat/stream (SSE: text / tool_call / tool_result / trun
|
||||
GET /insights/chat/history?path=... (rendered transcript with tool invocations)
|
||||
POST /insights/chat/rewind (truncate transcript at a rendered index)
|
||||
|
||||
// Persona chat — "Continue with persona" from an SMS thread (src/ai/sms_anchor.rs)
|
||||
POST /persona_chat/conversations (body: { persona_id, anchor?: { contact_id, message_id,
|
||||
mode?: "about" | "as_contact" } }; anchor snapshots ~40
|
||||
SMS-API messages up to message_id into the row, 502 if
|
||||
SMS-API can't read the thread. `as_contact` (practice) needs
|
||||
no persona: stored under PRACTICE_PERSONA_ID, listed as
|
||||
"Practice")
|
||||
GET /persona_chat/conversations?contact_id=N (only chats anchored on that thread; DAO query)
|
||||
// Anchored turns append the thread + mode guidance to the prompt and add the thread
|
||||
// tools (ToolGateOpts.sms_thread_tools): get_thread_messages / look_at_message_image are
|
||||
// bound server-side to the chat's own thread (a model-named contact_id is refused), and
|
||||
// about-mode also gets get_photos_near_time / look_at_photo. Practice keeps only the two
|
||||
// thread tools. Image tools use the LOCAL vision model; MMS downloads are image-only and
|
||||
// capped (MMS_IMAGE_MAX_BYTES); look_at_photo only opens indexed photos.
|
||||
|
||||
// User AI Prefs (owner defaults + passive mirror; src/prefs.rs)
|
||||
GET /prefs (current row; auth)
|
||||
PUT /prefs (partial update: omitted=unchanged, null=clear, value=set; auth)
|
||||
@@ -891,9 +906,18 @@ endpoint.
|
||||
a deploy-time decision, not a request-time one).
|
||||
- `hybrid → hybrid` allowed.
|
||||
- `hybrid → local` allowed (the inlined description replays as text).
|
||||
- `local → hybrid` rejected — the stored transcript has raw images in the
|
||||
first user message and OpenRouter providers don't accept that shape
|
||||
consistently. Regenerate the insight in hybrid mode instead.
|
||||
- `local → hybrid` allowed. The stored transcript carries the raw photo in
|
||||
its first user message, so each hybrid turn swaps it for a local vision
|
||||
description (cached per photo) in the copy sent to OpenRouter, then puts
|
||||
the photo back before saving (`InsightChatService::inline_images_as_text`).
|
||||
If the local model can't describe it, the turn fails. A stored model id is
|
||||
only reused on the same backend (`continuation_model`).
|
||||
|
||||
**Photos never go to OpenRouter.** `OpenRouterClient` refuses any request
|
||||
carrying image data (`IMAGES_STAY_LOCAL`) and its `describe_image` always
|
||||
errors — the client is the choke point, so a caller bug fails the turn
|
||||
instead of uploading a photo. Image work in hybrid mode goes through
|
||||
`ResolvedBackend::local()`.
|
||||
|
||||
**Insight Chat Continuation:**
|
||||
|
||||
@@ -924,9 +948,9 @@ clients whether chat is available for a given insight.
|
||||
|
||||
Backend routing rules (matches agentic-insight generation):
|
||||
- Stored `backend` on the insight row is authoritative by default.
|
||||
- `request.backend` may override per-turn. `local -> hybrid` is rejected in
|
||||
v1 (would require on-the-fly visual-description rewrite); `hybrid -> local`
|
||||
replays verbatim since the description is already inlined as text.
|
||||
- `request.backend` may override per-turn, in either direction (see the
|
||||
cross-replay matrix above); `hybrid -> local` replays verbatim since the
|
||||
description is already inlined as text.
|
||||
- `request.model` overrides the chat model (an Ollama id in local mode, an
|
||||
OpenRouter id in hybrid mode).
|
||||
|
||||
|
||||
Generated
+1
-1
@@ -2070,7 +2070,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "image-api"
|
||||
version = "1.5.0"
|
||||
version = "1.6.0"
|
||||
dependencies = [
|
||||
"actix",
|
||||
"actix-cors",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "image-api"
|
||||
version = "1.5.0"
|
||||
version = "1.6.0"
|
||||
authors = ["Cameron Cordes <cameronc.dev@gmail.com>"]
|
||||
edition = "2024"
|
||||
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
DROP INDEX IF EXISTS idx_persona_chat_anchor_contact;
|
||||
ALTER TABLE persona_chat_conversations DROP COLUMN anchor_mode;
|
||||
ALTER TABLE persona_chat_conversations DROP COLUMN anchor_context;
|
||||
ALTER TABLE persona_chat_conversations DROP COLUMN anchor_ts;
|
||||
ALTER TABLE persona_chat_conversations DROP COLUMN anchor_message_id;
|
||||
ALTER TABLE persona_chat_conversations DROP COLUMN anchor_contact_name;
|
||||
ALTER TABLE persona_chat_conversations DROP COLUMN anchor_contact_id;
|
||||
@@ -0,0 +1,28 @@
|
||||
-- Anchor a persona conversation on a point in an SMS thread.
|
||||
--
|
||||
-- "Continue with persona" starts a persona chat from a long-pressed message
|
||||
-- in the Messages screen. The conversation remembers which thread and which
|
||||
-- message it grew out of, so the SMS screen can list the persona chats that
|
||||
-- belong to a thread and the chat screen can show where it picked up.
|
||||
--
|
||||
-- `anchor_context` is a snapshot of the thread leading up to the anchor,
|
||||
-- rendered once at creation and spliced into the system prompt on every
|
||||
-- turn. Snapshotting (rather than refetching per turn) keeps the prompt
|
||||
-- prefix byte-stable for KV-cache reuse and keeps the persona's view of the
|
||||
-- thread fixed even if SMS-API is re-imported underneath it.
|
||||
--
|
||||
-- `anchor_mode` is how the chat relates to the thread: 'about' (a persona
|
||||
-- talks with the user about it) or 'as_contact' (the model plays the
|
||||
-- contact so the user can practise the conversation). NULL reads as 'about'.
|
||||
--
|
||||
-- All nullable: an unanchored (open) persona chat leaves them NULL.
|
||||
|
||||
ALTER TABLE persona_chat_conversations ADD COLUMN anchor_contact_id BIGINT;
|
||||
ALTER TABLE persona_chat_conversations ADD COLUMN anchor_contact_name TEXT;
|
||||
ALTER TABLE persona_chat_conversations ADD COLUMN anchor_message_id BIGINT;
|
||||
ALTER TABLE persona_chat_conversations ADD COLUMN anchor_ts BIGINT;
|
||||
ALTER TABLE persona_chat_conversations ADD COLUMN anchor_context TEXT;
|
||||
ALTER TABLE persona_chat_conversations ADD COLUMN anchor_mode TEXT;
|
||||
|
||||
CREATE INDEX idx_persona_chat_anchor_contact
|
||||
ON persona_chat_conversations (user_id, anchor_contact_id, updated_at DESC);
|
||||
@@ -0,0 +1,375 @@
|
||||
//! The `date_info` agent tool: date arithmetic the model shouldn't do in
|
||||
//! its head.
|
||||
//!
|
||||
//! Small local models are reliably shaky at weekday lookups, "how long ago
|
||||
//! was …" across year boundaries and day counts between two dates — and
|
||||
//! they state the wrong answer confidently. Persona chats anchored on old
|
||||
//! SMS threads ask exactly these questions, so the tool answers them
|
||||
//! exactly. It also reports "now", replacing the old `get_current_datetime`.
|
||||
//!
|
||||
//! Everything is computed in the owner's display zone, so the dates match
|
||||
//! the timestamps the model reads in thread transcripts and photo metadata.
|
||||
|
||||
use chrono::{DateTime, Datelike, Months, NaiveDate, NaiveDateTime, TimeZone, Utc};
|
||||
use chrono_tz::Tz;
|
||||
|
||||
/// A date argument as the model gave it. A bare date stays a date (no
|
||||
/// invented time of day); a date with a time keeps it.
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
enum Moment {
|
||||
Day(NaiveDate),
|
||||
At(NaiveDateTime),
|
||||
}
|
||||
|
||||
impl Moment {
|
||||
fn date(self) -> NaiveDate {
|
||||
match self {
|
||||
Moment::Day(d) => d,
|
||||
Moment::At(dt) => dt.date(),
|
||||
}
|
||||
}
|
||||
|
||||
fn label(self) -> String {
|
||||
match self {
|
||||
Moment::Day(d) => format!("{} ({})", d.format("%Y-%m-%d"), d.format("%A")),
|
||||
Moment::At(dt) => format!("{} ({})", dt.format("%Y-%m-%d %H:%M"), dt.format("%A")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse `YYYY-MM-DD`, `YYYY-MM-DD HH:MM` (or with `T` / seconds), or unix
|
||||
/// seconds (read in `zone`).
|
||||
fn parse_moment(value: &serde_json::Value, zone: Tz) -> Option<Moment> {
|
||||
if let Some(ts) = value.as_i64() {
|
||||
let utc = DateTime::<Utc>::from_timestamp(ts, 0)?;
|
||||
return Some(Moment::At(utc.with_timezone(&zone).naive_local()));
|
||||
}
|
||||
let raw = value.as_str()?.trim();
|
||||
for fmt in [
|
||||
"%Y-%m-%d %H:%M",
|
||||
"%Y-%m-%dT%H:%M",
|
||||
"%Y-%m-%d %H:%M:%S",
|
||||
"%Y-%m-%dT%H:%M:%S",
|
||||
] {
|
||||
if let Ok(dt) = NaiveDateTime::parse_from_str(raw, fmt) {
|
||||
return Some(Moment::At(dt));
|
||||
}
|
||||
}
|
||||
if let Ok(ts) = raw.parse::<i64>() {
|
||||
return parse_moment(&serde_json::json!(ts), zone);
|
||||
}
|
||||
NaiveDate::parse_from_str(raw, "%Y-%m-%d")
|
||||
.ok()
|
||||
.map(Moment::Day)
|
||||
}
|
||||
|
||||
/// A moment for tools that search *around* a time (`get_photos_near_time`),
|
||||
/// as unix seconds. Same formats as `date_info`; a bare date centres on
|
||||
/// local midday, the least-wrong centre for a whole day.
|
||||
pub fn parse_center(value: &serde_json::Value, zone: Tz) -> Option<i64> {
|
||||
let local = match parse_moment(value, zone)? {
|
||||
Moment::At(dt) => dt,
|
||||
Moment::Day(d) => d.and_hms_opt(12, 0, 0)?,
|
||||
};
|
||||
zone.from_local_datetime(&local)
|
||||
.earliest()
|
||||
.map(|dt| dt.timestamp())
|
||||
}
|
||||
|
||||
/// Whole years, months and days from `a` to `b` (`a <= b`), counted the
|
||||
/// calendar way: Jan 31 + 1 month is Feb 28/29, as people count ages.
|
||||
fn calendar_gap(a: NaiveDate, b: NaiveDate) -> (u32, u32, u32) {
|
||||
let mut months = (b.year() - a.year()) * 12 + b.month() as i32 - a.month() as i32;
|
||||
let mut anchor = a.checked_add_months(Months::new(months.max(0) as u32));
|
||||
if anchor.is_none_or(|d| d > b) {
|
||||
months -= 1;
|
||||
anchor = a.checked_add_months(Months::new(months.max(0) as u32));
|
||||
}
|
||||
let months = months.max(0) as u32;
|
||||
let days = anchor.map(|d| (b - d).num_days()).unwrap_or(0).max(0) as u32;
|
||||
(months / 12, months % 12, days)
|
||||
}
|
||||
|
||||
fn plural(n: impl Into<i64>, unit: &str) -> String {
|
||||
let n = n.into();
|
||||
if n == 1 {
|
||||
format!("1 {unit}")
|
||||
} else {
|
||||
format!("{n} {unit}s")
|
||||
}
|
||||
}
|
||||
|
||||
/// "7 years, 3 months and 15 days" — zero parts dropped, "0 days" when the
|
||||
/// dates are the same.
|
||||
fn describe_gap(a: NaiveDate, b: NaiveDate) -> String {
|
||||
let (y, m, d) = calendar_gap(a, b);
|
||||
let mut parts = Vec::new();
|
||||
if y > 0 {
|
||||
parts.push(plural(y, "year"));
|
||||
}
|
||||
if m > 0 {
|
||||
parts.push(plural(m, "month"));
|
||||
}
|
||||
if d > 0 || parts.is_empty() {
|
||||
parts.push(plural(d, "day"));
|
||||
}
|
||||
match parts.len() {
|
||||
1 => parts.remove(0),
|
||||
_ => {
|
||||
let last = parts.pop().unwrap_or_default();
|
||||
format!("{} and {}", parts.join(", "), last)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The span between two dates: a headline phrase, plus a detail (exact
|
||||
/// day count, weeks) the caller places after any "ago" / "in".
|
||||
fn span(a: NaiveDate, b: NaiveDate) -> (String, String) {
|
||||
let (lo, hi) = if a <= b { (a, b) } else { (b, a) };
|
||||
let days = (hi - lo).num_days();
|
||||
let weeks = if days >= 14 {
|
||||
format!("about {}", plural(days / 7, "week"))
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
if days < 31 {
|
||||
(plural(days, "day"), weeks)
|
||||
} else {
|
||||
let detail = if weeks.is_empty() {
|
||||
plural(days, "day")
|
||||
} else {
|
||||
format!("{}, {weeks}", plural(days, "day"))
|
||||
};
|
||||
(describe_gap(lo, hi), detail)
|
||||
}
|
||||
}
|
||||
|
||||
/// `headline` with `(detail)` appended when there is one.
|
||||
fn with_detail(headline: String, detail: &str) -> String {
|
||||
if detail.is_empty() {
|
||||
headline
|
||||
} else {
|
||||
format!("{headline} ({detail})")
|
||||
}
|
||||
}
|
||||
|
||||
/// Run the tool. `now` is injected so tests are deterministic.
|
||||
pub fn date_info(args: &serde_json::Value, now: DateTime<Utc>, zone: Tz) -> String {
|
||||
let now_local = now.with_timezone(&zone);
|
||||
let today = now_local.date_naive();
|
||||
let mut out = format!(
|
||||
"Now: {} ({}), {}",
|
||||
now_local.format("%Y-%m-%d %H:%M"),
|
||||
now_local.format("%A"),
|
||||
zone.name()
|
||||
);
|
||||
|
||||
let first = args.get("date").filter(|v| !v.is_null());
|
||||
let second = args.get("other_date").filter(|v| !v.is_null());
|
||||
let add_days = args.get("add_days").and_then(|v| v.as_i64());
|
||||
let Some(first) = first else {
|
||||
return out;
|
||||
};
|
||||
let Some(a) = parse_moment(first, zone) else {
|
||||
return format!(
|
||||
"Error: couldn't read date {first} — use YYYY-MM-DD, YYYY-MM-DD HH:MM or unix seconds"
|
||||
);
|
||||
};
|
||||
|
||||
// Offset: "what date is N days after/before X".
|
||||
if let Some(n) = add_days {
|
||||
let Some(result) = a.date().checked_add_signed(chrono::Duration::days(n)) else {
|
||||
return "Error: add_days is out of range".to_string();
|
||||
};
|
||||
let direction = if n >= 0 { "after" } else { "before" };
|
||||
out.push_str(&format!(
|
||||
"\n{} {direction} {} is {}.",
|
||||
plural(n.abs(), "day"),
|
||||
a.label(),
|
||||
Moment::Day(result).label()
|
||||
));
|
||||
return out;
|
||||
}
|
||||
|
||||
match second {
|
||||
None => {
|
||||
let d = a.date();
|
||||
let relation = match d.cmp(&today) {
|
||||
std::cmp::Ordering::Equal => "today".to_string(),
|
||||
std::cmp::Ordering::Less => {
|
||||
let (head, detail) = span(d, today);
|
||||
with_detail(format!("{head} ago"), &detail)
|
||||
}
|
||||
std::cmp::Ordering::Greater => {
|
||||
let (head, detail) = span(today, d);
|
||||
with_detail(format!("in {head}"), &detail)
|
||||
}
|
||||
};
|
||||
out.push_str(&format!("\n{} — {relation}.", a.label()));
|
||||
}
|
||||
Some(second) => {
|
||||
let Some(b) = parse_moment(second, zone) else {
|
||||
return format!(
|
||||
"Error: couldn't read other_date {second} — use YYYY-MM-DD, YYYY-MM-DD HH:MM or unix seconds"
|
||||
);
|
||||
};
|
||||
let (from, to) = if a.date() <= b.date() { (a, b) } else { (b, a) };
|
||||
let mut gap = if from.date() == to.date() {
|
||||
"the same day".to_string()
|
||||
} else {
|
||||
let (head, detail) = span(from.date(), to.date());
|
||||
with_detail(head, &detail)
|
||||
};
|
||||
// Both with times and close together: the hours are what matter.
|
||||
if let (Moment::At(x), Moment::At(y)) = (from, to) {
|
||||
let minutes = (y - x).num_minutes().abs();
|
||||
if minutes < 48 * 60 {
|
||||
gap.push_str(&format!(" ({}h {}m apart)", minutes / 60, minutes % 60));
|
||||
}
|
||||
}
|
||||
out.push_str(&format!(
|
||||
"\nFrom {} to {}: {gap}.",
|
||||
from.label(),
|
||||
to.label()
|
||||
));
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
const NY: Tz = chrono_tz::America::New_York;
|
||||
|
||||
fn at(zone: Tz, y: i32, m: u32, d: u32, h: u32, min: u32) -> DateTime<Utc> {
|
||||
zone.with_ymd_and_hms(y, m, d, h, min, 0)
|
||||
.single()
|
||||
.expect("unambiguous local time")
|
||||
.with_timezone(&Utc)
|
||||
}
|
||||
|
||||
fn now() -> DateTime<Utc> {
|
||||
// Sunday 2026-10-18 20:15 in New York.
|
||||
at(NY, 2026, 10, 18, 20, 15)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_date_reports_now_in_the_display_zone() {
|
||||
assert_eq!(
|
||||
date_info(&json!({}), now(), NY),
|
||||
"Now: 2026-10-18 20:15 (Sunday), America/New_York"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_past_date_gets_its_weekday_and_how_long_ago() {
|
||||
let out = date_info(&json!({"date": "2019-07-03"}), now(), NY);
|
||||
assert!(
|
||||
out.ends_with(
|
||||
"2019-07-03 (Wednesday) — 7 years, 3 months and 15 days ago (2664 days, about 380 weeks)."
|
||||
),
|
||||
"{out}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn near_dates_read_as_days() {
|
||||
let out = date_info(&json!({"date": "2026-10-15"}), now(), NY);
|
||||
assert!(
|
||||
out.ends_with("2026-10-15 (Thursday) — 3 days ago."),
|
||||
"{out}"
|
||||
);
|
||||
let out = date_info(&json!({"date": "2026-10-18"}), now(), NY);
|
||||
assert!(out.ends_with("— today."), "{out}");
|
||||
let out = date_info(&json!({"date": "2026-11-01"}), now(), NY);
|
||||
assert!(out.ends_with("— in 14 days (about 2 weeks)."), "{out}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn today_follows_the_display_zone_not_utc() {
|
||||
// 02:30 UTC on the 19th is still the evening of the 18th in New York.
|
||||
let late = at(NY, 2026, 10, 18, 22, 30);
|
||||
let out = date_info(&json!({"date": "2026-10-18"}), late, NY);
|
||||
assert!(out.ends_with("— today."), "{out}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unix_seconds_read_in_the_display_zone() {
|
||||
// 2019-07-04 02:47 UTC = 2019-07-03 22:47 in New York.
|
||||
let out = date_info(&json!({"date": 1562208420}), now(), NY);
|
||||
assert!(out.contains("2019-07-03 22:47 (Wednesday)"), "{out}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_dates_give_the_gap_in_either_order() {
|
||||
let a = date_info(
|
||||
&json!({"date": "2019-07-05", "other_date": "2019-07-03"}),
|
||||
now(),
|
||||
NY,
|
||||
);
|
||||
assert!(
|
||||
a.ends_with("From 2019-07-03 (Wednesday) to 2019-07-05 (Friday): 2 days."),
|
||||
"{a}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_close_times_include_the_hours() {
|
||||
let out = date_info(
|
||||
&json!({"date": "2019-07-03 22:47", "other_date": "2019-07-04 08:40"}),
|
||||
now(),
|
||||
NY,
|
||||
);
|
||||
assert!(
|
||||
out.ends_with("2019-07-04 08:40 (Thursday): 1 day (9h 53m apart)."),
|
||||
"{out}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn add_days_moves_forward_and_back() {
|
||||
let out = date_info(&json!({"date": "2019-07-03", "add_days": 90}), now(), NY);
|
||||
assert!(
|
||||
out.ends_with("90 days after 2019-07-03 (Wednesday) is 2019-10-01 (Tuesday)."),
|
||||
"{out}"
|
||||
);
|
||||
let out = date_info(&json!({"date": "2024-03-01", "add_days": -1}), now(), NY);
|
||||
assert!(out.ends_with("is 2024-02-29 (Thursday)."), "{out}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn calendar_gap_counts_like_people_do() {
|
||||
let d = |y, m, dd| NaiveDate::from_ymd_opt(y, m, dd).unwrap();
|
||||
// Month-end clamps: Jan 31 + 1 month = Feb 29 in a leap year.
|
||||
assert_eq!(calendar_gap(d(2020, 1, 31), d(2020, 2, 29)), (0, 1, 0));
|
||||
assert_eq!(calendar_gap(d(2020, 1, 31), d(2020, 3, 1)), (0, 1, 1));
|
||||
assert_eq!(calendar_gap(d(2016, 2, 29), d(2026, 2, 28)), (10, 0, 0));
|
||||
assert_eq!(calendar_gap(d(2019, 7, 3), d(2026, 10, 18)), (7, 3, 15));
|
||||
assert_eq!(describe_gap(d(2025, 10, 18), d(2026, 10, 18)), "1 year");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_center_reads_unix_wall_clock_and_bare_dates() {
|
||||
assert_eq!(parse_center(&json!(1_700_000_000), NY), Some(1_700_000_000));
|
||||
// 2023-11-14 17:13 in New York (EST) = 22:13 UTC.
|
||||
assert_eq!(
|
||||
parse_center(&json!("2023-11-14 17:13"), NY),
|
||||
Some(1_699_999_980)
|
||||
);
|
||||
// A bare date centres on local midday.
|
||||
assert_eq!(parse_center(&json!("2023-11-14"), NY), Some(1_699_981_200));
|
||||
assert_eq!(parse_center(&json!("soon"), NY), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bad_input_says_what_format_to_use() {
|
||||
let out = date_info(&json!({"date": "last tuesday"}), now(), NY);
|
||||
assert!(
|
||||
out.starts_with("Error:") && out.contains("YYYY-MM-DD"),
|
||||
"{out}"
|
||||
);
|
||||
}
|
||||
}
|
||||
+310
-47
@@ -119,8 +119,16 @@ pub struct InsightChatService {
|
||||
generator: Arc<InsightGenerator>,
|
||||
insight_dao: Arc<Mutex<Box<dyn InsightDao>>>,
|
||||
chat_locks: ChatLockMap,
|
||||
/// Local vision descriptions of transcript images, keyed by a hash of
|
||||
/// the image data, so a local-generated chat continued on a text-only
|
||||
/// (hybrid) model describes its photo once rather than every turn.
|
||||
image_descriptions: Arc<Mutex<HashMap<u64, String>>>,
|
||||
}
|
||||
|
||||
/// Cap on cached image descriptions; the cache is cleared when it fills.
|
||||
/// One entry per photo chatted about since the server started.
|
||||
const IMAGE_DESCRIPTION_CACHE_MAX: usize = 256;
|
||||
|
||||
impl InsightChatService {
|
||||
pub fn new(
|
||||
generator: Arc<InsightGenerator>,
|
||||
@@ -131,9 +139,62 @@ impl InsightChatService {
|
||||
generator,
|
||||
insight_dao,
|
||||
chat_locks,
|
||||
image_descriptions: Arc::new(Mutex::new(HashMap::new())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Prepare a replayed transcript for a backend that can't take images.
|
||||
///
|
||||
/// A chat generated on the local backend carries the photo itself in its
|
||||
/// first user message. Continuing it on hybrid would send that photo to
|
||||
/// OpenRouter — which the OpenRouter client refuses — so each image is
|
||||
/// described by the local vision model and the text takes its place in
|
||||
/// the copy sent to the model. The returned stash puts the images back
|
||||
/// before the transcript is saved, so the stored chat keeps its photo and
|
||||
/// a later local turn still sees it.
|
||||
///
|
||||
/// A no-op for backends that take images inline. If the local model
|
||||
/// can't describe an image the turn fails: going ahead without the
|
||||
/// photo would quietly change what the chat is about.
|
||||
async fn inline_images_as_text(
|
||||
&self,
|
||||
backend: &ResolvedBackend,
|
||||
messages: &mut [ChatMessage],
|
||||
) -> Result<ImageStash> {
|
||||
if backend.images_inline {
|
||||
return Ok(ImageStash::default());
|
||||
}
|
||||
let mut descriptions: HashMap<String, String> = HashMap::new();
|
||||
for image in transcript_images(messages) {
|
||||
let key = image_cache_key(&image);
|
||||
let cached = self
|
||||
.image_descriptions
|
||||
.lock()
|
||||
.ok()
|
||||
.and_then(|c| c.get(&key).cloned());
|
||||
let description = match cached {
|
||||
Some(d) => d,
|
||||
None => {
|
||||
let d = backend.local().describe_image(&image).await.map_err(|e| {
|
||||
anyhow!(
|
||||
"couldn't describe the photo with the local vision model, so this \
|
||||
chat can't continue on a text-only model: {e}"
|
||||
)
|
||||
})?;
|
||||
if let Ok(mut cache) = self.image_descriptions.lock() {
|
||||
if cache.len() >= IMAGE_DESCRIPTION_CACHE_MAX {
|
||||
cache.clear();
|
||||
}
|
||||
cache.insert(key, d.clone());
|
||||
}
|
||||
d
|
||||
}
|
||||
};
|
||||
descriptions.insert(image, description);
|
||||
}
|
||||
Ok(apply_image_descriptions(messages, &descriptions))
|
||||
}
|
||||
|
||||
/// Accessor for the insight DAO (used by async job completion).
|
||||
pub fn insight_dao(&self) -> &Arc<Mutex<Box<dyn InsightDao>>> {
|
||||
&self.insight_dao
|
||||
@@ -285,11 +346,12 @@ impl InsightChatService {
|
||||
|
||||
let stored_model = insight.model_version.clone();
|
||||
let overrides = SamplingOverrides {
|
||||
model: req
|
||||
.model
|
||||
.clone()
|
||||
.or_else(|| Some(stored_model.clone()))
|
||||
.filter(|m| !m.is_empty()),
|
||||
model: continuation_model(
|
||||
req.model.clone(),
|
||||
&stored_model,
|
||||
&stored_backend,
|
||||
kind.as_str(),
|
||||
),
|
||||
num_ctx: req.num_ctx,
|
||||
temperature: req.temperature,
|
||||
top_p: req.top_p,
|
||||
@@ -298,6 +360,9 @@ impl InsightChatService {
|
||||
enable_thinking: req.enable_thinking,
|
||||
};
|
||||
let backend = self.generator.resolve_backend(kind, &overrides).await?;
|
||||
// Text-only backend (hybrid) on a transcript holding the photo:
|
||||
// swap it for a local description. Restored before saving.
|
||||
let image_stash = self.inline_images_as_text(&backend, &mut messages).await?;
|
||||
let model_used = backend.model().to_string();
|
||||
span.set_attribute("model", model_used.clone());
|
||||
|
||||
@@ -408,6 +473,7 @@ impl InsightChatService {
|
||||
&normalized,
|
||||
req.user_id,
|
||||
&active_persona,
|
||||
None,
|
||||
)
|
||||
.instrument(loop_span.clone())
|
||||
.await;
|
||||
@@ -449,6 +515,8 @@ impl InsightChatService {
|
||||
// Drop the per-turn iteration-budget note from the system message
|
||||
// before we persist so it doesn't snowball on each subsequent turn.
|
||||
restore_system_content(&mut messages, original_system_content);
|
||||
// Put the photo back so the stored transcript keeps it.
|
||||
image_stash.restore(&mut messages);
|
||||
|
||||
// Append mode: undo the per-turn system-prompt override so the
|
||||
// stored transcript keeps the original baked persona. Amend mode:
|
||||
@@ -979,11 +1047,12 @@ impl InsightChatService {
|
||||
|
||||
let stored_model = insight.model_version.clone();
|
||||
let overrides = SamplingOverrides {
|
||||
model: req
|
||||
.model
|
||||
.clone()
|
||||
.or_else(|| Some(stored_model.clone()))
|
||||
.filter(|m| !m.is_empty()),
|
||||
model: continuation_model(
|
||||
req.model.clone(),
|
||||
&stored_model,
|
||||
&stored_backend,
|
||||
kind.as_str(),
|
||||
),
|
||||
num_ctx: req.num_ctx,
|
||||
temperature: req.temperature,
|
||||
top_p: req.top_p,
|
||||
@@ -992,6 +1061,9 @@ impl InsightChatService {
|
||||
enable_thinking: req.enable_thinking,
|
||||
};
|
||||
let backend = self.generator.resolve_backend(kind, &overrides).await?;
|
||||
// Text-only backend (hybrid) on a transcript holding the photo:
|
||||
// swap it for a local description. Restored before saving.
|
||||
let image_stash = self.inline_images_as_text(&backend, &mut messages).await?;
|
||||
let model_used = backend.model().to_string();
|
||||
// llm.* attributes on the turn-level span (issue #130): clients can't
|
||||
// know local-vs-hybrid, but the turn knows its resolved backend.
|
||||
@@ -1064,6 +1136,8 @@ impl InsightChatService {
|
||||
}
|
||||
|
||||
restore_system_content(&mut messages, original_system_content);
|
||||
// Put the photo back so the stored transcript keeps it.
|
||||
image_stash.restore(&mut messages);
|
||||
|
||||
if !req.amend {
|
||||
restore_system_prompt_override(&mut messages, override_stash);
|
||||
@@ -1362,6 +1436,7 @@ impl InsightChatService {
|
||||
normalized,
|
||||
user_id,
|
||||
active_persona,
|
||||
None,
|
||||
max_iterations,
|
||||
entry,
|
||||
fitter,
|
||||
@@ -1455,11 +1530,12 @@ impl InsightChatService {
|
||||
|
||||
let stored_model = insight.model_version.clone();
|
||||
let overrides = SamplingOverrides {
|
||||
model: req
|
||||
.model
|
||||
.clone()
|
||||
.or_else(|| Some(stored_model.clone()))
|
||||
.filter(|m| !m.is_empty()),
|
||||
model: continuation_model(
|
||||
req.model.clone(),
|
||||
&stored_model,
|
||||
&stored_backend,
|
||||
kind.as_str(),
|
||||
),
|
||||
num_ctx: req.num_ctx,
|
||||
temperature: req.temperature,
|
||||
top_p: req.top_p,
|
||||
@@ -1468,6 +1544,9 @@ impl InsightChatService {
|
||||
enable_thinking: req.enable_thinking,
|
||||
};
|
||||
let backend = self.generator.resolve_backend(kind, &overrides).await?;
|
||||
// Text-only backend (hybrid) on a transcript holding the photo:
|
||||
// swap it for a local description. Restored before saving.
|
||||
let image_stash = self.inline_images_as_text(&backend, &mut messages).await?;
|
||||
let model_used = backend.model().to_string();
|
||||
|
||||
// Tool set — images_inline mode + first user turn carries an image →
|
||||
@@ -1533,6 +1612,8 @@ impl InsightChatService {
|
||||
// Drop the per-turn iteration-budget note before persisting so it
|
||||
// doesn't snowball on subsequent turns.
|
||||
restore_system_content(&mut messages, original_system_content);
|
||||
// Put the photo back so the stored transcript keeps it.
|
||||
image_stash.restore(&mut messages);
|
||||
|
||||
// Append mode: undo the per-turn system-prompt override.
|
||||
// Amend mode: keep it — it becomes the new row's system message.
|
||||
@@ -1911,6 +1992,7 @@ impl InsightChatService {
|
||||
normalized,
|
||||
user_id,
|
||||
active_persona,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
let (result_preview, result_truncated) = truncate_tool_result(&result);
|
||||
@@ -2071,34 +2153,122 @@ fn resolve_date_taken_for_context(
|
||||
})
|
||||
}
|
||||
|
||||
/// Validate a stored→effective backend transition for a chat continuation.
|
||||
/// Continuation runs against a transcript that was generated with a specific
|
||||
/// backend; the only blocked transition is `local → hybrid`, because the
|
||||
/// stored transcript has images embedded in the first user message and the
|
||||
/// hybrid path (OpenRouter chat with describe-then-inline) can't replay
|
||||
/// raw image bytes through OpenRouter consistently across providers.
|
||||
/// `hybrid → local` is allowed (the inlined description replays verbatim
|
||||
/// as text).
|
||||
/// Validate the backend a chat continuation is switching to.
|
||||
///
|
||||
/// Every transition is allowed now. `local → hybrid` used to be refused
|
||||
/// because a local transcript carries the raw photo; the turn now replaces
|
||||
/// it with a local vision description before anything reaches OpenRouter
|
||||
/// (`InsightChatService::inline_images_as_text`), and the OpenRouter client
|
||||
/// refuses image data outright as a backstop. `hybrid → local` replays the
|
||||
/// inlined description as text. `stored` is kept for the call sites'
|
||||
/// symmetry and future rules.
|
||||
///
|
||||
/// Whether "local" routes through Ollama or llama-swap is decided at
|
||||
/// startup by `LLM_BACKEND`; both share the same transcript shape from
|
||||
/// the chat-replay perspective.
|
||||
fn validate_cross_replay(stored: &str, effective: &str) -> Result<()> {
|
||||
fn validate_cross_replay(_stored: &str, effective: &str) -> Result<()> {
|
||||
if !matches!(effective, "local" | "hybrid") {
|
||||
bail!(
|
||||
"unknown backend '{}'; expected 'local' or 'hybrid'",
|
||||
effective
|
||||
);
|
||||
}
|
||||
if stored == "local" && effective == "hybrid" {
|
||||
bail!(
|
||||
"switching from local to hybrid mid-chat isn't supported; \
|
||||
regenerate the insight in hybrid mode if you want OpenRouter chat"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The model to continue a chat with when the request names none: the one
|
||||
/// the insight was generated with, but only on the same backend. After a
|
||||
/// switch, the stored id belongs to the other backend (a llama-swap slot is
|
||||
/// not an OpenRouter model), so the new backend's default is used instead.
|
||||
fn continuation_model(
|
||||
requested: Option<String>,
|
||||
stored_model: &str,
|
||||
stored_backend: &str,
|
||||
effective_backend: &str,
|
||||
) -> Option<String> {
|
||||
requested
|
||||
.or_else(|| (stored_backend == effective_backend).then(|| stored_model.to_string()))
|
||||
.filter(|m| !m.is_empty())
|
||||
}
|
||||
|
||||
/// Every distinct image in a transcript, in order of first appearance.
|
||||
fn transcript_images(messages: &[ChatMessage]) -> Vec<String> {
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
messages
|
||||
.iter()
|
||||
.flat_map(|m| m.images.iter().flatten())
|
||||
.filter(|img| seen.insert((*img).clone()))
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn image_cache_key(image: &str) -> u64 {
|
||||
use std::hash::{Hash, Hasher};
|
||||
let mut h = std::collections::hash_map::DefaultHasher::new();
|
||||
image.hash(&mut h);
|
||||
h.finish()
|
||||
}
|
||||
|
||||
/// What `apply_image_descriptions` took out of a transcript, so it can be
|
||||
/// put back before the transcript is saved.
|
||||
#[derive(Debug, Default)]
|
||||
struct ImageStash {
|
||||
entries: Vec<StashedImages>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct StashedImages {
|
||||
index: usize,
|
||||
original_content: String,
|
||||
rewritten_content: String,
|
||||
images: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
impl ImageStash {
|
||||
/// Put the original content and images back. Each entry is matched by
|
||||
/// index *and* by still carrying the rewritten text, so a turn that
|
||||
/// reshaped the transcript can't get the wrong message restored.
|
||||
fn restore(self, messages: &mut [ChatMessage]) {
|
||||
for e in self.entries {
|
||||
if let Some(m) = messages.get_mut(e.index)
|
||||
&& m.content == e.rewritten_content
|
||||
{
|
||||
m.content = e.original_content;
|
||||
m.images = e.images;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Replace each image in `messages` with its description, appended to the
|
||||
/// message text. Returns what was taken out, for `ImageStash::restore`.
|
||||
fn apply_image_descriptions(
|
||||
messages: &mut [ChatMessage],
|
||||
descriptions: &HashMap<String, String>,
|
||||
) -> ImageStash {
|
||||
let mut stash = ImageStash::default();
|
||||
for (idx, m) in messages.iter_mut().enumerate() {
|
||||
let Some(images) = m.images.as_ref().filter(|i| !i.is_empty()) else {
|
||||
continue;
|
||||
};
|
||||
let mut content = m.content.clone();
|
||||
for img in images {
|
||||
if let Some(desc) = descriptions.get(img) {
|
||||
content.push_str(&format!(
|
||||
"\n\nVisual description of the photo (from the local vision model):\n{desc}"
|
||||
));
|
||||
}
|
||||
}
|
||||
stash.entries.push(StashedImages {
|
||||
index: idx,
|
||||
original_content: std::mem::replace(&mut m.content, content.clone()),
|
||||
rewritten_content: content,
|
||||
images: m.images.take(),
|
||||
});
|
||||
}
|
||||
stash
|
||||
}
|
||||
|
||||
/// Pick the backend label for bootstrap. Bootstrap has no stored insight
|
||||
/// to defer to (that's continuation's behaviour), so the default is
|
||||
/// `"local"`. Returns an error if the supplied label is non-empty but
|
||||
@@ -2457,6 +2627,7 @@ pub async fn run_streaming_agentic_loop_with_entry(
|
||||
normalized: &str,
|
||||
user_id: i32,
|
||||
active_persona: &str,
|
||||
thread_contact: Option<i64>,
|
||||
max_iterations: usize,
|
||||
entry: &Arc<TurnEntry>,
|
||||
fitter: &mut ContextFitter,
|
||||
@@ -2466,6 +2637,13 @@ pub async fn run_streaming_agentic_loop_with_entry(
|
||||
let mut last_prompt_eval_count: Option<i32> = None;
|
||||
let mut last_eval_count: Option<i32> = None;
|
||||
let mut final_content = String::new();
|
||||
// Only tools in this turn's catalog may run. The catalog is how every
|
||||
// gate (per-persona corrections, practice mode's narrowed set, absent
|
||||
// backing tables) is expressed, and `execute_tool` dispatches on name
|
||||
// alone — without this a model that names a tool it was never offered
|
||||
// (one it saw in an earlier turn, or simply guessed) would still run it.
|
||||
let offered: std::collections::HashSet<String> =
|
||||
tools.iter().map(|t| t.function.name.clone()).collect();
|
||||
|
||||
for iteration in 0..max_iterations {
|
||||
if !entry.is_running() {
|
||||
@@ -2536,17 +2714,29 @@ pub async fn run_streaming_agentic_loop_with_entry(
|
||||
arguments: tool_call.function.arguments.clone(),
|
||||
})
|
||||
.await;
|
||||
let result = generator
|
||||
.execute_tool(
|
||||
&tool_call.function.name,
|
||||
&tool_call.function.arguments,
|
||||
backend,
|
||||
image_base64,
|
||||
normalized,
|
||||
user_id,
|
||||
active_persona,
|
||||
let result = if offered.contains(&tool_call.function.name) {
|
||||
generator
|
||||
.execute_tool(
|
||||
&tool_call.function.name,
|
||||
&tool_call.function.arguments,
|
||||
backend,
|
||||
image_base64,
|
||||
normalized,
|
||||
user_id,
|
||||
active_persona,
|
||||
thread_contact,
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
log::warn!(
|
||||
"refusing tool '{}': not offered this turn",
|
||||
tool_call.function.name
|
||||
);
|
||||
format!(
|
||||
"Error: tool '{}' is not available here.",
|
||||
tool_call.function.name
|
||||
)
|
||||
.await;
|
||||
};
|
||||
let (result_preview, result_truncated) = truncate_tool_result(&result);
|
||||
let _ = entry
|
||||
.push_event(ChatStreamEvent::ToolResult {
|
||||
@@ -3654,17 +3844,90 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cross_replay_rejects_local_to_hybrid() {
|
||||
let err = validate_cross_replay("local", "hybrid").unwrap_err();
|
||||
assert!(format!("{}", err).contains("local to hybrid"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cross_replay_allows_supported_transitions() {
|
||||
fn cross_replay_allows_every_known_transition() {
|
||||
assert!(validate_cross_replay("local", "local").is_ok());
|
||||
assert!(validate_cross_replay("hybrid", "hybrid").is_ok());
|
||||
// Hybrid → local replays the inlined description as plain text.
|
||||
assert!(validate_cross_replay("hybrid", "local").is_ok());
|
||||
// Local → hybrid swaps the photo for a local description first.
|
||||
assert!(validate_cross_replay("local", "hybrid").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn continuation_model_drops_the_stored_model_across_a_switch() {
|
||||
// Same backend: keep the model the insight was generated with.
|
||||
assert_eq!(
|
||||
continuation_model(None, "gemma4-31", "local", "local").as_deref(),
|
||||
Some("gemma4-31")
|
||||
);
|
||||
// Switched: a llama-swap slot name is not an OpenRouter model id.
|
||||
assert_eq!(
|
||||
continuation_model(None, "gemma4-31", "local", "hybrid"),
|
||||
None
|
||||
);
|
||||
// An explicit request always wins.
|
||||
assert_eq!(
|
||||
continuation_model(Some("x/y".into()), "gemma4-31", "local", "hybrid").as_deref(),
|
||||
Some("x/y")
|
||||
);
|
||||
assert_eq!(continuation_model(None, "", "local", "local"), None);
|
||||
}
|
||||
|
||||
fn with_image(text: &str, img: &str) -> ChatMessage {
|
||||
let mut m = ChatMessage::user(text.to_string());
|
||||
m.images = Some(vec![img.to_string()]);
|
||||
m
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transcript_images_lists_each_image_once() {
|
||||
let msgs = vec![
|
||||
ChatMessage::system("sys"),
|
||||
with_image("describe", "IMG_A"),
|
||||
ChatMessage::user("more"),
|
||||
with_image("again", "IMG_A"),
|
||||
];
|
||||
assert_eq!(transcript_images(&msgs), vec!["IMG_A".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn photo_is_swapped_for_text_then_restored_exactly() {
|
||||
let original = vec![
|
||||
ChatMessage::system("sys"),
|
||||
with_image("What's in this photo?", "IMG_A"),
|
||||
assistant_text("A beach."),
|
||||
ChatMessage::user("who's there?"),
|
||||
];
|
||||
let mut msgs = original.clone();
|
||||
let descriptions =
|
||||
HashMap::from([("IMG_A".to_string(), "Two people on sand.".to_string())]);
|
||||
|
||||
let stash = apply_image_descriptions(&mut msgs, &descriptions);
|
||||
// Nothing the text-only model receives carries image data.
|
||||
assert!(msgs.iter().all(|m| m.images.is_none()));
|
||||
assert!(msgs[1].content.starts_with("What's in this photo?"));
|
||||
assert!(msgs[1].content.contains("Two people on sand."));
|
||||
|
||||
// The turn appends its exchange, then the stash is restored before
|
||||
// saving: the stored transcript keeps the photo, unchanged.
|
||||
msgs.push(assistant_text("Two friends."));
|
||||
stash.restore(&mut msgs);
|
||||
assert_eq!(
|
||||
serde_json::to_string(&msgs[..original.len()]).unwrap(),
|
||||
serde_json::to_string(&original).unwrap()
|
||||
);
|
||||
assert_eq!(msgs[1].images, Some(vec!["IMG_A".to_string()]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restore_leaves_a_message_the_turn_changed() {
|
||||
let mut msgs = vec![with_image("q", "IMG_A")];
|
||||
let descriptions = HashMap::from([("IMG_A".to_string(), "desc".to_string())]);
|
||||
let stash = apply_image_descriptions(&mut msgs, &descriptions);
|
||||
msgs[0].content = "something else entirely".into();
|
||||
stash.restore(&mut msgs);
|
||||
assert_eq!(msgs[0].content, "something else entirely");
|
||||
assert!(msgs[0].images.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
+446
-22
@@ -286,6 +286,12 @@ pub struct ToolGateOpts {
|
||||
/// tool name still 404s, and the agent doesn't waste iterations
|
||||
/// trying corrections it isn't allowed to do.
|
||||
pub allow_agent_corrections: bool,
|
||||
/// Persona chats anchored on an SMS thread ("Continue with persona")
|
||||
/// get the thread tools: page the thread, look at an MMS photo, find
|
||||
/// library photos around a moment and look at one. Off everywhere else
|
||||
/// — the file chat already has its photo, and an open chat has no
|
||||
/// thread to page.
|
||||
pub sms_thread_tools: bool,
|
||||
}
|
||||
|
||||
impl InsightGenerator {
|
||||
@@ -421,6 +427,12 @@ impl InsightGenerator {
|
||||
/// tool-definition gating (no point offering `get_personal_place_at`
|
||||
/// when Apollo is unreachable) — exposed publicly so `insight_chat`
|
||||
/// can call `build_tool_definitions` with the same flag.
|
||||
/// The SMS-API client, for callers outside the tool loop (the anchored
|
||||
/// persona chat snapshots a thread with it at creation).
|
||||
pub fn sms_client(&self) -> &SmsApiClient {
|
||||
&self.sms_client
|
||||
}
|
||||
|
||||
pub fn apollo_enabled(&self) -> bool {
|
||||
self.apollo_client.is_enabled()
|
||||
}
|
||||
@@ -485,6 +497,7 @@ impl InsightGenerator {
|
||||
location_history_present,
|
||||
faces_present,
|
||||
allow_agent_corrections,
|
||||
sms_thread_tools: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1878,6 +1891,7 @@ Return ONLY the summary, nothing else."#,
|
||||
file_path: &str,
|
||||
user_id: i32,
|
||||
persona_id: &str,
|
||||
thread_contact: Option<i64>,
|
||||
) -> String {
|
||||
let model = backend.model();
|
||||
let backend_label = backend.kind.as_str();
|
||||
@@ -1924,7 +1938,19 @@ Return ONLY the summary, nothing else."#,
|
||||
self.tool_supersede_fact(arguments, user_id, persona_id, model, backend_label)
|
||||
.await
|
||||
}
|
||||
"get_current_datetime" => Self::tool_get_current_datetime(),
|
||||
"date_info" => {
|
||||
crate::ai::date_info::date_info(arguments, Utc::now(), self.display_zone())
|
||||
}
|
||||
"get_thread_messages" => {
|
||||
self.tool_get_thread_messages(arguments, thread_contact)
|
||||
.await
|
||||
}
|
||||
"look_at_message_image" => {
|
||||
self.tool_look_at_message_image(arguments, thread_contact, backend.local())
|
||||
.await
|
||||
}
|
||||
"get_photos_near_time" => self.tool_get_photos_near_time(arguments).await,
|
||||
"look_at_photo" => self.tool_look_at_photo(arguments, backend.local()).await,
|
||||
unknown => format!("Unknown tool: {}", unknown),
|
||||
};
|
||||
if result.starts_with("Error") || result.starts_with("No ") {
|
||||
@@ -2386,6 +2412,289 @@ Return ONLY the summary, nothing else."#,
|
||||
}
|
||||
}
|
||||
|
||||
/// Tool: get_thread_messages — page the anchored SMS thread from a
|
||||
/// message id. Lines are rendered exactly like the anchor snapshot so
|
||||
/// the model can keep paging from any id it sees.
|
||||
async fn tool_get_thread_messages(
|
||||
&self,
|
||||
args: &serde_json::Value,
|
||||
thread_contact: Option<i64>,
|
||||
) -> String {
|
||||
let contact_id = match Self::bound_thread(args, thread_contact) {
|
||||
Ok(c) => c,
|
||||
Err(e) => return e,
|
||||
};
|
||||
let Some(message_id) = args.get("message_id").and_then(|v| v.as_i64()) else {
|
||||
return "Error: missing required parameter 'message_id'".to_string();
|
||||
};
|
||||
let limit = args
|
||||
.get("limit")
|
||||
.and_then(|v| v.as_i64())
|
||||
.unwrap_or(30)
|
||||
.clamp(1, 80) as usize;
|
||||
let (before, after) = match args.get("direction").and_then(|v| v.as_str()) {
|
||||
Some("after") => (0, limit),
|
||||
Some("around") => (limit / 2, limit - limit / 2),
|
||||
_ => (limit, 0),
|
||||
};
|
||||
|
||||
match self
|
||||
.sms_client
|
||||
.fetch_thread_window(contact_id, message_id, before, after)
|
||||
.await
|
||||
{
|
||||
Ok(window) if window.messages.len() > 1 => {
|
||||
let rendered = crate::ai::sms_anchor::render_thread_window(
|
||||
&window,
|
||||
&user_display_name(),
|
||||
self.display_zone(),
|
||||
);
|
||||
format!(
|
||||
"Found {} messages with {} (▶ marks #{}):\n{}",
|
||||
window.messages.len(),
|
||||
window.contact.display_name(),
|
||||
message_id,
|
||||
rendered
|
||||
)
|
||||
}
|
||||
Ok(_) => "No more messages in that direction.".to_string(),
|
||||
Err(e) => {
|
||||
log::warn!("tool_get_thread_messages failed: {}", e);
|
||||
format!("Error fetching thread messages: {}", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Tool: look_at_message_image — download an MMS photo from SMS-API and
|
||||
/// describe it with the local vision model.
|
||||
async fn tool_look_at_message_image(
|
||||
&self,
|
||||
args: &serde_json::Value,
|
||||
thread_contact: Option<i64>,
|
||||
local: &dyn LlmClient,
|
||||
) -> String {
|
||||
let contact_id = match Self::bound_thread(args, thread_contact) {
|
||||
Ok(c) => c,
|
||||
Err(e) => return e,
|
||||
};
|
||||
let Some(message_id) = args.get("message_id").and_then(|v| v.as_i64()) else {
|
||||
return "Error: missing required parameter 'message_id'".to_string();
|
||||
};
|
||||
let media_index = args
|
||||
.get("media_index")
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(0) as usize;
|
||||
|
||||
let bytes = match self
|
||||
.sms_client
|
||||
.fetch_mms_image(contact_id, message_id, media_index)
|
||||
.await
|
||||
{
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
return format!(
|
||||
"Error: couldn't look at that attachment ({e}); only photos can be viewed."
|
||||
);
|
||||
}
|
||||
};
|
||||
let image_base64 = match Self::encode_image_bytes_for_vision(&bytes) {
|
||||
Ok(b64) => b64,
|
||||
Err(_) => {
|
||||
return "Error: that attachment isn't a viewable image (videos, audio and some \
|
||||
formats can't be looked at)."
|
||||
.to_string();
|
||||
}
|
||||
};
|
||||
match local.describe_image(&image_base64).await {
|
||||
Ok(desc) => desc,
|
||||
Err(e) => format!("Error describing attachment: {}", e),
|
||||
}
|
||||
}
|
||||
|
||||
/// Tool: get_photos_near_time — library photos within ±hours of a
|
||||
/// moment, closest first, with tags and recognised people.
|
||||
async fn tool_get_photos_near_time(&self, args: &serde_json::Value) -> String {
|
||||
let zone = self.display_zone();
|
||||
let server_zone = crate::camera_tz::server_zone();
|
||||
let center_arg = args
|
||||
.get("timestamp")
|
||||
.filter(|v| !v.is_null())
|
||||
.or_else(|| args.get("datetime"));
|
||||
let Some(center) = center_arg.and_then(|v| crate::ai::date_info::parse_center(v, zone))
|
||||
else {
|
||||
return "Error: pass 'timestamp' (unix seconds) or 'datetime' (YYYY-MM-DD HH:MM)"
|
||||
.to_string();
|
||||
};
|
||||
let hours = args
|
||||
.get("hours_radius")
|
||||
.and_then(|v| v.as_i64())
|
||||
.unwrap_or(3)
|
||||
.clamp(1, 72);
|
||||
let limit = args
|
||||
.get("limit")
|
||||
.and_then(|v| v.as_i64())
|
||||
.unwrap_or(15)
|
||||
.clamp(1, 40) as usize;
|
||||
|
||||
let range = crate::date_view::InstantRange::new(
|
||||
Some(center - hours * 3600),
|
||||
Some(center + hours * 3600),
|
||||
zone,
|
||||
server_zone,
|
||||
);
|
||||
let rows = {
|
||||
let mut dao = self.exif_dao.lock().expect("Unable to lock ExifDao");
|
||||
match dao.query_by_exif(None, None, None, None, None, range) {
|
||||
Ok(r) => r,
|
||||
Err(e) => return format!("Error querying photos: {:?}", e),
|
||||
}
|
||||
};
|
||||
|
||||
// Closest first; the same bytes in two libraries list once.
|
||||
let mut dated: Vec<(i64, crate::database::models::ImageExif)> = rows
|
||||
.into_iter()
|
||||
.filter_map(|r| {
|
||||
crate::date_view::row_true_instant(&r, zone, server_zone).map(|t| (t, r))
|
||||
})
|
||||
.collect();
|
||||
dated.sort_by_key(|(t, _)| (t - center).abs());
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
dated.retain(|(_, r)| match &r.content_hash {
|
||||
Some(h) => seen.insert(h.clone()),
|
||||
None => true,
|
||||
});
|
||||
let total = dated.len();
|
||||
if total == 0 {
|
||||
return format!("No photos within ±{} hours of that moment.", hours);
|
||||
}
|
||||
|
||||
let mut lines = Vec::new();
|
||||
for (_, row) in dated.into_iter().take(limit) {
|
||||
let when = crate::date_view::row_display_wall(&row, zone, server_zone)
|
||||
.map(|w| w.format("%Y-%m-%d %H:%M").to_string())
|
||||
.unwrap_or_else(|| "unknown time".to_string());
|
||||
// Tags follow the bytes: union over every path holding the same
|
||||
// content, so a copy in another library shows its tags too.
|
||||
let sibling_paths = {
|
||||
let mut dao = self.exif_dao.lock().expect("Unable to lock ExifDao");
|
||||
dao.get_rel_paths_sharing_content(row.library_id, &row.file_path)
|
||||
.unwrap_or_else(|_| vec![row.file_path.clone()])
|
||||
};
|
||||
let tags: Vec<String> = {
|
||||
let mut dao = self.tag_dao.lock().expect("Unable to lock TagDao");
|
||||
let mut names: Vec<String> = dao
|
||||
.get_tags_for_paths(&sibling_paths)
|
||||
.map(|t| t.into_iter().map(|t| t.name).collect())
|
||||
.unwrap_or_default();
|
||||
names.sort();
|
||||
names.dedup();
|
||||
names
|
||||
};
|
||||
let people: Vec<String> = match &row.content_hash {
|
||||
Some(hash) => {
|
||||
let mut dao = self.face_dao.lock().expect("Unable to lock FaceDao");
|
||||
let mut names: Vec<String> = dao
|
||||
.list_for_content_hash(hash)
|
||||
.map(|faces| faces.into_iter().filter_map(|f| f.person_name).collect())
|
||||
.unwrap_or_default();
|
||||
names.sort();
|
||||
names.dedup();
|
||||
names
|
||||
}
|
||||
None => Vec::new(),
|
||||
};
|
||||
let mut line = format!("- {} — {}", row.file_path, when);
|
||||
if !tags.is_empty() {
|
||||
line.push_str(&format!(" · tags: {}", tags.join(", ")));
|
||||
}
|
||||
if !people.is_empty() {
|
||||
line.push_str(&format!(" · people: {}", people.join(", ")));
|
||||
}
|
||||
if crate::file_types::is_video_file(std::path::Path::new(&row.file_path)) {
|
||||
line.push_str(" · video");
|
||||
}
|
||||
lines.push(line);
|
||||
}
|
||||
format!(
|
||||
"{}\n{}",
|
||||
Self::found_header(total, lines.len(), "photos"),
|
||||
lines.join("\n")
|
||||
)
|
||||
}
|
||||
|
||||
/// Tool: look_at_photo — describe a library photo by path with the
|
||||
/// local vision model.
|
||||
async fn tool_look_at_photo(&self, args: &serde_json::Value, local: &dyn LlmClient) -> String {
|
||||
let file_path = match args.get("file_path").and_then(|v| v.as_str()) {
|
||||
Some(p) if Self::photo_path_is_safe(p.trim()) => p.trim().to_string(),
|
||||
Some(_) => return "Error: file_path must be a library-relative path".to_string(),
|
||||
None => return "Error: missing required parameter 'file_path'".to_string(),
|
||||
};
|
||||
if crate::file_types::is_video_file(std::path::Path::new(&file_path)) {
|
||||
return "Error: that's a video — only photos can be looked at.".to_string();
|
||||
}
|
||||
// Only photos the library has indexed: excluded folders (EXCLUDED_DIRS,
|
||||
// e.g. `private`) are never indexed, so a guessed path into one is
|
||||
// refused rather than read off disk.
|
||||
if self.fetch_exif(&file_path).is_none() {
|
||||
return "Error: that isn't a photo in the library — use a file_path from \
|
||||
get_photos_near_time."
|
||||
.to_string();
|
||||
}
|
||||
let image_base64 = match self.load_image_as_base64(&file_path) {
|
||||
Ok(b) => b,
|
||||
Err(e) => return format!("Error loading photo: {}", e),
|
||||
};
|
||||
match local.describe_image(&image_base64).await {
|
||||
Ok(desc) => desc,
|
||||
Err(e) => format!("Error describing photo: {}", e),
|
||||
}
|
||||
}
|
||||
|
||||
/// The thread a thread tool may read: the one the chat was started from,
|
||||
/// fixed server-side. A `contact_id` the model passes is only checked
|
||||
/// against it — never trusted — so a chat (practice mode especially) can
|
||||
/// never be talked into reading another conversation.
|
||||
fn bound_thread(args: &serde_json::Value, thread_contact: Option<i64>) -> Result<i64, String> {
|
||||
let Some(bound) = thread_contact else {
|
||||
return Err(
|
||||
"Error: thread tools are only available in a chat started from a \
|
||||
text thread."
|
||||
.to_string(),
|
||||
);
|
||||
};
|
||||
match args.get("contact_id").and_then(|v| v.as_i64()) {
|
||||
Some(asked) if asked != bound => {
|
||||
Err("Error: only this chat's own text thread can be read here.".to_string())
|
||||
}
|
||||
_ => Ok(bound),
|
||||
}
|
||||
}
|
||||
|
||||
/// A model-supplied path is joined onto each library root, so refuse
|
||||
/// anything that could climb out of it.
|
||||
fn photo_path_is_safe(path: &str) -> bool {
|
||||
use std::path::{Component, Path};
|
||||
!path.is_empty()
|
||||
&& Path::new(path)
|
||||
.components()
|
||||
.all(|c| matches!(c, Component::Normal(_) | Component::CurDir))
|
||||
}
|
||||
|
||||
/// Decode arbitrary image bytes (an MMS attachment), shrink to the
|
||||
/// vision models' 1024px working size and re-encode as base64 JPEG —
|
||||
/// the same shape `load_image_as_base64` produces for library files.
|
||||
fn encode_image_bytes_for_vision(bytes: &[u8]) -> Result<String> {
|
||||
use image::imageops::FilterType;
|
||||
let img = image::load_from_memory(bytes)?;
|
||||
// JPEG has no alpha channel; a PNG screenshot would fail to encode.
|
||||
let resized =
|
||||
image::DynamicImage::ImageRgb8(img.resize(1024, 1024, FilterType::Lanczos3).to_rgb8());
|
||||
let mut buffer = Vec::new();
|
||||
resized.write_to(&mut Cursor::new(&mut buffer), ImageFormat::Jpeg)?;
|
||||
Ok(base64::engine::general_purpose::STANDARD.encode(&buffer))
|
||||
}
|
||||
|
||||
/// Tool: get_calendar_events — fetch calendar events near a date
|
||||
async fn tool_get_calendar_events(&self, args: &serde_json::Value) -> String {
|
||||
let date_str = match args.get("date").and_then(|v| v.as_str()) {
|
||||
@@ -3433,21 +3742,11 @@ Return ONLY the summary, nothing else."#,
|
||||
}
|
||||
}
|
||||
|
||||
/// Tool: get_current_datetime — returns the current local date and time
|
||||
fn tool_get_current_datetime() -> String {
|
||||
let now = Local::now();
|
||||
format!(
|
||||
"Current date/time: {} ({})",
|
||||
now.format("%Y-%m-%d %H:%M:%S %Z"),
|
||||
now.format("%A")
|
||||
)
|
||||
}
|
||||
|
||||
// ── Agentic insight generation ──────────────────────────────────────
|
||||
|
||||
/// Build the list of tool definitions for the agentic loop, gated by
|
||||
/// `opts`. Always-on tools: `search_messages`, `get_sms_messages`,
|
||||
/// `get_file_tags`, `reverse_geocode`, `get_current_datetime`, the
|
||||
/// `get_file_tags`, `reverse_geocode`, `date_info`, the
|
||||
/// five knowledge-memory tools. Conditional: `describe_photo` (vision
|
||||
/// model), `get_personal_place_at` (Apollo configured), `search_rag`
|
||||
/// (daily_summaries populated), `get_calendar_events` (calendar
|
||||
@@ -3773,14 +4072,95 @@ Return ONLY the summary, nothing else."#,
|
||||
}
|
||||
|
||||
tools.push(Tool::function(
|
||||
"get_current_datetime",
|
||||
"Get the current date and time. Useful when reasoning about how long ago a photo was taken.",
|
||||
"date_info",
|
||||
"Exact date arithmetic — use it instead of working dates out yourself. With no arguments: \
|
||||
the current date and time. With `date`: that day's weekday and how long ago (or until) it \
|
||||
is. With `date` and `other_date`: the gap between them in days, weeks and years/months/days \
|
||||
(and hours, when both have times). With `date` and `add_days`: the date that many days \
|
||||
after (or, negative, before). Dates are YYYY-MM-DD, YYYY-MM-DD HH:MM, or unix seconds, in \
|
||||
the user's local time. \
|
||||
Examples: `{}` — now. `{date: \"2019-07-03\"}` — weekday and how long ago. \
|
||||
`{date: \"2019-07-03\", other_date: \"2019-08-14\"}` — time between. \
|
||||
`{date: \"2019-07-03\", add_days: 90}` — the date 90 days later.",
|
||||
serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {}
|
||||
"properties": {
|
||||
"date": { "type": "string", "description": "YYYY-MM-DD, YYYY-MM-DD HH:MM, or unix seconds." },
|
||||
"other_date": { "type": "string", "description": "Second date, to measure the gap between the two." },
|
||||
"add_days": { "type": "integer", "description": "Days to add to `date` (negative to go back)." }
|
||||
}
|
||||
}),
|
||||
));
|
||||
|
||||
if opts.sms_thread_tools {
|
||||
tools.push(Tool::function(
|
||||
"get_thread_messages",
|
||||
"Read more of the SMS thread this chat was opened from. Pages from a message id in the \
|
||||
thread (the #numbers in the transcript) — `before` for older messages, `after` for what \
|
||||
came next, `around` for both sides. Lines carry their own #id, so call again from the \
|
||||
oldest/newest id to keep paging. \
|
||||
Example: `{message_id: 812, direction: \"before\", limit: 30}`.",
|
||||
serde_json::json!({
|
||||
"type": "object",
|
||||
"required": ["message_id"],
|
||||
"properties": {
|
||||
"message_id": { "type": "integer", "description": "Message id to page from." },
|
||||
"direction": { "type": "string", "enum": ["before", "after", "around"], "description": "Which side of message_id to read (default before)." },
|
||||
"limit": { "type": "integer", "description": "Max messages (default 30, max 80)." }
|
||||
}
|
||||
}),
|
||||
));
|
||||
|
||||
tools.push(Tool::function(
|
||||
"look_at_message_image",
|
||||
"Look at a photo that was sent in the SMS thread, using the local vision model. Returns a \
|
||||
visual description. Attachments appear in the transcript as `image #N` on a message line — \
|
||||
pass that message's id and N as media_index. Videos and audio can't be viewed. \
|
||||
Example: `{message_id: 815, media_index: 0}`.",
|
||||
serde_json::json!({
|
||||
"type": "object",
|
||||
"required": ["message_id"],
|
||||
"properties": {
|
||||
"message_id": { "type": "integer", "description": "Id of the message carrying the photo." },
|
||||
"media_index": { "type": "integer", "description": "Which attachment on that message (default 0)." }
|
||||
}
|
||||
}),
|
||||
));
|
||||
|
||||
tools.push(Tool::function(
|
||||
"get_photos_near_time",
|
||||
"List photos and videos from the user's library taken around a moment, closest first, with \
|
||||
their user-applied tags and the names of people recognised in them. Use to see what was \
|
||||
happening around a message (pass its time). Follow up with `look_at_photo` to see one. \
|
||||
Pass either `timestamp` (unix seconds) or `datetime` (local, \"YYYY-MM-DD HH:MM\" or \
|
||||
\"YYYY-MM-DD\"). \
|
||||
Example: `{datetime: \"2019-06-12 18:30\", hours_radius: 4}`.",
|
||||
serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"timestamp": { "type": "integer", "description": "Centre moment, unix seconds." },
|
||||
"datetime": { "type": "string", "description": "Centre moment in local time, YYYY-MM-DD HH:MM (or YYYY-MM-DD for the whole day)." },
|
||||
"hours_radius": { "type": "integer", "description": "Hours before and after to include (default 3, max 72)." },
|
||||
"limit": { "type": "integer", "description": "Max photos (default 15, max 40)." }
|
||||
}
|
||||
}),
|
||||
));
|
||||
|
||||
tools.push(Tool::function(
|
||||
"look_at_photo",
|
||||
"Look at one photo from the user's library with the local vision model and return a visual \
|
||||
description — people, place, activity, mood. Pass a file_path from `get_photos_near_time`. \
|
||||
Example: `{file_path: \"2019/06/IMG_4242.jpg\"}`.",
|
||||
serde_json::json!({
|
||||
"type": "object",
|
||||
"required": ["file_path"],
|
||||
"properties": {
|
||||
"file_path": { "type": "string", "description": "Library-relative path of the photo." }
|
||||
}
|
||||
}),
|
||||
));
|
||||
}
|
||||
|
||||
if opts.has_vision {
|
||||
tools.push(Tool::function(
|
||||
"describe_photo",
|
||||
@@ -3915,7 +4295,8 @@ Return ONLY the summary, nothing else."#,
|
||||
&& let Ok(n) = n_str.parse::<usize>()
|
||||
{
|
||||
let kind = match tool_name {
|
||||
"search_messages" | "get_sms_messages" => "messages",
|
||||
"search_messages" | "get_sms_messages" | "get_thread_messages" => "messages",
|
||||
"get_photos_near_time" => "photos",
|
||||
"get_calendar_events" => "events",
|
||||
"get_location_history" => "location records",
|
||||
_ => "results",
|
||||
@@ -3963,7 +4344,11 @@ Return ONLY the summary, nothing else."#,
|
||||
.find_map(|tok| tok.strip_prefix("ID:"))
|
||||
.map(|id| format!("stored id={}", id.trim_end_matches(',')))
|
||||
.unwrap_or_else(|| "stored".to_string()),
|
||||
"get_current_datetime" => "time noted".to_string(),
|
||||
"date_info" => "date worked out".to_string(),
|
||||
"look_at_message_image" | "look_at_photo" => {
|
||||
let short: String = raw.chars().take(80).collect();
|
||||
format!("looked: \"{}...\"", short)
|
||||
}
|
||||
_ => format!("{} chars", raw.len()),
|
||||
}
|
||||
}
|
||||
@@ -4493,6 +4878,7 @@ Return ONLY the summary, nothing else."#,
|
||||
&file_path,
|
||||
user_id,
|
||||
&persona_id,
|
||||
None,
|
||||
)
|
||||
.instrument(loop_span.clone())
|
||||
.await;
|
||||
@@ -4545,6 +4931,7 @@ Return ONLY the summary, nothing else."#,
|
||||
&file_path,
|
||||
user_id,
|
||||
&persona_id,
|
||||
None,
|
||||
)
|
||||
.instrument(loop_span.clone())
|
||||
.await;
|
||||
@@ -4733,6 +5120,7 @@ Return ONLY the summary, nothing else."#,
|
||||
"",
|
||||
0,
|
||||
"",
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
messages.push(ChatMessage::tool_result(result));
|
||||
@@ -4774,6 +5162,7 @@ Return ONLY the summary, nothing else."#,
|
||||
"",
|
||||
0,
|
||||
"",
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
messages.push(ChatMessage::tool_result(result));
|
||||
@@ -4882,6 +5271,7 @@ mod tests {
|
||||
location_history_present: false,
|
||||
faces_present: false,
|
||||
allow_agent_corrections: false,
|
||||
sms_thread_tools: false,
|
||||
};
|
||||
let tools = InsightGenerator::build_tool_definitions(opts);
|
||||
let names: Vec<&str> = tools.iter().map(|t| t.function.name.as_str()).collect();
|
||||
@@ -4891,7 +5281,7 @@ mod tests {
|
||||
assert!(names.contains(&"get_sms_messages"));
|
||||
assert!(names.contains(&"get_file_tags"));
|
||||
assert!(names.contains(&"reverse_geocode"));
|
||||
assert!(names.contains(&"get_current_datetime"));
|
||||
assert!(names.contains(&"date_info"));
|
||||
assert!(names.contains(&"recall_entities"));
|
||||
assert!(names.contains(&"recall_facts_for_photo"));
|
||||
assert!(names.contains(&"recall_facts_for_entity"));
|
||||
@@ -4908,6 +5298,11 @@ mod tests {
|
||||
// Agent-correction tools are absent without the gate.
|
||||
assert!(!names.contains(&"update_fact"));
|
||||
assert!(!names.contains(&"supersede_fact"));
|
||||
// SMS-thread tools only exist for an anchored persona chat.
|
||||
assert!(!names.contains(&"get_thread_messages"));
|
||||
assert!(!names.contains(&"look_at_message_image"));
|
||||
assert!(!names.contains(&"get_photos_near_time"));
|
||||
assert!(!names.contains(&"look_at_photo"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4920,6 +5315,7 @@ mod tests {
|
||||
location_history_present: true,
|
||||
faces_present: true,
|
||||
allow_agent_corrections: true,
|
||||
sms_thread_tools: true,
|
||||
};
|
||||
let tools = InsightGenerator::build_tool_definitions(opts);
|
||||
let names: Vec<&str> = tools.iter().map(|t| t.function.name.as_str()).collect();
|
||||
@@ -4931,6 +5327,34 @@ mod tests {
|
||||
assert!(names.contains(&"get_faces_in_photo"));
|
||||
assert!(names.contains(&"update_fact"));
|
||||
assert!(names.contains(&"supersede_fact"));
|
||||
assert!(names.contains(&"get_thread_messages"));
|
||||
assert!(names.contains(&"look_at_message_image"));
|
||||
assert!(names.contains(&"get_photos_near_time"));
|
||||
assert!(names.contains(&"look_at_photo"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thread_tools_only_read_the_chats_own_thread() {
|
||||
use serde_json::json;
|
||||
// The bound thread is used; the model needn't (and can't) choose.
|
||||
assert_eq!(InsightGenerator::bound_thread(&json!({}), Some(42)), Ok(42));
|
||||
assert_eq!(
|
||||
InsightGenerator::bound_thread(&json!({"contact_id": 42}), Some(42)),
|
||||
Ok(42)
|
||||
);
|
||||
// Another thread is refused, even if the model names it.
|
||||
assert!(InsightGenerator::bound_thread(&json!({"contact_id": 7}), Some(42)).is_err());
|
||||
// No bound thread (not a thread chat): nothing to read.
|
||||
assert!(InsightGenerator::bound_thread(&json!({"contact_id": 42}), None).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn photo_path_is_safe_rejects_escapes() {
|
||||
assert!(InsightGenerator::photo_path_is_safe("2019/06/IMG_1.jpg"));
|
||||
assert!(!InsightGenerator::photo_path_is_safe("../etc/passwd"));
|
||||
assert!(!InsightGenerator::photo_path_is_safe("2019/../../x.jpg"));
|
||||
assert!(!InsightGenerator::photo_path_is_safe("/etc/passwd"));
|
||||
assert!(!InsightGenerator::photo_path_is_safe(""));
|
||||
}
|
||||
|
||||
fn place(name: &str, description: &str) -> ApolloPlace {
|
||||
@@ -5394,13 +5818,13 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summarize_current_datetime() {
|
||||
fn summarize_date_info() {
|
||||
assert_eq!(
|
||||
InsightGenerator::summarize_tool_result(
|
||||
"get_current_datetime",
|
||||
"Current date/time: 2024-01-15 12:00:00 PST (Monday)"
|
||||
"date_info",
|
||||
"Now: 2024-01-15 12:00 (Monday), America/New_York"
|
||||
),
|
||||
"time noted"
|
||||
"date worked out"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ pub mod apollo_client;
|
||||
pub mod backend;
|
||||
pub mod clip_client;
|
||||
pub mod daily_summary_job;
|
||||
pub mod date_info;
|
||||
pub mod face_client;
|
||||
pub mod gpu;
|
||||
pub mod handlers;
|
||||
@@ -15,6 +16,7 @@ pub mod ollama;
|
||||
pub mod openrouter;
|
||||
pub mod persona_chat;
|
||||
pub mod pronunciation;
|
||||
pub mod sms_anchor;
|
||||
pub mod sms_client;
|
||||
pub mod tts;
|
||||
pub mod turn_registry;
|
||||
|
||||
+59
-82
@@ -48,6 +48,10 @@ impl<T> CachedEntry<T> {
|
||||
}
|
||||
}
|
||||
|
||||
/// The error every image-carrying OpenRouter request fails with.
|
||||
pub const IMAGES_STAY_LOCAL: &str = "refusing to send image data to OpenRouter: photos stay on \
|
||||
the local network (describe them with the local vision model and send the text)";
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
static ref MODEL_CAPABILITIES_CACHE: Arc<Mutex<HashMap<String, CachedEntry<Vec<ModelCapabilities>>>>> =
|
||||
Arc::new(Mutex::new(HashMap::new()));
|
||||
@@ -58,7 +62,11 @@ lazy_static::lazy_static! {
|
||||
/// Translates canonical `ChatMessage` / `Tool` shapes to OpenAI wire format:
|
||||
/// - Tool-call `arguments` serialized as JSON-encoded strings (vs Ollama's
|
||||
/// native JSON).
|
||||
/// - Image content rewritten into content-parts array with `image_url` entries.
|
||||
/// - **Never sends images.** Photos stay on the local network: hybrid mode
|
||||
/// describes them with the local vision model and sends only the text.
|
||||
/// A message that still carries image data is refused (see
|
||||
/// [`IMAGES_STAY_LOCAL`]) rather than uploaded, so a caller bug fails a
|
||||
/// turn instead of leaking a photo.
|
||||
/// - `role=tool` messages attach a `tool_call_id` inferred from the preceding
|
||||
/// assistant turn's tool call.
|
||||
#[derive(Clone)]
|
||||
@@ -160,7 +168,18 @@ impl OpenRouterClient {
|
||||
///
|
||||
/// Walks in order so it can attach `tool_call_id` to `role=tool` messages
|
||||
/// based on the most recent assistant turn's tool call.
|
||||
fn messages_to_openai(messages: &[ChatMessage]) -> Vec<Value> {
|
||||
///
|
||||
/// Errors if any message carries image data: this is the one choke point
|
||||
/// every OpenRouter request passes through, so the "photos never leave
|
||||
/// the network" rule is enforced here rather than trusted to callers.
|
||||
fn messages_to_openai(messages: &[ChatMessage]) -> Result<Vec<Value>> {
|
||||
if messages
|
||||
.iter()
|
||||
.any(|m| m.images.as_ref().is_some_and(|imgs| !imgs.is_empty()))
|
||||
{
|
||||
log::error!("{IMAGES_STAY_LOCAL}");
|
||||
bail!(IMAGES_STAY_LOCAL);
|
||||
}
|
||||
let mut out = Vec::with_capacity(messages.len());
|
||||
let mut last_tool_call_ids: Vec<String> = Vec::new();
|
||||
let mut next_tool_result_idx: usize = 0;
|
||||
@@ -169,26 +188,8 @@ impl OpenRouterClient {
|
||||
let mut obj = serde_json::Map::new();
|
||||
obj.insert("role".into(), Value::String(msg.role.clone()));
|
||||
|
||||
// Content: string OR content-parts array (when images present).
|
||||
match &msg.images {
|
||||
Some(images) if !images.is_empty() => {
|
||||
let mut parts: Vec<Value> = Vec::new();
|
||||
if !msg.content.is_empty() {
|
||||
parts.push(json!({"type": "text", "text": msg.content}));
|
||||
}
|
||||
for img in images {
|
||||
let url = image_to_data_url(img);
|
||||
parts.push(json!({
|
||||
"type": "image_url",
|
||||
"image_url": { "url": url }
|
||||
}));
|
||||
}
|
||||
obj.insert("content".into(), Value::Array(parts));
|
||||
}
|
||||
_ => {
|
||||
obj.insert("content".into(), Value::String(msg.content.clone()));
|
||||
}
|
||||
}
|
||||
// Text only — images were refused above.
|
||||
obj.insert("content".into(), Value::String(msg.content.clone()));
|
||||
|
||||
// Assistant message with tool_calls: stringify arguments, remember
|
||||
// the ids so the subsequent tool messages can reference them.
|
||||
@@ -233,7 +234,7 @@ impl OpenRouterClient {
|
||||
out.push(Value::Object(obj));
|
||||
}
|
||||
|
||||
out
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Parse an OpenAI-compatible assistant message back into canonical shape.
|
||||
@@ -367,7 +368,7 @@ impl LlmClient for OpenRouterClient {
|
||||
body.insert("model".into(), Value::String(self.primary_model.clone()));
|
||||
body.insert(
|
||||
"messages".into(),
|
||||
Value::Array(Self::messages_to_openai(&messages)),
|
||||
Value::Array(Self::messages_to_openai(&messages)?),
|
||||
);
|
||||
body.insert("stream".into(), Value::Bool(true));
|
||||
// Ask for usage data in the final chunk (OpenAI + OpenRouter
|
||||
@@ -633,24 +634,12 @@ impl LlmClient for OpenRouterClient {
|
||||
Ok(parsed.data.into_iter().map(|i| i.embedding).collect())
|
||||
}
|
||||
|
||||
async fn describe_image(&self, image_base64: &str) -> Result<String> {
|
||||
// Own span with operation=describe_image — the funnel into
|
||||
// chat_with_tools_impl keeps the request on a single span.
|
||||
let span = self.start_request_span("describe_image", &self.primary_model);
|
||||
let prompt = "Briefly describe what you see in this image in 1-2 sentences. \
|
||||
Focus on the people, location, and activity.";
|
||||
let mut messages: Vec<ChatMessage> = Vec::new();
|
||||
messages.push(ChatMessage::system(
|
||||
"You are a scene description assistant. Be concise and factual.",
|
||||
));
|
||||
let mut user = ChatMessage::user(prompt);
|
||||
user.images = Some(vec![image_base64.to_string()]);
|
||||
messages.push(user);
|
||||
|
||||
let (reply, _, _) = self
|
||||
.chat_with_tools_impl(Some(&span), messages, Vec::new())
|
||||
.await?;
|
||||
Ok(reply.content)
|
||||
async fn describe_image(&self, _image_base64: &str) -> Result<String> {
|
||||
// Describing an image means uploading it. Hybrid mode always
|
||||
// describes on the local backend (`ResolvedBackend::local`), so this
|
||||
// is never the intended path — refuse loudly if something reaches it.
|
||||
log::error!("{IMAGES_STAY_LOCAL}");
|
||||
bail!(IMAGES_STAY_LOCAL)
|
||||
}
|
||||
|
||||
async fn list_models(&self) -> Result<Vec<ModelCapabilities>> {
|
||||
@@ -720,7 +709,7 @@ impl OpenRouterClient {
|
||||
body.insert("model".into(), Value::String(self.primary_model.clone()));
|
||||
body.insert(
|
||||
"messages".into(),
|
||||
Value::Array(Self::messages_to_openai(&messages)),
|
||||
Value::Array(Self::messages_to_openai(&messages)?),
|
||||
);
|
||||
body.insert("stream".into(), Value::Bool(false));
|
||||
if !tools.is_empty() {
|
||||
@@ -871,15 +860,6 @@ fn find_double_newline(buf: &[u8]) -> Option<usize> {
|
||||
None
|
||||
}
|
||||
|
||||
/// Build a `data:` URL if the provided string is raw base64, otherwise pass it through.
|
||||
fn image_to_data_url(img: &str) -> String {
|
||||
if img.starts_with("data:") {
|
||||
img.to_string()
|
||||
} else {
|
||||
format!("data:image/jpeg;base64,{}", img)
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_model_capabilities(m: &Value) -> ModelCapabilities {
|
||||
let name = m
|
||||
.get("id")
|
||||
@@ -926,7 +906,7 @@ mod tests {
|
||||
msg.tool_calls.as_mut().unwrap()[0].function.arguments =
|
||||
json!({"query": "hello", "limit": 5});
|
||||
|
||||
let wire = OpenRouterClient::messages_to_openai(&[msg]);
|
||||
let wire = OpenRouterClient::messages_to_openai(&[msg]).unwrap();
|
||||
let tcs = wire[0]
|
||||
.get("tool_calls")
|
||||
.and_then(|v| v.as_array())
|
||||
@@ -986,40 +966,37 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn images_become_content_parts() {
|
||||
let mut msg = ChatMessage::user("What is in this photo?");
|
||||
msg.images = Some(vec!["BASE64DATA".into()]);
|
||||
|
||||
let wire = OpenRouterClient::messages_to_openai(&[msg]);
|
||||
let content = wire[0].get("content").and_then(|v| v.as_array()).unwrap();
|
||||
assert_eq!(content.len(), 2);
|
||||
assert_eq!(content[0]["type"], "text");
|
||||
assert_eq!(content[0]["text"], "What is in this photo?");
|
||||
assert_eq!(content[1]["type"], "image_url");
|
||||
assert_eq!(
|
||||
content[1]["image_url"]["url"],
|
||||
"data:image/jpeg;base64,BASE64DATA"
|
||||
);
|
||||
fn any_image_anywhere_in_the_transcript_is_refused() {
|
||||
// Not just the latest message: a replayed transcript whose first
|
||||
// user turn still holds the photo must be refused too.
|
||||
let mut first = ChatMessage::user("What is in this photo?");
|
||||
first.images = Some(vec!["BASE64DATA".into()]);
|
||||
let later = ChatMessage::user("and then?");
|
||||
let err = OpenRouterClient::messages_to_openai(&[first, later]).unwrap_err();
|
||||
assert!(err.to_string().contains("photos stay on the local network"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_url_images_pass_through_unchanged() {
|
||||
let mut msg = ChatMessage::user("");
|
||||
msg.images = Some(vec!["data:image/png;base64,ABCDEF".into()]);
|
||||
let wire = OpenRouterClient::messages_to_openai(&[msg]);
|
||||
let content = wire[0].get("content").and_then(|v| v.as_array()).unwrap();
|
||||
// No text part when content is empty.
|
||||
assert_eq!(content.len(), 1);
|
||||
assert_eq!(
|
||||
content[0]["image_url"]["url"],
|
||||
"data:image/png;base64,ABCDEF"
|
||||
);
|
||||
fn an_empty_image_list_is_not_an_image() {
|
||||
let mut msg = ChatMessage::user("hi");
|
||||
msg.images = Some(Vec::new());
|
||||
assert!(OpenRouterClient::messages_to_openai(&[msg]).is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn describe_image_refuses_without_a_network_call() {
|
||||
// Points at a closed port: if the guard ever stopped refusing, this
|
||||
// would fail with a connection error instead of the guard's.
|
||||
let client =
|
||||
OpenRouterClient::new("key".into(), Some("http://127.0.0.1:9".into()), "m".into());
|
||||
let err = client.describe_image("BASE64DATA").await.unwrap_err();
|
||||
assert!(err.to_string().contains("photos stay on the local network"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_only_message_stays_string() {
|
||||
let msg = ChatMessage::user("hello");
|
||||
let wire = OpenRouterClient::messages_to_openai(&[msg]);
|
||||
let wire = OpenRouterClient::messages_to_openai(&[msg]).unwrap();
|
||||
assert_eq!(wire[0]["content"], "hello");
|
||||
assert!(wire[0]["content"].as_str().is_some());
|
||||
}
|
||||
@@ -1040,7 +1017,7 @@ mod tests {
|
||||
};
|
||||
let tool_result = ChatMessage::tool_result("found it");
|
||||
|
||||
let wire = OpenRouterClient::messages_to_openai(&[assistant, tool_result]);
|
||||
let wire = OpenRouterClient::messages_to_openai(&[assistant, tool_result]).unwrap();
|
||||
assert_eq!(wire[1]["role"], "tool");
|
||||
assert_eq!(wire[1]["tool_call_id"], "call_42");
|
||||
}
|
||||
@@ -1071,7 +1048,7 @@ mod tests {
|
||||
let r1 = ChatMessage::tool_result("a result");
|
||||
let r2 = ChatMessage::tool_result("b result");
|
||||
|
||||
let wire = OpenRouterClient::messages_to_openai(&[assistant, r1, r2]);
|
||||
let wire = OpenRouterClient::messages_to_openai(&[assistant, r1, r2]).unwrap();
|
||||
assert_eq!(wire[1]["tool_call_id"], "call_A");
|
||||
assert_eq!(wire[2]["tool_call_id"], "call_B");
|
||||
}
|
||||
@@ -1090,7 +1067,7 @@ mod tests {
|
||||
}]),
|
||||
images: None,
|
||||
};
|
||||
let wire = OpenRouterClient::messages_to_openai(&[assistant]);
|
||||
let wire = OpenRouterClient::messages_to_openai(&[assistant]).unwrap();
|
||||
let tcs = wire[0]
|
||||
.get("tool_calls")
|
||||
.and_then(|v| v.as_array())
|
||||
|
||||
+301
-19
@@ -59,6 +59,9 @@ pub enum PersonaChatError {
|
||||
EmptyMessage,
|
||||
MessageTooLong,
|
||||
ConcurrentTurn,
|
||||
/// "Continue with persona" couldn't read the SMS thread it was asked to
|
||||
/// anchor on — SMS-API down, or the message isn't in that thread.
|
||||
SmsThread(String),
|
||||
Db(anyhow::Error),
|
||||
}
|
||||
|
||||
@@ -74,6 +77,7 @@ impl std::fmt::Display for PersonaChatError {
|
||||
PersonaChatError::ConcurrentTurn => {
|
||||
write!(f, "another turn for this persona is in flight")
|
||||
}
|
||||
PersonaChatError::SmsThread(e) => write!(f, "could not read the SMS thread: {e}"),
|
||||
PersonaChatError::Db(e) => write!(f, "database error: {e}"),
|
||||
}
|
||||
}
|
||||
@@ -118,6 +122,40 @@ pub fn seed_messages_for_persona(system_prompt: &str) -> Vec<ChatMessage> {
|
||||
vec![ChatMessage::system(system_prompt.to_string())]
|
||||
}
|
||||
|
||||
/// The persona id a practice conversation is stored under. Practice mode
|
||||
/// plays the SMS contact, not a persona, so the conversation belongs to no
|
||||
/// persona row: deleting or editing personas can't hide or break it.
|
||||
pub const PRACTICE_PERSONA_ID: &str = "";
|
||||
|
||||
/// Name shown for a practice conversation where a persona name would go.
|
||||
pub const PRACTICE_PERSONA_NAME: &str = "Practice";
|
||||
|
||||
/// Whether a stored conversation is a practice one (plays the contact).
|
||||
fn is_practice(row: &crate::database::persona_dao::PersonaChatRow) -> bool {
|
||||
row.anchor
|
||||
.as_ref()
|
||||
.is_some_and(|a| a.mode == crate::database::AnchorMode::AsContact)
|
||||
}
|
||||
|
||||
/// The stand-in persona a practice turn runs with. Its prompt is never used
|
||||
/// (`anchored_system_prompt` ignores the persona in practice mode) and it
|
||||
/// can't correct facts — practice is offered no knowledge tools at all.
|
||||
fn practice_persona(user_id: i32) -> crate::database::models::Persona {
|
||||
crate::database::models::Persona {
|
||||
id: 0,
|
||||
user_id,
|
||||
persona_id: PRACTICE_PERSONA_ID.to_string(),
|
||||
name: PRACTICE_PERSONA_NAME.to_string(),
|
||||
system_prompt: String::new(),
|
||||
is_built_in: false,
|
||||
include_all_memories: false,
|
||||
created_at: 0,
|
||||
updated_at: 0,
|
||||
reviewed_only_facts: false,
|
||||
allow_agent_corrections: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a persisted transcript into the conversation tree.
|
||||
///
|
||||
/// Two on-disk formats are accepted, mirroring the file chat's reader:
|
||||
@@ -220,6 +258,40 @@ pub struct PersonaChatHistoryView {
|
||||
/// Empty for a conversation that has never been rewound.
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
pub fork_info: Vec<Option<crate::ai::handlers::ChatForkInfo>>,
|
||||
/// Where in an SMS thread this conversation picked up. Absent for an
|
||||
/// open chat.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub anchor: Option<PersonaChatAnchorView>,
|
||||
}
|
||||
|
||||
/// Wire shape of a conversation's SMS anchor. Enough for the client to show
|
||||
/// a "picked up from" banner and to jump back to the message in the thread;
|
||||
/// the full snapshot stays server-side in the system prompt.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct PersonaChatAnchorView {
|
||||
pub contact_id: i64,
|
||||
pub contact_name: String,
|
||||
pub message_id: i64,
|
||||
/// Unix seconds of the anchor message.
|
||||
pub timestamp: i64,
|
||||
/// The anchor message as "Speaker: text", recovered from the snapshot.
|
||||
pub preview: String,
|
||||
/// `about` (a persona discusses the thread) or `as_contact` (the model
|
||||
/// plays the contact for practice).
|
||||
pub mode: crate::database::AnchorMode,
|
||||
}
|
||||
|
||||
impl From<&crate::database::SmsAnchor> for PersonaChatAnchorView {
|
||||
fn from(a: &crate::database::SmsAnchor) -> Self {
|
||||
PersonaChatAnchorView {
|
||||
contact_id: a.contact_id,
|
||||
contact_name: a.contact_name.clone(),
|
||||
message_id: a.message_id,
|
||||
timestamp: a.timestamp,
|
||||
preview: crate::ai::sms_anchor::anchor_preview(&a.context),
|
||||
mode: a.mode,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One rendered line of the persona transcript. Mirrors the file chat's
|
||||
@@ -420,6 +492,9 @@ pub struct PersonaConversationSummary {
|
||||
/// Whether the conversation has been forked — the list marks these so a
|
||||
/// user can tell which transcripts have alternates parked in them.
|
||||
pub has_branches: bool,
|
||||
/// SMS anchor for "Continue with persona" chats; absent for open chats.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub anchor: Option<PersonaChatAnchorView>,
|
||||
}
|
||||
|
||||
impl PersonaChatHistoryView {
|
||||
@@ -435,6 +510,7 @@ impl PersonaChatHistoryView {
|
||||
active_leaf_id: 0,
|
||||
viewing_branch_id: 0,
|
||||
fork_info: Vec::new(),
|
||||
anchor: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -543,6 +619,74 @@ impl PersonaChatSession {
|
||||
.map_err(|e| PersonaChatError::Db(anyhow!("{e:?}")))
|
||||
}
|
||||
|
||||
/// Start a conversation anchored on a message in an SMS thread
|
||||
/// ("Continue with persona").
|
||||
///
|
||||
/// The thread leading up to the anchor is fetched once, here, and stored
|
||||
/// as a rendered snapshot: every turn splices it into the system prompt,
|
||||
/// so it has to be byte-stable (KV-cache prefix reuse) and has to
|
||||
/// exist before the first turn. A thread that can't be read fails the
|
||||
/// create outright — an anchored chat with no thread is just an open
|
||||
/// chat with a misleading banner.
|
||||
pub async fn create_anchored_conversation(
|
||||
&self,
|
||||
user_id: i32,
|
||||
persona_id: &str,
|
||||
contact_id: i64,
|
||||
message_id: i64,
|
||||
mode: crate::database::AnchorMode,
|
||||
) -> Result<String, PersonaChatError> {
|
||||
// Practice plays the contact; any persona the client sent is ignored.
|
||||
let persona_id = if mode == crate::database::AnchorMode::AsContact {
|
||||
PRACTICE_PERSONA_ID
|
||||
} else {
|
||||
persona_id.trim()
|
||||
};
|
||||
if mode != crate::database::AnchorMode::AsContact {
|
||||
let mut dao = self.persona_dao.lock().expect("persona_dao poisoned");
|
||||
let persona = dao
|
||||
.get_persona(user_id, persona_id)
|
||||
.map_err(|e| PersonaChatError::Db(anyhow!("{e:?}")))?;
|
||||
if persona.is_none() {
|
||||
return Err(PersonaChatError::UnknownPersona(persona_id.to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
let window = self
|
||||
.generator
|
||||
.sms_client()
|
||||
.fetch_thread_window(
|
||||
contact_id,
|
||||
message_id,
|
||||
crate::ai::sms_anchor::ANCHOR_CONTEXT_BEFORE,
|
||||
0,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| PersonaChatError::SmsThread(format!("{e}")))?;
|
||||
let context = crate::ai::sms_anchor::render_thread_window(
|
||||
&window,
|
||||
&crate::ai::user_display_name(),
|
||||
self.generator.display_zone(),
|
||||
);
|
||||
let anchor = crate::database::SmsAnchor {
|
||||
contact_id,
|
||||
contact_name: window.contact.display_name(),
|
||||
message_id,
|
||||
timestamp: window.messages[window.anchor_index].timestamp,
|
||||
context,
|
||||
mode,
|
||||
};
|
||||
|
||||
let mut dao = self.persona_dao.lock().expect("persona_dao poisoned");
|
||||
dao.create_anchored_persona_chat(
|
||||
user_id,
|
||||
persona_id,
|
||||
Utc::now().timestamp_millis(),
|
||||
&anchor,
|
||||
)
|
||||
.map_err(|e| PersonaChatError::Db(anyhow!("{e:?}")))
|
||||
}
|
||||
|
||||
/// Delete a conversation and its transcript outright.
|
||||
pub fn delete_conversation(&self, user_id: i32, conversation_id: &str) -> Result<()> {
|
||||
let mut dao = self.persona_dao.lock().expect("persona_dao poisoned");
|
||||
@@ -577,6 +721,7 @@ impl PersonaChatSession {
|
||||
conversation_id: row.conversation_id.clone(),
|
||||
persona_id: row.persona_id.clone(),
|
||||
title: row.title.clone(),
|
||||
anchor: row.anchor.as_ref().map(PersonaChatAnchorView::from),
|
||||
..PersonaChatHistoryView::empty()
|
||||
};
|
||||
// A conversation holding only its seed system node renders as the
|
||||
@@ -634,10 +779,13 @@ impl PersonaChatSession {
|
||||
.get_persona_chat(user_id, &conversation_id)
|
||||
.map_err(|e| PersonaChatError::Db(anyhow!("{e:?}")))?
|
||||
.ok_or_else(|| PersonaChatError::UnknownConversation(conversation_id.clone()))?;
|
||||
let persona = dao
|
||||
.get_persona(user_id, &row.persona_id)
|
||||
.map_err(|e| PersonaChatError::Db(anyhow!("{e:?}")))?
|
||||
.ok_or_else(|| PersonaChatError::UnknownPersona(row.persona_id.clone()))?;
|
||||
let persona = if is_practice(&row) {
|
||||
practice_persona(user_id)
|
||||
} else {
|
||||
dao.get_persona(user_id, &row.persona_id)
|
||||
.map_err(|e| PersonaChatError::Db(anyhow!("{e:?}")))?
|
||||
.ok_or_else(|| PersonaChatError::UnknownPersona(row.persona_id.clone()))?
|
||||
};
|
||||
(persona, row.persona_id)
|
||||
};
|
||||
|
||||
@@ -748,6 +896,17 @@ impl PersonaChatSession {
|
||||
|
||||
let system_prompt = resolve_persona_system_prompt(Some(&persona))
|
||||
.unwrap_or_else(|| "You are a helpful assistant.".to_string());
|
||||
// An anchored chat carries the SMS thread in its system prompt,
|
||||
// re-composed each turn like the persona prompt itself.
|
||||
let system_prompt = match &row.anchor {
|
||||
Some(anchor) => crate::ai::sms_anchor::anchored_system_prompt(
|
||||
&system_prompt,
|
||||
anchor,
|
||||
&crate::ai::user_display_name(),
|
||||
self.generator.display_zone(),
|
||||
),
|
||||
None => system_prompt,
|
||||
};
|
||||
|
||||
let stored = Some(decode_store(&row.messages_json)?);
|
||||
// `seeded` marks a conversation whose messages are NOT yet backed by
|
||||
@@ -818,11 +977,22 @@ impl PersonaChatSession {
|
||||
// Build a no-photo tool catalog. `current_gate_opts_for_persona`
|
||||
// already probes each tool's backing table for presence and
|
||||
// honours the persona's `allow_agent_corrections` toggle.
|
||||
let gate_opts = self.generator.current_gate_opts_for_persona(
|
||||
backend.images_inline,
|
||||
Some((user_id, &persona.persona_id)),
|
||||
);
|
||||
// `has_vision` only gates `describe_photo`, which describes the file
|
||||
// chat's photo. A persona chat has no photo, so the tool could only
|
||||
// ever answer "No image available" and waste a round. Vision in a
|
||||
// persona chat goes through the thread tools (look_at_message_image,
|
||||
// look_at_photo), which use the local vision model directly.
|
||||
let mut gate_opts = self
|
||||
.generator
|
||||
.current_gate_opts_for_persona(false, Some((user_id, &persona.persona_id)));
|
||||
gate_opts.sms_thread_tools = row.anchor.is_some();
|
||||
let tools = InsightGenerator::build_tool_definitions(gate_opts);
|
||||
// Practice mode narrows the catalog to the thread itself — no
|
||||
// memory, library or calendar access for a simulated contact.
|
||||
let tools = match &row.anchor {
|
||||
Some(anchor) => crate::ai::sms_anchor::tools_for_mode(tools, anchor.mode),
|
||||
None => tools,
|
||||
};
|
||||
|
||||
// Append the new user turn + apply the per-turn system-prompt
|
||||
// override. Per-persona overrides are NOT persisted (the persona
|
||||
@@ -876,6 +1046,8 @@ impl PersonaChatSession {
|
||||
"",
|
||||
user_id,
|
||||
&persona.persona_id,
|
||||
// Thread tools only ever read this chat's own thread.
|
||||
row.anchor.as_ref().map(|a| a.contact_id),
|
||||
max_iterations,
|
||||
&entry,
|
||||
&mut fitter,
|
||||
@@ -981,12 +1153,21 @@ impl PersonaChatSession {
|
||||
/// 404), so listing them would only offer a dead-end tap target.
|
||||
/// Conversations that hold nothing but the seed are omitted too — the
|
||||
/// user has never actually said anything in them.
|
||||
pub fn list_conversations(&self, user_id: i32) -> Result<Vec<PersonaConversationSummary>> {
|
||||
///
|
||||
/// `contact_id` narrows the list to conversations anchored on that SMS
|
||||
/// thread — the Messages screen's per-thread persona list.
|
||||
pub fn list_conversations(
|
||||
&self,
|
||||
user_id: i32,
|
||||
contact_id: Option<i64>,
|
||||
) -> Result<Vec<PersonaConversationSummary>> {
|
||||
let (rows, personas) = {
|
||||
let mut dao = self.persona_dao.lock().expect("persona_dao poisoned");
|
||||
let rows = dao
|
||||
.list_persona_chats(user_id)
|
||||
.map_err(|e| anyhow!("failed to list persona chats: {e:?}"))?;
|
||||
let rows = match contact_id {
|
||||
Some(cid) => dao.list_persona_chats_for_contact(user_id, cid),
|
||||
None => dao.list_persona_chats(user_id),
|
||||
}
|
||||
.map_err(|e| anyhow!("failed to list persona chats: {e:?}"))?;
|
||||
let personas = dao
|
||||
.list_personas(user_id)
|
||||
.map_err(|e| anyhow!("failed to list personas: {e:?}"))?;
|
||||
@@ -999,8 +1180,13 @@ impl PersonaChatSession {
|
||||
|
||||
let mut out = Vec::with_capacity(rows.len());
|
||||
for entry in rows {
|
||||
let Some(persona_name) = names.get(&entry.persona_id).cloned() else {
|
||||
continue;
|
||||
let persona_name = if is_practice(&entry) {
|
||||
PRACTICE_PERSONA_NAME.to_string()
|
||||
} else {
|
||||
match names.get(&entry.persona_id) {
|
||||
Some(name) => name.clone(),
|
||||
None => continue,
|
||||
}
|
||||
};
|
||||
// A single unreadable transcript shouldn't take the whole list
|
||||
// down with it — skip the row and keep going.
|
||||
@@ -1036,6 +1222,7 @@ impl PersonaChatSession {
|
||||
created_at: entry.created_at,
|
||||
updated_at: entry.updated_at,
|
||||
has_branches: store.leaves().len() > 1,
|
||||
anchor: entry.anchor.as_ref().map(PersonaChatAnchorView::from),
|
||||
});
|
||||
}
|
||||
Ok(out)
|
||||
@@ -1231,7 +1418,32 @@ pub struct PersonaChatHistoryQuery {
|
||||
/// Body for POST /persona_chat/conversations — start a new conversation.
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct PersonaChatCreateRequest {
|
||||
/// Required except for a practice (`as_contact`) anchor, which plays
|
||||
/// the contact rather than a persona.
|
||||
#[serde(default)]
|
||||
pub persona_id: String,
|
||||
/// Anchor the conversation on a message in an SMS thread ("Continue
|
||||
/// with persona"). Omitted for an open chat.
|
||||
#[serde(default)]
|
||||
pub anchor: Option<PersonaChatAnchorRequest>,
|
||||
}
|
||||
|
||||
/// SMS-API ids of the thread and message to pick up from.
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct PersonaChatAnchorRequest {
|
||||
pub contact_id: i64,
|
||||
pub message_id: i64,
|
||||
/// `about` (default) or `as_contact`.
|
||||
#[serde(default)]
|
||||
pub mode: crate::database::AnchorMode,
|
||||
}
|
||||
|
||||
/// Query params for GET /persona_chat/conversations.
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct PersonaChatListQuery {
|
||||
/// Only conversations anchored on this SMS thread.
|
||||
#[serde(default)]
|
||||
pub contact_id: Option<i64>,
|
||||
}
|
||||
|
||||
/// GET /persona_chat/history?persona_id=... — the rendered rolling
|
||||
@@ -1315,6 +1527,7 @@ pub async fn persona_chat_turn_handler(
|
||||
HttpResponse::BadRequest()
|
||||
}
|
||||
PersonaChatError::ConcurrentTurn => HttpResponse::Conflict(),
|
||||
PersonaChatError::SmsThread(_) => HttpResponse::BadGateway(),
|
||||
PersonaChatError::Db(_) => {
|
||||
log::error!("persona chat dispatch failed: {e}");
|
||||
HttpResponse::InternalServerError()
|
||||
@@ -1335,10 +1548,22 @@ pub async fn persona_chat_create_conversation_handler(
|
||||
app_state: web::Data<AppState>,
|
||||
) -> impl Responder {
|
||||
let user_id = claims.sub.parse::<i32>().unwrap_or(1);
|
||||
match app_state
|
||||
.persona_chat_session
|
||||
.create_conversation(user_id, &request.persona_id)
|
||||
{
|
||||
let session = &app_state.persona_chat_session;
|
||||
let created = match &request.anchor {
|
||||
Some(anchor) => {
|
||||
session
|
||||
.create_anchored_conversation(
|
||||
user_id,
|
||||
&request.persona_id,
|
||||
anchor.contact_id,
|
||||
anchor.message_id,
|
||||
anchor.mode,
|
||||
)
|
||||
.await
|
||||
}
|
||||
None => session.create_conversation(user_id, &request.persona_id),
|
||||
};
|
||||
match created {
|
||||
Ok(conversation_id) => HttpResponse::Created().json(serde_json::json!({
|
||||
"conversation_id": conversation_id,
|
||||
})),
|
||||
@@ -1346,6 +1571,10 @@ pub async fn persona_chat_create_conversation_handler(
|
||||
PersonaChatError::UnknownPersona(_) => {
|
||||
HttpResponse::NotFound().json(serde_json::json!({ "error": format!("{e}") }))
|
||||
}
|
||||
PersonaChatError::SmsThread(_) => {
|
||||
log::warn!("anchored persona chat create failed: {e}");
|
||||
HttpResponse::BadGateway().json(serde_json::json!({ "error": format!("{e}") }))
|
||||
}
|
||||
_ => {
|
||||
log::error!("persona chat create failed: {e}");
|
||||
HttpResponse::InternalServerError()
|
||||
@@ -1384,10 +1613,14 @@ pub async fn persona_chat_delete_conversation_handler(
|
||||
#[get("/persona_chat/conversations")]
|
||||
pub async fn persona_chat_conversations_handler(
|
||||
claims: Claims,
|
||||
query: web::Query<PersonaChatListQuery>,
|
||||
app_state: web::Data<AppState>,
|
||||
) -> impl Responder {
|
||||
let user_id = claims.sub.parse::<i32>().unwrap_or(1);
|
||||
match app_state.persona_chat_session.list_conversations(user_id) {
|
||||
match app_state
|
||||
.persona_chat_session
|
||||
.list_conversations(user_id, query.contact_id)
|
||||
{
|
||||
Ok(conversations) => HttpResponse::Ok().json(serde_json::json!({
|
||||
"conversations": conversations,
|
||||
})),
|
||||
@@ -1854,6 +2087,42 @@ mod tests {
|
||||
assert!(tracker.claim("conv-a", "t7").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn practice_create_request_needs_no_persona() {
|
||||
let req: PersonaChatCreateRequest = serde_json::from_str(
|
||||
r#"{"anchor":{"contact_id":42,"message_id":812,"mode":"as_contact"}}"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(req.persona_id, "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn practice_persona_is_inert() {
|
||||
let p = practice_persona(7);
|
||||
assert_eq!(p.persona_id, PRACTICE_PERSONA_ID);
|
||||
assert_eq!(p.user_id, 7);
|
||||
assert!(p.system_prompt.is_empty());
|
||||
assert!(!p.allow_agent_corrections);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_request_anchor_mode_defaults_to_about() {
|
||||
use crate::database::AnchorMode;
|
||||
let req: PersonaChatCreateRequest = serde_json::from_str(
|
||||
r#"{"persona_id":"journal","anchor":{"contact_id":42,"message_id":812}}"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(req.anchor.unwrap().mode, AnchorMode::About);
|
||||
let req: PersonaChatCreateRequest = serde_json::from_str(
|
||||
r#"{"persona_id":"journal","anchor":{"contact_id":42,"message_id":812,"mode":"as_contact"}}"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(req.anchor.unwrap().mode, AnchorMode::AsContact);
|
||||
let req: PersonaChatCreateRequest =
|
||||
serde_json::from_str(r#"{"persona_id":"journal"}"#).unwrap();
|
||||
assert!(req.anchor.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persona_chat_turn_request_deserializes_snake_case_wire_body() {
|
||||
// Pins the mobile-client contract: snake_case keys, same as the
|
||||
@@ -1904,6 +2173,7 @@ mod tests {
|
||||
total: 2,
|
||||
node_id: 3,
|
||||
})],
|
||||
anchor: None,
|
||||
};
|
||||
let json = serde_json::to_value(&view).unwrap();
|
||||
// The client's shared ChatHistoryView reads snake_case keys.
|
||||
@@ -2102,8 +2372,20 @@ mod tests {
|
||||
created_at: 1_600_000_000_000,
|
||||
updated_at: 1_700_000_000_000,
|
||||
has_branches: true,
|
||||
anchor: Some(PersonaChatAnchorView {
|
||||
contact_id: 42,
|
||||
contact_name: "Sam".to_string(),
|
||||
message_id: 812,
|
||||
timestamp: 1_700_000_000,
|
||||
preview: "Sam: see you at 6".to_string(),
|
||||
mode: crate::database::AnchorMode::AsContact,
|
||||
}),
|
||||
};
|
||||
let json = serde_json::to_value(&summary).unwrap();
|
||||
assert_eq!(json["anchor"]["contact_id"], 42);
|
||||
assert_eq!(json["anchor"]["message_id"], 812);
|
||||
assert_eq!(json["anchor"]["preview"], "Sam: see you at 6");
|
||||
assert_eq!(json["anchor"]["mode"], "as_contact");
|
||||
assert_eq!(json["conversation_id"], "conv-1");
|
||||
assert_eq!(json["title"], "June recap");
|
||||
assert_eq!(json["persona_id"], "journal");
|
||||
|
||||
@@ -0,0 +1,464 @@
|
||||
//! "Continue with persona" — a persona chat anchored on an SMS thread.
|
||||
//!
|
||||
//! The user long-presses a message in the Messages screen and picks a
|
||||
//! persona; the new persona conversation remembers the thread (contact) and
|
||||
//! the message it picked up from. This module owns the pure parts of that:
|
||||
//!
|
||||
//! - rendering a slice of the thread as a compact transcript the model can
|
||||
//! read (and that names message ids / attachment indices, so the thread
|
||||
//! tools can page further back or look at a photo),
|
||||
//! - the system prompt for each [`AnchorMode`]: a persona talking with the
|
||||
//! user *about* the thread, or the model playing the contact so the user
|
||||
//! can practise the conversation,
|
||||
//! - which tools each mode is offered.
|
||||
//!
|
||||
//! The network side (`SmsApiClient::fetch_thread_window`) and the tools
|
||||
//! (`get_thread_messages`, `look_at_message_image`, `get_photos_near_time`,
|
||||
//! `look_at_photo` in `insight_generator.rs`) live with their neighbours.
|
||||
|
||||
use chrono::{DateTime, TimeZone};
|
||||
use chrono_tz::Tz;
|
||||
|
||||
use crate::ai::ollama::Tool;
|
||||
use crate::ai::sms_client::{ThreadMessage, ThreadWindow};
|
||||
use crate::database::{AnchorMode, SmsAnchor};
|
||||
|
||||
/// Everything a simulated contact may call: the thread they were part of
|
||||
/// and the photos sent in it. Nothing from the user's own life — library,
|
||||
/// memories, calendar, location, other threads — which the real contact
|
||||
/// would never see, and no knowledge writes: facts recorded while the model
|
||||
/// is improvising someone's side of a conversation would be fiction filed
|
||||
/// as memory.
|
||||
const PRACTICE_TOOLS: &[&str] = &["get_thread_messages", "look_at_message_image"];
|
||||
|
||||
/// Messages of history before the anchor captured into the system prompt.
|
||||
/// Enough to pick up tone and the running topic; the model can page
|
||||
/// further back with `get_thread_messages`.
|
||||
pub const ANCHOR_CONTEXT_BEFORE: usize = 40;
|
||||
|
||||
/// Per-message cap inside the rendered transcript. A pasted article or a
|
||||
/// wall of text shouldn't crowd out the rest of the thread.
|
||||
const LINE_MAX_CHARS: usize = 600;
|
||||
|
||||
/// Render one thread message as a transcript line:
|
||||
/// `[#812 · 2019-06-12 14:02] Sam: see you at 6 (image #0 "IMG_1.jpg")`.
|
||||
///
|
||||
/// `#id` and the attachment index are exactly what the thread tools take,
|
||||
/// so the model can act on any line it reads.
|
||||
pub fn render_thread_line(
|
||||
msg: &ThreadMessage,
|
||||
contact_name: &str,
|
||||
user_name: &str,
|
||||
tz: Tz,
|
||||
) -> String {
|
||||
let ts = DateTime::from_timestamp(msg.timestamp, 0)
|
||||
.map(|dt| {
|
||||
tz.from_utc_datetime(&dt.naive_utc())
|
||||
.format("%Y-%m-%d %H:%M")
|
||||
.to_string()
|
||||
})
|
||||
.unwrap_or_else(|| "unknown time".to_string());
|
||||
let speaker = if msg.is_sent { user_name } else { contact_name };
|
||||
|
||||
let mut body = msg.body.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
if body.chars().count() > LINE_MAX_CHARS {
|
||||
body = format!("{}…", body.chars().take(LINE_MAX_CHARS).collect::<String>());
|
||||
}
|
||||
let attachments: Vec<String> = msg
|
||||
.attachments
|
||||
.iter()
|
||||
.map(|a| {
|
||||
let kind = a.content_type.split('/').next().unwrap_or("file");
|
||||
if a.filename.is_empty() {
|
||||
format!("{kind} #{}", a.media_index)
|
||||
} else {
|
||||
format!("{kind} #{} \"{}\"", a.media_index, a.filename)
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let mut line = format!("[#{} · {}] {}:", msg.id, ts, speaker);
|
||||
if !body.is_empty() {
|
||||
line.push(' ');
|
||||
line.push_str(&body);
|
||||
}
|
||||
if !attachments.is_empty() {
|
||||
line.push_str(&format!(" ({})", attachments.join(", ")));
|
||||
}
|
||||
line
|
||||
}
|
||||
|
||||
/// Render a thread window as a transcript, marking the anchor message with
|
||||
/// `▶` so the model can tell where the user picked the thread up.
|
||||
pub fn render_thread_window(window: &ThreadWindow, user_name: &str, tz: Tz) -> String {
|
||||
let contact_name = window.contact.display_name();
|
||||
window
|
||||
.messages
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, m)| {
|
||||
let line = render_thread_line(m, &contact_name, user_name, tz);
|
||||
if i == window.anchor_index {
|
||||
format!("▶ {line}")
|
||||
} else {
|
||||
line
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
/// The anchor message's text, recovered from the stored snapshot, for the
|
||||
/// chat screen's "picked up from" banner and the list row. Reads the `▶`
|
||||
/// line so no extra column (or SMS-API round-trip) is needed.
|
||||
pub fn anchor_preview(context: &str) -> String {
|
||||
context
|
||||
.lines()
|
||||
.find_map(|l| l.strip_prefix("▶ "))
|
||||
.map(|l| match l.split_once("] ") {
|
||||
Some((_, rest)) => rest.to_string(),
|
||||
None => l.to_string(),
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// The tool catalog narrowed to what `mode` may use. `About` keeps the full
|
||||
/// persona catalog; `AsContact` keeps only [`PRACTICE_TOOLS`].
|
||||
pub fn tools_for_mode(tools: Vec<Tool>, mode: AnchorMode) -> Vec<Tool> {
|
||||
match mode {
|
||||
AnchorMode::About => tools,
|
||||
AnchorMode::AsContact => tools
|
||||
.into_iter()
|
||||
.filter(|t| PRACTICE_TOOLS.contains(&t.function.name.as_str()))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The values both prompts interpolate.
|
||||
struct PromptParts {
|
||||
contact: String,
|
||||
when: String,
|
||||
thread: String,
|
||||
mid: i64,
|
||||
}
|
||||
|
||||
fn prompt_parts(anchor: &SmsAnchor, tz: Tz) -> PromptParts {
|
||||
let contact = if anchor.contact_name.trim().is_empty() {
|
||||
"someone".to_string()
|
||||
} else {
|
||||
anchor.contact_name.clone()
|
||||
};
|
||||
let when = DateTime::from_timestamp(anchor.timestamp, 0)
|
||||
.map(|dt| {
|
||||
tz.from_utc_datetime(&dt.naive_utc())
|
||||
.format("%A, %B %-d, %Y at %-I:%M %p")
|
||||
.to_string()
|
||||
})
|
||||
.unwrap_or_else(|| "an unknown date".to_string());
|
||||
let thread = if anchor.context.trim().is_empty() {
|
||||
"(The thread could not be loaded — use get_thread_messages to read it.)".to_string()
|
||||
} else {
|
||||
anchor.context.clone()
|
||||
};
|
||||
PromptParts {
|
||||
contact,
|
||||
when,
|
||||
thread,
|
||||
mid: anchor.message_id,
|
||||
}
|
||||
}
|
||||
|
||||
/// The system prompt for an anchored conversation.
|
||||
///
|
||||
/// Re-composed every turn from the persona's current prompt and the stored
|
||||
/// anchor snapshot, so editing a persona still reaches conversations already
|
||||
/// in flight (as it does for open chats) while the thread snapshot itself
|
||||
/// stays byte-stable. Practice mode doesn't use the persona prompt at all:
|
||||
/// the model is the contact, and a persona's voice would bleed into theirs.
|
||||
pub fn anchored_system_prompt(
|
||||
persona_prompt: &str,
|
||||
anchor: &SmsAnchor,
|
||||
user_name: &str,
|
||||
tz: Tz,
|
||||
) -> String {
|
||||
let parts = prompt_parts(anchor, tz);
|
||||
match anchor.mode {
|
||||
AnchorMode::About => about_prompt(persona_prompt, &parts, user_name),
|
||||
AnchorMode::AsContact => practice_prompt(&parts, user_name),
|
||||
}
|
||||
}
|
||||
|
||||
/// A persona talking with the user about the thread.
|
||||
fn about_prompt(persona_prompt: &str, parts: &PromptParts, user_name: &str) -> String {
|
||||
let PromptParts {
|
||||
contact,
|
||||
when,
|
||||
thread,
|
||||
mid,
|
||||
} = parts;
|
||||
format!(
|
||||
"{persona_prompt}
|
||||
|
||||
## Picking up a text thread
|
||||
|
||||
{user_name} opened this chat from their text conversation with {contact}. \
|
||||
Below is that thread up to the message they picked (marked ▶, sent {when}). \
|
||||
They want to keep talking with you from that point — about what was going on, \
|
||||
what came of it, or just to carry on the conversation.
|
||||
|
||||
How to talk here:
|
||||
- Keep your own voice and values from above, but this is a chat, not a \
|
||||
write-up. Reply like a text message: short, warm, one or two thoughts at a \
|
||||
time. No headings, no bullet lists, no summaries unless asked.
|
||||
- Match the thread's register — its slang, emoji and punctuation habits, how \
|
||||
long its messages run. If the thread is playful, be playful.
|
||||
- You are not {contact}. Never write as them or put words in their mouth; you \
|
||||
are someone {user_name} trusts who has just read the thread over their shoulder.
|
||||
- Lead with specifics from the thread — names, plans, inside jokes, open \
|
||||
questions. Don't recap the thread back to {user_name}.
|
||||
- The thread is history: today's date is not {when}. If something was left \
|
||||
unresolved, it's natural to ask how it turned out.
|
||||
- Don't invent details. When you need more than the thread below shows, look \
|
||||
it up with your tools first, then answer in your own words — don't narrate \
|
||||
the lookups. Look at one or two photos that matter, not every one you find.
|
||||
- You can read and look, but you can't send photos or texts to anyone — \
|
||||
don't offer to.
|
||||
|
||||
Tools for this thread (#numbers below are message ids):
|
||||
- get_thread_messages {{message_id: {mid}, direction: \"before\"}} — earlier \
|
||||
messages; \"after\" for what came next.
|
||||
- look_at_message_image {{message_id, media_index}} — see a photo sent in the \
|
||||
thread (attachments are listed as `image #N`).
|
||||
- get_photos_near_time {{timestamp}} — photos from {user_name}'s library taken \
|
||||
around a moment, with their tags and the people in them.
|
||||
- look_at_photo {{file_path}} — see one of those library photos.
|
||||
|
||||
Thread with {contact}:
|
||||
{thread}"
|
||||
)
|
||||
}
|
||||
|
||||
/// The model plays the contact so the user can rehearse the conversation.
|
||||
fn practice_prompt(parts: &PromptParts, user_name: &str) -> String {
|
||||
let PromptParts {
|
||||
contact,
|
||||
when,
|
||||
thread,
|
||||
mid,
|
||||
} = parts;
|
||||
format!(
|
||||
"You are playing {contact} in a text conversation with {user_name}, so \
|
||||
{user_name} can practise picking the conversation back up. This is a private \
|
||||
rehearsal for {user_name}; the real {contact} is not part of it.
|
||||
|
||||
Below is the real thread up to the message marked ▶ (sent {when}). Carry on \
|
||||
from there as {contact}: {user_name}'s messages to you are their next texts, \
|
||||
and every reply you write is {contact}'s next text.
|
||||
|
||||
How to play it:
|
||||
- Sound like {contact} does in the thread — their message length, slang, \
|
||||
emoji, punctuation, humour, how fast they get to the point. Mirror them, not \
|
||||
a generic texter.
|
||||
- Write only the text itself. No narration, no stage directions, no 'As \
|
||||
{contact}, I\'d say…'. Separate several short texts with a blank line.
|
||||
- Stay true to what {contact} knows and feels in the thread. Don't invent \
|
||||
big facts about their life — jobs, relationships, events — that the thread \
|
||||
doesn't support; if asked about something they couldn't know, answer vaguely \
|
||||
or deflect the way they plausibly would.
|
||||
- React like a real person: it's fine to be busy, short, unconvinced, or to \
|
||||
ask something back. Don't be more agreeable than {contact} actually is in \
|
||||
the thread.
|
||||
- It is shortly after the ▶ message unless {user_name} says time has passed.
|
||||
- If {user_name} steps out of the rehearsal ('pause', 'out of character', \
|
||||
'how did that sound?'), drop the role and answer briefly as a coach: honest, \
|
||||
specific feedback on how their texts are landing. Go back to being {contact} \
|
||||
when they carry on.
|
||||
|
||||
If you need to remember an earlier part of the thread, use \
|
||||
get_thread_messages {{message_id: {mid}, direction: \"before\"}}; to see a \
|
||||
photo sent in it (`image #N` on a line), use look_at_message_image \
|
||||
{{message_id, media_index}}. Never mention either.
|
||||
|
||||
Thread between {user_name} and {contact}:
|
||||
{thread}"
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::ai::sms_client::{ThreadAttachment, ThreadContact};
|
||||
|
||||
fn msg(id: i64, body: &str, sent: bool) -> ThreadMessage {
|
||||
ThreadMessage {
|
||||
id,
|
||||
body: body.to_string(),
|
||||
// 2023-11-15 03:13 UTC = 2023-11-14 22:13 in New York.
|
||||
timestamp: 1_700_018_000 - 1_000 + id,
|
||||
is_sent: sent,
|
||||
attachments: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn line_names_the_speaker_id_and_local_time() {
|
||||
let line = render_thread_line(
|
||||
&msg(1000, "on my way", true),
|
||||
"Sam",
|
||||
"Cam",
|
||||
Tz::America__New_York,
|
||||
);
|
||||
assert_eq!(line, "[#1000 · 2023-11-14 22:13] Cam: on my way");
|
||||
let line = render_thread_line(&msg(1000, "ok", false), "Sam", "Cam", Tz::America__New_York);
|
||||
assert!(line.ends_with("] Sam: ok"), "{line}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn line_lists_attachments_with_their_media_index() {
|
||||
let mut m = msg(5, "", false);
|
||||
m.attachments = vec![
|
||||
ThreadAttachment {
|
||||
media_index: 0,
|
||||
content_type: "image/jpeg".to_string(),
|
||||
filename: "IMG_1.jpg".to_string(),
|
||||
},
|
||||
ThreadAttachment {
|
||||
media_index: 1,
|
||||
content_type: "video/mp4".to_string(),
|
||||
filename: String::new(),
|
||||
},
|
||||
];
|
||||
let line = render_thread_line(&m, "Sam", "Cam", Tz::UTC);
|
||||
assert!(
|
||||
line.ends_with("Sam: (image #0 \"IMG_1.jpg\", video #1)"),
|
||||
"an empty body leaves just the attachments: {line}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn line_flattens_and_caps_long_bodies() {
|
||||
let long = format!("a\n\nb {}", "x".repeat(2000));
|
||||
let line = render_thread_line(&msg(1, &long, false), "Sam", "Cam", Tz::UTC);
|
||||
assert!(!line.contains('\n'));
|
||||
assert!(line.ends_with('…'));
|
||||
assert!(line.chars().count() < LINE_MAX_CHARS + 60);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn window_marks_only_the_anchor() {
|
||||
let window = ThreadWindow {
|
||||
contact: ThreadContact {
|
||||
id: 3,
|
||||
name: "Sam".to_string(),
|
||||
address: String::new(),
|
||||
},
|
||||
messages: vec![msg(1, "a", false), msg(2, "b", true), msg(3, "c", false)],
|
||||
anchor_index: 1,
|
||||
};
|
||||
let rendered = render_thread_window(&window, "Cam", Tz::UTC);
|
||||
let lines: Vec<&str> = rendered.lines().collect();
|
||||
assert_eq!(lines.len(), 3);
|
||||
assert!(!lines[0].starts_with('▶'));
|
||||
assert!(lines[1].starts_with("▶ [#2"));
|
||||
assert!(!lines[2].starts_with('▶'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn anchor_preview_reads_the_marked_line() {
|
||||
let ctx = "[#1 · 2023-11-14 22:10] Cam: hi\n▶ [#2 · 2023-11-14 22:13] Sam: see you at 6\n";
|
||||
assert_eq!(anchor_preview(ctx), "Sam: see you at 6");
|
||||
assert_eq!(anchor_preview("no marker"), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn system_prompt_keeps_the_persona_first_and_carries_the_ids() {
|
||||
let anchor = SmsAnchor {
|
||||
contact_id: 42,
|
||||
contact_name: "Sam".to_string(),
|
||||
message_id: 812,
|
||||
timestamp: 1_700_000_000,
|
||||
context: "[#812 · …] Sam: see you there".to_string(),
|
||||
mode: AnchorMode::About,
|
||||
};
|
||||
let prompt = anchored_system_prompt("You are Journal.", &anchor, "Cam", Tz::UTC);
|
||||
assert!(prompt.starts_with("You are Journal."));
|
||||
assert!(prompt.contains("message_id: 812"));
|
||||
// The thread is bound server-side; the prompt doesn't hand the
|
||||
// model a contact id to choose.
|
||||
assert!(!prompt.contains("contact_id"));
|
||||
assert!(prompt.contains("You are not Sam"));
|
||||
assert!(prompt.ends_with("Sam: see you there"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn system_prompt_survives_a_missing_snapshot() {
|
||||
let anchor = SmsAnchor {
|
||||
contact_id: 42,
|
||||
contact_name: String::new(),
|
||||
message_id: 812,
|
||||
timestamp: 0,
|
||||
context: String::new(),
|
||||
mode: AnchorMode::About,
|
||||
};
|
||||
let prompt = anchored_system_prompt("P", &anchor, "Cam", Tz::UTC);
|
||||
assert!(prompt.contains("could not be loaded"));
|
||||
assert!(prompt.contains("conversation with someone"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn practice_prompt_plays_the_contact_and_drops_the_persona() {
|
||||
let anchor = SmsAnchor {
|
||||
contact_id: 42,
|
||||
contact_name: "Sam".to_string(),
|
||||
message_id: 812,
|
||||
timestamp: 1_700_000_000,
|
||||
context: "▶ [#812 · …] Sam: see you there".to_string(),
|
||||
mode: AnchorMode::AsContact,
|
||||
};
|
||||
let prompt = anchored_system_prompt("You are Journal.", &anchor, "Cam", Tz::UTC);
|
||||
assert!(prompt.starts_with("You are playing Sam"));
|
||||
assert!(
|
||||
!prompt.contains("You are Journal."),
|
||||
"a persona voice would bleed into the contact's"
|
||||
);
|
||||
assert!(prompt.contains("message_id: 812"));
|
||||
assert!(!prompt.contains("contact_id"));
|
||||
assert!(prompt.contains("out of character"));
|
||||
assert!(prompt.ends_with("Sam: see you there"));
|
||||
}
|
||||
|
||||
fn tool(name: &str) -> Tool {
|
||||
Tool::function(name, "", serde_json::json!({"type": "object"}))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn practice_mode_keeps_only_the_thread_tools() {
|
||||
let all = || {
|
||||
[
|
||||
"get_thread_messages",
|
||||
"look_at_message_image",
|
||||
"get_photos_near_time",
|
||||
"look_at_photo",
|
||||
"recall_entities",
|
||||
"store_fact",
|
||||
"search_messages",
|
||||
"get_calendar_events",
|
||||
]
|
||||
.into_iter()
|
||||
.map(tool)
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let names = |tools: Vec<Tool>| -> Vec<String> {
|
||||
tools.into_iter().map(|t| t.function.name).collect()
|
||||
};
|
||||
assert_eq!(
|
||||
names(tools_for_mode(all(), AnchorMode::AsContact)),
|
||||
vec!["get_thread_messages", "look_at_message_image"]
|
||||
);
|
||||
assert_eq!(
|
||||
names(tools_for_mode(all(), AnchorMode::About)).len(),
|
||||
8,
|
||||
"talking about the thread keeps the full catalog, memory tools included"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -320,6 +320,157 @@ impl SmsApiClient {
|
||||
Ok(data.results)
|
||||
}
|
||||
|
||||
/// One page of a contact's thread from SMS-API's conversation endpoint,
|
||||
/// newest first. With `message_id`, SMS-API ignores `page` and returns
|
||||
/// the page that contains that message.
|
||||
pub async fn fetch_thread_page(
|
||||
&self,
|
||||
contact_id: i64,
|
||||
message_id: Option<i64>,
|
||||
page: usize,
|
||||
page_size: usize,
|
||||
) -> Result<ThreadPage> {
|
||||
let mut url = format!(
|
||||
"{}/api/conversations/{}/?page={}&page_size={}",
|
||||
self.base_url, contact_id, page, page_size
|
||||
);
|
||||
if let Some(mid) = message_id {
|
||||
url.push_str(&format!("&message_id={}", mid));
|
||||
}
|
||||
let mut request = self.client.get(&url);
|
||||
if let Some(token) = &self.token {
|
||||
request = request.header("Authorization", format!("Bearer {}", token));
|
||||
}
|
||||
let response = request.send().await?;
|
||||
if !response.status().is_success() {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
return Err(anyhow::anyhow!(
|
||||
"SMS conversation request failed: {} - {}",
|
||||
status,
|
||||
body
|
||||
));
|
||||
}
|
||||
let raw: RawThreadPage = response.json().await?;
|
||||
Ok(raw.into())
|
||||
}
|
||||
|
||||
/// The slice of a thread around `anchor_message_id`: up to `before`
|
||||
/// messages older than the anchor, the anchor itself, and up to `after`
|
||||
/// newer ones — oldest first, ready to render as a transcript.
|
||||
///
|
||||
/// SMS-API pages are fixed-size, so the anchor can sit at either edge of
|
||||
/// the page that holds it; the neighbouring page is pulled in whenever
|
||||
/// the requested side runs short.
|
||||
pub async fn fetch_thread_window(
|
||||
&self,
|
||||
contact_id: i64,
|
||||
anchor_message_id: i64,
|
||||
before: usize,
|
||||
after: usize,
|
||||
) -> Result<ThreadWindow> {
|
||||
let page_size = (before + after + 1).clamp(20, 200);
|
||||
let first = self
|
||||
.fetch_thread_page(contact_id, Some(anchor_message_id), 1, page_size)
|
||||
.await?;
|
||||
let contact = first.contact.clone();
|
||||
let Some(idx) = first
|
||||
.messages
|
||||
.iter()
|
||||
.position(|m| m.id == anchor_message_id)
|
||||
else {
|
||||
return Err(anyhow::anyhow!(
|
||||
"message {} not found in conversation {}",
|
||||
anchor_message_id,
|
||||
contact_id
|
||||
));
|
||||
};
|
||||
|
||||
// Newest first: index < idx is newer, > idx is older.
|
||||
let mut newer: Vec<ThreadMessage> = first.messages[..idx].to_vec();
|
||||
let anchor = first.messages[idx].clone();
|
||||
let mut older: Vec<ThreadMessage> = first.messages[idx + 1..].to_vec();
|
||||
|
||||
if older.len() < before && first.page < first.total_pages {
|
||||
let next = self
|
||||
.fetch_thread_page(contact_id, None, first.page + 1, page_size)
|
||||
.await?;
|
||||
older.extend(next.messages);
|
||||
}
|
||||
if newer.len() < after && first.page > 1 {
|
||||
let prev = self
|
||||
.fetch_thread_page(contact_id, None, first.page - 1, page_size)
|
||||
.await?;
|
||||
// The previous page is entirely newer; keep it newest-first so
|
||||
// the closest messages stay at the end of `newer`.
|
||||
let mut combined = prev.messages;
|
||||
combined.extend(newer);
|
||||
newer = combined;
|
||||
}
|
||||
|
||||
older.truncate(before);
|
||||
older.reverse();
|
||||
let newer_start = newer.len().saturating_sub(after);
|
||||
let mut newer: Vec<ThreadMessage> = newer[newer_start..].to_vec();
|
||||
newer.reverse();
|
||||
|
||||
let mut messages = older;
|
||||
let anchor_index = messages.len();
|
||||
messages.push(anchor);
|
||||
messages.extend(newer);
|
||||
Ok(ThreadWindow {
|
||||
contact,
|
||||
messages,
|
||||
anchor_index,
|
||||
})
|
||||
}
|
||||
|
||||
/// Download one MMS *image*. `media_index` is the attachment's position
|
||||
/// among the message's image/video/audio parts — the same index
|
||||
/// SMS-API's `/media/{message}/{index}/file/` route takes.
|
||||
///
|
||||
/// Checks the response before reading the body, so pointing this at a
|
||||
/// video or audio attachment costs a header round-trip, not the whole
|
||||
/// file in memory; images over [`MMS_IMAGE_MAX_BYTES`] are refused too.
|
||||
pub async fn fetch_mms_image(
|
||||
&self,
|
||||
contact_id: i64,
|
||||
message_id: i64,
|
||||
media_index: usize,
|
||||
) -> Result<Vec<u8>> {
|
||||
let url = format!(
|
||||
"{}/api/conversations/{}/media/{}/{}/file/",
|
||||
self.base_url, contact_id, message_id, media_index
|
||||
);
|
||||
let mut request = self.client.get(&url);
|
||||
if let Some(token) = &self.token {
|
||||
request = request.header("Authorization", format!("Bearer {}", token));
|
||||
}
|
||||
let mut response = request.send().await?;
|
||||
if !response.status().is_success() {
|
||||
return Err(anyhow::anyhow!(
|
||||
"MMS media request failed: {}",
|
||||
response.status()
|
||||
));
|
||||
}
|
||||
let content_type = response
|
||||
.headers()
|
||||
.get(reqwest::header::CONTENT_TYPE)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(str::to_string);
|
||||
check_mms_image_response(content_type.as_deref(), response.content_length())
|
||||
.map_err(|e| anyhow::anyhow!(e))?;
|
||||
// Content-Length can be absent (chunked); cap while reading too.
|
||||
let mut bytes = Vec::new();
|
||||
while let Some(chunk) = response.chunk().await? {
|
||||
if bytes.len() + chunk.len() > MMS_IMAGE_MAX_BYTES {
|
||||
return Err(anyhow::anyhow!("attachment is larger than the image limit"));
|
||||
}
|
||||
bytes.extend_from_slice(&chunk);
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
pub async fn summarize_context(
|
||||
&self,
|
||||
messages: &[SmsMessage],
|
||||
@@ -372,6 +523,163 @@ pub struct SmsMessage {
|
||||
pub is_sent: bool,
|
||||
}
|
||||
|
||||
/// Largest MMS image `fetch_mms_image` will download. Phone photos sent by
|
||||
/// MMS are a few MB; anything past this is not worth decoding for a
|
||||
/// 1024px vision description.
|
||||
pub const MMS_IMAGE_MAX_BYTES: usize = 20 * 1024 * 1024;
|
||||
|
||||
/// Decide from the response headers whether an MMS attachment is an image
|
||||
/// worth downloading. A missing content type is allowed through (the
|
||||
/// decoder decides); a declared non-image or an oversized length is not.
|
||||
fn check_mms_image_response(
|
||||
content_type: Option<&str>,
|
||||
content_length: Option<u64>,
|
||||
) -> Result<(), String> {
|
||||
if let Some(ct) = content_type
|
||||
&& !ct.trim().to_ascii_lowercase().starts_with("image/")
|
||||
{
|
||||
return Err(format!("attachment is {ct}, not an image"));
|
||||
}
|
||||
if content_length.is_some_and(|n| n > MMS_IMAGE_MAX_BYTES as u64) {
|
||||
return Err("attachment is larger than the image limit".to_string());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A thread participant as SMS-API's conversation endpoint describes it.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct ThreadContact {
|
||||
pub id: i64,
|
||||
#[serde(default)]
|
||||
pub name: String,
|
||||
#[serde(default)]
|
||||
pub address: String,
|
||||
}
|
||||
|
||||
impl ThreadContact {
|
||||
/// Name to show for the other side of the thread — the stored name, or
|
||||
/// the phone number for a contact that was never named.
|
||||
pub fn display_name(&self) -> String {
|
||||
if self.name.trim().is_empty() {
|
||||
self.address.clone()
|
||||
} else {
|
||||
self.name.clone()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One message of a thread. `attachments` holds only the media parts
|
||||
/// (image/video/audio); MMS text parts are folded into `body`.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ThreadMessage {
|
||||
pub id: i64,
|
||||
pub body: String,
|
||||
/// Unix seconds.
|
||||
pub timestamp: i64,
|
||||
pub is_sent: bool,
|
||||
pub attachments: Vec<ThreadAttachment>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct ThreadAttachment {
|
||||
/// Position among the message's media parts — the index
|
||||
/// `fetch_mms_media` takes.
|
||||
pub media_index: usize,
|
||||
pub content_type: String,
|
||||
pub filename: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ThreadPage {
|
||||
pub contact: ThreadContact,
|
||||
/// Newest first, as SMS-API returns them.
|
||||
pub messages: Vec<ThreadMessage>,
|
||||
pub page: usize,
|
||||
pub total_pages: usize,
|
||||
}
|
||||
|
||||
/// A slice of a thread around an anchor message, oldest first.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ThreadWindow {
|
||||
pub contact: ThreadContact,
|
||||
pub messages: Vec<ThreadMessage>,
|
||||
/// Index of the anchor message within `messages`.
|
||||
pub anchor_index: usize,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct RawThreadPage {
|
||||
contact: ThreadContact,
|
||||
messages: Vec<RawThreadMessage>,
|
||||
page: usize,
|
||||
total_pages: usize,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct RawThreadMessage {
|
||||
id: i64,
|
||||
#[serde(default)]
|
||||
body: String,
|
||||
date: i64,
|
||||
#[serde(rename = "type")]
|
||||
type_: i32,
|
||||
#[serde(default)]
|
||||
parts: Vec<RawThreadPart>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct RawThreadPart {
|
||||
content_type: String,
|
||||
#[serde(default)]
|
||||
filename: String,
|
||||
#[serde(default)]
|
||||
text: String,
|
||||
}
|
||||
|
||||
impl From<RawThreadPage> for ThreadPage {
|
||||
fn from(raw: RawThreadPage) -> Self {
|
||||
ThreadPage {
|
||||
contact: raw.contact,
|
||||
messages: raw.messages.into_iter().map(ThreadMessage::from).collect(),
|
||||
page: raw.page,
|
||||
total_pages: raw.total_pages,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<RawThreadMessage> for ThreadMessage {
|
||||
/// Media indices are counted the way SMS-API's media routes count them:
|
||||
/// SMIL layout parts are skipped and only image/video/audio parts take
|
||||
/// an index, in part order.
|
||||
fn from(raw: RawThreadMessage) -> Self {
|
||||
let mut attachments = Vec::new();
|
||||
let mut body = raw.body;
|
||||
for part in raw.parts {
|
||||
let ct = part.content_type.to_ascii_lowercase();
|
||||
if ct == "application/smil" {
|
||||
continue;
|
||||
}
|
||||
if ct.starts_with("image/") || ct.starts_with("video/") || ct.starts_with("audio/") {
|
||||
attachments.push(ThreadAttachment {
|
||||
media_index: attachments.len(),
|
||||
content_type: part.content_type,
|
||||
filename: part.filename,
|
||||
});
|
||||
} else if ct == "text/plain" && body.trim().is_empty() && !part.text.trim().is_empty() {
|
||||
// MMS keeps its caption in a text part, leaving `body` empty.
|
||||
body = part.text;
|
||||
}
|
||||
}
|
||||
ThreadMessage {
|
||||
id: raw.id,
|
||||
body,
|
||||
timestamp: raw.date,
|
||||
is_sent: raw.type_ == 2,
|
||||
attachments,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct SmsApiResponse {
|
||||
messages: Vec<SmsApiMessage>,
|
||||
@@ -451,6 +759,63 @@ mod tests {
|
||||
assert_eq!(end - 7 * 86400, center);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thread_message_indexes_media_parts_like_sms_api() {
|
||||
// SMIL is skipped and the text part only fills an empty body, so
|
||||
// the image and video take media indices 0 and 1 — matching the
|
||||
// `/media/{id}/{index}/file/` route.
|
||||
let raw: RawThreadMessage = serde_json::from_value(serde_json::json!({
|
||||
"id": 7,
|
||||
"body": "",
|
||||
"date": 1_700_000_000,
|
||||
"type": 2,
|
||||
"parts": [
|
||||
{ "content_type": "application/smil", "filename": "smil.xml" },
|
||||
{ "content_type": "image/jpeg", "filename": "IMG_1.jpg" },
|
||||
{ "content_type": "text/plain", "text": "look at this" },
|
||||
{ "content_type": "video/mp4", "filename": "clip.mp4" }
|
||||
]
|
||||
}))
|
||||
.unwrap();
|
||||
let msg = ThreadMessage::from(raw);
|
||||
assert!(msg.is_sent);
|
||||
assert_eq!(msg.body, "look at this");
|
||||
assert_eq!(
|
||||
msg.attachments
|
||||
.iter()
|
||||
.map(|a| (a.media_index, a.filename.as_str()))
|
||||
.collect::<Vec<_>>(),
|
||||
vec![(0, "IMG_1.jpg"), (1, "clip.mp4")]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mms_image_check_refuses_non_images_and_huge_files_before_download() {
|
||||
assert!(check_mms_image_response(Some("image/jpeg"), Some(2_000_000)).is_ok());
|
||||
assert!(check_mms_image_response(Some("IMAGE/PNG"), None).is_ok());
|
||||
assert!(check_mms_image_response(None, None).is_ok());
|
||||
assert!(
|
||||
check_mms_image_response(Some("video/mp4"), Some(1))
|
||||
.unwrap_err()
|
||||
.contains("video/mp4")
|
||||
);
|
||||
assert!(check_mms_image_response(Some("audio/amr"), None).is_err());
|
||||
assert!(
|
||||
check_mms_image_response(Some("image/jpeg"), Some(MMS_IMAGE_MAX_BYTES as u64 + 1))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thread_contact_falls_back_to_the_address() {
|
||||
let c = ThreadContact {
|
||||
id: 1,
|
||||
name: " ".to_string(),
|
||||
address: "+15551234".to_string(),
|
||||
};
|
||||
assert_eq!(c.display_name(), "+15551234");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn window_for_radius_clamps_zero_to_one() {
|
||||
let (start, end) = SmsApiClient::window_for_radius(100_000, 0);
|
||||
|
||||
+3
-1
@@ -64,7 +64,9 @@ pub use knowledge_dao::{
|
||||
KnowledgeDao, PersonaFilter, RecentActivity, SqliteKnowledgeDao,
|
||||
};
|
||||
pub use location_dao::{LocationHistoryDao, SqliteLocationHistoryDao};
|
||||
pub use persona_dao::{ImportPersona, PersonaDao, PersonaPatch, SqlitePersonaDao};
|
||||
pub use persona_dao::{
|
||||
AnchorMode, ImportPersona, PersonaDao, PersonaPatch, SmsAnchor, SqlitePersonaDao,
|
||||
};
|
||||
pub use precomputed_reel_dao::{PrecomputedReelDao, SqlitePrecomputedReelDao};
|
||||
pub use preview_dao::{PreviewDao, SqlitePreviewDao};
|
||||
pub use search_dao::{SearchHistoryDao, SqliteSearchHistoryDao};
|
||||
|
||||
+268
-31
@@ -29,21 +29,127 @@ pub struct PersonaChatRow {
|
||||
pub turn_count: i32,
|
||||
pub created_at: i64,
|
||||
pub updated_at: i64,
|
||||
/// The SMS thread position this conversation grew out of, for chats
|
||||
/// started via "Continue with persona". None for an open chat.
|
||||
pub anchor: Option<SmsAnchor>,
|
||||
}
|
||||
|
||||
/// Column tuple → `PersonaChatRow`. Shared by the single-row and list
|
||||
/// queries so their `select(...)` orders can never drift apart.
|
||||
/// Where in an SMS thread an anchored persona conversation picks up.
|
||||
///
|
||||
/// `contact_id` / `message_id` are SMS-API ids. `context` is the rendered
|
||||
/// snapshot of the thread leading up to the anchor, captured once at
|
||||
/// creation and spliced into the system prompt on every turn.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct SmsAnchor {
|
||||
pub contact_id: i64,
|
||||
pub contact_name: String,
|
||||
pub message_id: i64,
|
||||
/// Unix seconds of the anchor message.
|
||||
pub timestamp: i64,
|
||||
pub context: String,
|
||||
pub mode: AnchorMode,
|
||||
}
|
||||
|
||||
/// How an anchored conversation relates to its SMS thread.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum AnchorMode {
|
||||
/// A persona talks with the user about the thread.
|
||||
#[default]
|
||||
About,
|
||||
/// The model plays the contact so the user can practise the
|
||||
/// conversation.
|
||||
AsContact,
|
||||
}
|
||||
|
||||
impl AnchorMode {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
AnchorMode::About => "about",
|
||||
AnchorMode::AsContact => "as_contact",
|
||||
}
|
||||
}
|
||||
|
||||
/// Stored value → mode. NULL and anything unrecognised read as `About`,
|
||||
/// the behaviour rows had before the column existed.
|
||||
pub fn from_column(value: Option<&str>) -> Self {
|
||||
match value {
|
||||
Some("as_contact") => AnchorMode::AsContact,
|
||||
_ => AnchorMode::About,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The column tuple every persona-chat read selects. Shared by the
|
||||
/// single-row and list queries so their `select(...)` orders can never
|
||||
/// drift apart.
|
||||
type PersonaChatColumns = (
|
||||
String,
|
||||
String,
|
||||
String,
|
||||
String,
|
||||
i32,
|
||||
i64,
|
||||
i64,
|
||||
Option<i64>,
|
||||
Option<String>,
|
||||
Option<i64>,
|
||||
Option<i64>,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
);
|
||||
|
||||
macro_rules! persona_chat_select {
|
||||
() => {
|
||||
(
|
||||
conversation_id,
|
||||
persona_id,
|
||||
title,
|
||||
messages_json,
|
||||
turn_count,
|
||||
created_at,
|
||||
updated_at,
|
||||
anchor_contact_id,
|
||||
anchor_contact_name,
|
||||
anchor_message_id,
|
||||
anchor_ts,
|
||||
anchor_context,
|
||||
anchor_mode,
|
||||
)
|
||||
};
|
||||
}
|
||||
|
||||
/// Column tuple → `PersonaChatRow`.
|
||||
fn persona_chat_row(
|
||||
(conversation_id, persona_id, title, messages_json, turn_count, created_at, updated_at): (
|
||||
String,
|
||||
String,
|
||||
String,
|
||||
String,
|
||||
i32,
|
||||
i64,
|
||||
i64,
|
||||
),
|
||||
(
|
||||
conversation_id,
|
||||
persona_id,
|
||||
title,
|
||||
messages_json,
|
||||
turn_count,
|
||||
created_at,
|
||||
updated_at,
|
||||
anchor_contact_id,
|
||||
anchor_contact_name,
|
||||
anchor_message_id,
|
||||
anchor_ts,
|
||||
anchor_context,
|
||||
anchor_mode,
|
||||
): PersonaChatColumns,
|
||||
) -> PersonaChatRow {
|
||||
// An anchor needs at least the thread and the message; the rest has a
|
||||
// usable default if a row was written half-way by hand.
|
||||
let anchor = match (anchor_contact_id, anchor_message_id) {
|
||||
(Some(contact_id), Some(message_id)) => Some(SmsAnchor {
|
||||
contact_id,
|
||||
contact_name: anchor_contact_name.unwrap_or_default(),
|
||||
message_id,
|
||||
timestamp: anchor_ts.unwrap_or(0),
|
||||
context: anchor_context.unwrap_or_default(),
|
||||
mode: AnchorMode::from_column(anchor_mode.as_deref()),
|
||||
}),
|
||||
_ => None,
|
||||
};
|
||||
PersonaChatRow {
|
||||
conversation_id,
|
||||
persona_id,
|
||||
@@ -52,6 +158,7 @@ fn persona_chat_row(
|
||||
turn_count,
|
||||
created_at,
|
||||
updated_at,
|
||||
anchor,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,6 +232,15 @@ pub trait PersonaDao: Sync + Send {
|
||||
/// derived from the tree rather than denormalized into its own column.
|
||||
fn list_persona_chats(&mut self, user_id: i32) -> Result<Vec<PersonaChatRow>, DbError>;
|
||||
|
||||
/// Conversations anchored on one SMS thread, newest first. Backs the
|
||||
/// Messages screen's per-thread list; served by
|
||||
/// `idx_persona_chat_anchor_contact`.
|
||||
fn list_persona_chats_for_contact(
|
||||
&mut self,
|
||||
user_id: i32,
|
||||
contact_id: i64,
|
||||
) -> Result<Vec<PersonaChatRow>, DbError>;
|
||||
|
||||
/// Start a new conversation with a persona, returning its id. Several
|
||||
/// conversations with the same persona are expected, so this never
|
||||
/// reuses an existing row.
|
||||
@@ -135,6 +251,17 @@ pub trait PersonaDao: Sync + Send {
|
||||
created_at: i64,
|
||||
) -> Result<String, DbError>;
|
||||
|
||||
/// Start a new conversation anchored on a point in an SMS thread
|
||||
/// ("Continue with persona"). Same as `create_persona_chat` plus the
|
||||
/// anchor columns.
|
||||
fn create_anchored_persona_chat(
|
||||
&mut self,
|
||||
user_id: i32,
|
||||
persona_id: &str,
|
||||
created_at: i64,
|
||||
anchor: &SmsAnchor,
|
||||
) -> Result<String, DbError>;
|
||||
|
||||
/// Replace a conversation's transcript. Called once per completed turn
|
||||
/// with the full new `messages_json`; `turn_count` is the cumulative
|
||||
/// assistant-turn count on the resulting active branch. Returns the
|
||||
@@ -382,16 +509,8 @@ impl PersonaDao for SqlitePersonaDao {
|
||||
// Scoped by user as well as id: the id is a bearer token for
|
||||
// someone's transcript, so it never grants access on its own.
|
||||
.filter(user_id.eq(uid))
|
||||
.select((
|
||||
conversation_id,
|
||||
persona_id,
|
||||
title,
|
||||
messages_json,
|
||||
turn_count,
|
||||
created_at,
|
||||
updated_at,
|
||||
))
|
||||
.first::<(String, String, String, String, i32, i64, i64)>(conn.deref_mut())
|
||||
.select(persona_chat_select!())
|
||||
.first::<PersonaChatColumns>(conn.deref_mut())
|
||||
.optional()
|
||||
.map(|opt| opt.map(persona_chat_row))
|
||||
.map_err(|e| anyhow::anyhow!("Query error: {}", e))
|
||||
@@ -406,16 +525,28 @@ impl PersonaDao for SqlitePersonaDao {
|
||||
persona_chat_conversations
|
||||
.filter(user_id.eq(uid))
|
||||
.order(updated_at.desc())
|
||||
.select((
|
||||
conversation_id,
|
||||
persona_id,
|
||||
title,
|
||||
messages_json,
|
||||
turn_count,
|
||||
created_at,
|
||||
updated_at,
|
||||
))
|
||||
.load::<(String, String, String, String, i32, i64, i64)>(conn.deref_mut())
|
||||
.select(persona_chat_select!())
|
||||
.load::<PersonaChatColumns>(conn.deref_mut())
|
||||
.map(|rows| rows.into_iter().map(persona_chat_row).collect())
|
||||
.map_err(|e| anyhow::anyhow!("Query error: {}", e))
|
||||
})
|
||||
.map_err(|e| DbError::log(DbErrorKind::QueryError, e))
|
||||
}
|
||||
|
||||
fn list_persona_chats_for_contact(
|
||||
&mut self,
|
||||
uid: i32,
|
||||
cid: i64,
|
||||
) -> Result<Vec<PersonaChatRow>, DbError> {
|
||||
trace_db_call("query", "list_persona_chats_for_contact", |_span| {
|
||||
use schema::persona_chat_conversations::dsl::*;
|
||||
let mut conn = self.connection.lock().expect("PersonaDao lock");
|
||||
persona_chat_conversations
|
||||
.filter(user_id.eq(uid))
|
||||
.filter(anchor_contact_id.eq(cid))
|
||||
.order(updated_at.desc())
|
||||
.select(persona_chat_select!())
|
||||
.load::<PersonaChatColumns>(conn.deref_mut())
|
||||
.map(|rows| rows.into_iter().map(persona_chat_row).collect())
|
||||
.map_err(|e| anyhow::anyhow!("Query error: {}", e))
|
||||
})
|
||||
@@ -452,6 +583,41 @@ impl PersonaDao for SqlitePersonaDao {
|
||||
.map_err(|e| DbError::log(DbErrorKind::InsertError, e))
|
||||
}
|
||||
|
||||
fn create_anchored_persona_chat(
|
||||
&mut self,
|
||||
uid: i32,
|
||||
pid: &str,
|
||||
created: i64,
|
||||
anchor: &SmsAnchor,
|
||||
) -> Result<String, DbError> {
|
||||
trace_db_call("insert", "create_anchored_persona_chat", |_span| {
|
||||
use schema::persona_chat_conversations::dsl::*;
|
||||
let mut conn = self.connection.lock().expect("PersonaDao lock");
|
||||
let new_id = uuid::Uuid::new_v4().to_string();
|
||||
diesel::insert_into(persona_chat_conversations)
|
||||
.values((
|
||||
conversation_id.eq(&new_id),
|
||||
user_id.eq(uid),
|
||||
persona_id.eq(pid),
|
||||
title.eq(""),
|
||||
messages_json.eq("[]"),
|
||||
turn_count.eq(0),
|
||||
created_at.eq(created),
|
||||
updated_at.eq(created),
|
||||
anchor_contact_id.eq(Some(anchor.contact_id)),
|
||||
anchor_contact_name.eq(Some(&anchor.contact_name)),
|
||||
anchor_message_id.eq(Some(anchor.message_id)),
|
||||
anchor_ts.eq(Some(anchor.timestamp)),
|
||||
anchor_context.eq(Some(&anchor.context)),
|
||||
anchor_mode.eq(Some(anchor.mode.as_str())),
|
||||
))
|
||||
.execute(conn.deref_mut())
|
||||
.map_err(|e| anyhow::anyhow!("Insert error: {}", e))?;
|
||||
Ok(new_id)
|
||||
})
|
||||
.map_err(|e| DbError::log(DbErrorKind::InsertError, e))
|
||||
}
|
||||
|
||||
fn update_persona_chat(
|
||||
&mut self,
|
||||
uid: i32,
|
||||
@@ -766,6 +932,77 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persona_chat_anchor_round_trips_and_open_chats_have_none() {
|
||||
let (mut dao, uid) = dao_with_user("p10");
|
||||
let anchor = SmsAnchor {
|
||||
contact_id: 42,
|
||||
contact_name: "Sam".to_string(),
|
||||
message_id: 9001,
|
||||
timestamp: 1_700_000_000,
|
||||
context: "[2023-11-14 22:13] Sam: see you there".to_string(),
|
||||
mode: AnchorMode::AsContact,
|
||||
};
|
||||
let anchored = dao
|
||||
.create_anchored_persona_chat(uid, "journal", 100, &anchor)
|
||||
.unwrap();
|
||||
let open = dao.create_persona_chat(uid, "journal", 200).unwrap();
|
||||
|
||||
let row = dao.get_persona_chat(uid, &anchored).unwrap().unwrap();
|
||||
assert_eq!(row.anchor, Some(anchor));
|
||||
let row = dao.get_persona_chat(uid, &open).unwrap().unwrap();
|
||||
assert_eq!(row.anchor, None);
|
||||
assert!(
|
||||
dao.list_persona_chats(uid)
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|r| r.anchor.is_some()),
|
||||
"the list query carries the anchor too"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persona_chat_list_for_contact_returns_only_that_thread() {
|
||||
let (mut dao, uid) = dao_with_user("p11");
|
||||
let anchor = |contact_id| SmsAnchor {
|
||||
contact_id,
|
||||
contact_name: "x".to_string(),
|
||||
message_id: 1,
|
||||
timestamp: 0,
|
||||
context: String::new(),
|
||||
mode: AnchorMode::About,
|
||||
};
|
||||
let older = dao
|
||||
.create_anchored_persona_chat(uid, "journal", 100, &anchor(42))
|
||||
.unwrap();
|
||||
let _other = dao
|
||||
.create_anchored_persona_chat(uid, "journal", 150, &anchor(7))
|
||||
.unwrap();
|
||||
let _open = dao.create_persona_chat(uid, "journal", 175).unwrap();
|
||||
let newer = dao
|
||||
.create_anchored_persona_chat(uid, "journal", 200, &anchor(42))
|
||||
.unwrap();
|
||||
|
||||
let ids: Vec<String> = dao
|
||||
.list_persona_chats_for_contact(uid, 42)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|r| r.conversation_id)
|
||||
.collect();
|
||||
assert_eq!(ids, vec![newer, older]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn anchor_mode_reads_null_and_unknown_as_about() {
|
||||
assert_eq!(AnchorMode::from_column(None), AnchorMode::About);
|
||||
assert_eq!(AnchorMode::from_column(Some("weird")), AnchorMode::About);
|
||||
assert_eq!(AnchorMode::from_column(Some("about")), AnchorMode::About);
|
||||
assert_eq!(
|
||||
AnchorMode::from_column(Some(AnchorMode::AsContact.as_str())),
|
||||
AnchorMode::AsContact
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persona_chat_list_is_empty_for_a_user_who_has_never_chatted() {
|
||||
let (mut dao, uid) = dao_with_user("p8");
|
||||
|
||||
@@ -197,6 +197,12 @@ diesel::table! {
|
||||
turn_count -> Integer,
|
||||
created_at -> BigInt,
|
||||
updated_at -> BigInt,
|
||||
anchor_contact_id -> Nullable<BigInt>,
|
||||
anchor_contact_name -> Nullable<Text>,
|
||||
anchor_message_id -> Nullable<BigInt>,
|
||||
anchor_ts -> Nullable<BigInt>,
|
||||
anchor_context -> Nullable<Text>,
|
||||
anchor_mode -> Nullable<Text>,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -55,7 +55,7 @@ can be read aloud in a few seconds. Avoid generic filler like \"what a \
|
||||
wonderful day\" — if you have little to go on, simply describe the moment \
|
||||
plainly.\n\nYou may call read-only tools (search_rag, search_messages, \
|
||||
get_sms_messages, get_calendar_events, get_location_history, reverse_geocode, \
|
||||
get_personal_place_at, recall_entities, get_current_datetime) to ground each \
|
||||
get_personal_place_at, recall_entities, date_info) to ground each \
|
||||
line in real context — e.g. reverse_geocode a moment's GPS to name the place, \
|
||||
or check the calendar/messages around its date. Never invent details. Return \
|
||||
ONLY the JSON object, no prose or code fences.";
|
||||
@@ -335,7 +335,7 @@ pub async fn generate_script_agentic(
|
||||
"reverse_geocode",
|
||||
"get_personal_place_at",
|
||||
"recall_entities",
|
||||
"get_current_datetime",
|
||||
"date_info",
|
||||
]
|
||||
.into_iter()
|
||||
.collect();
|
||||
|
||||
Reference in new issue
Block a user