Apps/ImageApi
1
0 Fork 0
Code Issues 7 Pull Requests Releases Wiki Activity

feat(persona-chat): continue an SMS thread with a persona

Browse Source 
Persona conversations can now be anchored on a message in an SMS thread.
Creating one snapshots the thread up to that message (via SMS-API) into the
row; every turn appends it to the persona prompt with guidance to talk like a
text thread rather than write an insight.

Anchored turns get four tools: get_thread_messages (page the thread),
look_at_message_image (local vision on an MMS photo), get_photos_near_time
(library photos with tags and people) and look_at_photo (local vision on a
library photo). The conversation list filters by contact_id.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Cameron CordesandClaude Opus 5.5 committed 2026-10-17 18:07:49 -04:00
1 parent fdbcc75391
commit 4ac574d4d2
11 files changed
+1389 -40
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/ai/insight_generator.rs
+393 -1
View File
@@ -286,6 +286,12 @@ pub struct ToolGateOpts {
/// tool name still 404s, and the agent doesn't waste iterations
/// trying corrections it isn't allowed to do.
pub allow_agent_corrections: bool,
/// Persona chats anchored on an SMS thread ("Continue with persona")
/// get the thread tools: page the thread, look at an MMS photo, find
/// library photos around a moment and look at one. Off everywhere else
/// — the file chat already has its photo, and an open chat has no
/// thread to page.
pub sms_thread_tools: bool,
}
impl InsightGenerator {
@@ -421,6 +427,12 @@ impl InsightGenerator {
/// tool-definition gating (no point offering `get_personal_place_at`
/// when Apollo is unreachable) — exposed publicly so `insight_chat`
/// can call `build_tool_definitions` with the same flag.
/// The SMS-API client, for callers outside the tool loop (the anchored
/// persona chat snapshots a thread with it at creation).
pub fn sms_client(&self) -> &SmsApiClient {
&self.sms_client
}
pub fn apollo_enabled(&self) -> bool {
self.apollo_client.is_enabled()
}
@@ -485,6 +497,7 @@ impl InsightGenerator {
location_history_present,
faces_present,
allow_agent_corrections,
sms_thread_tools: false,
}
}
@@ -1925,6 +1938,13 @@ Return ONLY the summary, nothing else."#,
.await
}
"get_current_datetime" => Self::tool_get_current_datetime(),
"get_thread_messages" => self.tool_get_thread_messages(arguments).await,
"look_at_message_image" => {
self.tool_look_at_message_image(arguments, backend.local())
.await
}
"get_photos_near_time" => self.tool_get_photos_near_time(arguments).await,
"look_at_photo" => self.tool_look_at_photo(arguments, backend.local()).await,
unknown => format!("Unknown tool: {}", unknown),
};
if result.starts_with("Error") || result.starts_with("No ") {
@@ -2386,6 +2406,256 @@ Return ONLY the summary, nothing else."#,
}
}
/// Tool: get_thread_messages — page the anchored SMS thread from a
/// message id. Lines are rendered exactly like the anchor snapshot so
/// the model can keep paging from any id it sees.
async fn tool_get_thread_messages(&self, args: &serde_json::Value) -> String {
let Some(contact_id) = args.get("contact_id").and_then(|v| v.as_i64()) else {
return "Error: missing required parameter 'contact_id'".to_string();
};
let Some(message_id) = args.get("message_id").and_then(|v| v.as_i64()) else {
return "Error: missing required parameter 'message_id'".to_string();
};
let limit = args
.get("limit")
.and_then(|v| v.as_i64())
.unwrap_or(30)
.clamp(1, 80) as usize;
let (before, after) = match args.get("direction").and_then(|v| v.as_str()) {
Some("after") => (0, limit),
Some("around") => (limit / 2, limit - limit / 2),
_ => (limit, 0),
};
match self
.sms_client
.fetch_thread_window(contact_id, message_id, before, after)
.await
{
Ok(window) if window.messages.len() > 1 => {
let rendered = crate::ai::sms_anchor::render_thread_window(
&window,
&user_display_name(),
self.display_zone(),
);
format!(
"Found {} messages with {} (▶ marks #{}):\n{}",
window.messages.len(),
window.contact.display_name(),
message_id,
rendered
)
}
Ok(_) => "No more messages in that direction.".to_string(),
Err(e) => {
log::warn!("tool_get_thread_messages failed: {}", e);
format!("Error fetching thread messages: {}", e)
}
}
}
/// Tool: look_at_message_image — download an MMS photo from SMS-API and
/// describe it with the local vision model.
async fn tool_look_at_message_image(
&self,
args: &serde_json::Value,
local: &dyn LlmClient,
) -> String {
let Some(contact_id) = args.get("contact_id").and_then(|v| v.as_i64()) else {
return "Error: missing required parameter 'contact_id'".to_string();
};
let Some(message_id) = args.get("message_id").and_then(|v| v.as_i64()) else {
return "Error: missing required parameter 'message_id'".to_string();
};
let media_index = args
.get("media_index")
.and_then(|v| v.as_u64())
.unwrap_or(0) as usize;
let bytes = match self
.sms_client
.fetch_mms_media(contact_id, message_id, media_index)
.await
{
Ok(b) => b,
Err(e) => return format!("Error fetching attachment: {}", e),
};
let image_base64 = match Self::encode_image_bytes_for_vision(&bytes) {
Ok(b64) => b64,
Err(_) => {
return "Error: that attachment isn't a viewable image (videos, audio and some \
formats can't be looked at)."
.to_string();
}
};
match local.describe_image(&image_base64).await {
Ok(desc) => desc,
Err(e) => format!("Error describing attachment: {}", e),
}
}
/// Tool: get_photos_near_time — library photos within ±hours of a
/// moment, closest first, with tags and recognised people.
async fn tool_get_photos_near_time(&self, args: &serde_json::Value) -> String {
let zone = self.display_zone();
let server_zone = crate::camera_tz::server_zone();
let Some(center) = Self::parse_moment(args, zone) else {
return "Error: pass 'timestamp' (unix seconds) or 'datetime' (YYYY-MM-DD HH:MM)"
.to_string();
};
let hours = args
.get("hours_radius")
.and_then(|v| v.as_i64())
.unwrap_or(3)
.clamp(1, 72);
let limit = args
.get("limit")
.and_then(|v| v.as_i64())
.unwrap_or(15)
.clamp(1, 40) as usize;
let range = crate::date_view::InstantRange::new(
Some(center - hours * 3600),
Some(center + hours * 3600),
zone,
server_zone,
);
let rows = {
let mut dao = self.exif_dao.lock().expect("Unable to lock ExifDao");
match dao.query_by_exif(None, None, None, None, None, range) {
Ok(r) => r,
Err(e) => return format!("Error querying photos: {:?}", e),
}
};
// Closest first; the same bytes in two libraries list once.
let mut dated: Vec<(i64, crate::database::models::ImageExif)> = rows
.into_iter()
.filter_map(|r| {
crate::date_view::row_true_instant(&r, zone, server_zone).map(|t| (t, r))
})
.collect();
dated.sort_by_key(|(t, _)| (t - center).abs());
let mut seen = std::collections::HashSet::new();
dated.retain(|(_, r)| match &r.content_hash {
Some(h) => seen.insert(h.clone()),
None => true,
});
let total = dated.len();
if total == 0 {
return format!("No photos within ±{} hours of that moment.", hours);
}
let mut lines = Vec::new();
for (_, row) in dated.into_iter().take(limit) {
let when = crate::date_view::row_display_wall(&row, zone, server_zone)
.map(|w| w.format("%Y-%m-%d %H:%M").to_string())
.unwrap_or_else(|| "unknown time".to_string());
let tags: Vec<String> = {
let mut dao = self.tag_dao.lock().expect("Unable to lock TagDao");
dao.get_tags_for_path(&row.file_path)
.map(|t| t.into_iter().map(|t| t.name).collect())
.unwrap_or_default()
};
let people: Vec<String> = match &row.content_hash {
Some(hash) => {
let mut dao = self.face_dao.lock().expect("Unable to lock FaceDao");
let mut names: Vec<String> = dao
.list_for_content_hash(hash)
.map(|faces| faces.into_iter().filter_map(|f| f.person_name).collect())
.unwrap_or_default();
names.sort();
names.dedup();
names
}
None => Vec::new(),
};
let mut line = format!("- {} — {}", row.file_path, when);
if !tags.is_empty() {
line.push_str(&format!(" · tags: {}", tags.join(", ")));
}
if !people.is_empty() {
line.push_str(&format!(" · people: {}", people.join(", ")));
}
if crate::file_types::is_video_file(std::path::Path::new(&row.file_path)) {
line.push_str(" · video");
}
lines.push(line);
}
format!(
"{}\n{}",
Self::found_header(total, lines.len(), "photos"),
lines.join("\n")
)
}
/// Tool: look_at_photo — describe a library photo by path with the
/// local vision model.
async fn tool_look_at_photo(&self, args: &serde_json::Value, local: &dyn LlmClient) -> String {
let file_path = match args.get("file_path").and_then(|v| v.as_str()) {
Some(p) if Self::photo_path_is_safe(p.trim()) => p.trim().to_string(),
Some(_) => return "Error: file_path must be a library-relative path".to_string(),
None => return "Error: missing required parameter 'file_path'".to_string(),
};
if crate::file_types::is_video_file(std::path::Path::new(&file_path)) {
return "Error: that's a video — only photos can be looked at.".to_string();
}
let image_base64 = match self.load_image_as_base64(&file_path) {
Ok(b) => b,
Err(e) => return format!("Error loading photo: {}", e),
};
match local.describe_image(&image_base64).await {
Ok(desc) => desc,
Err(e) => format!("Error describing photo: {}", e),
}
}
/// A model-supplied path is joined onto each library root, so refuse
/// anything that could climb out of it.
fn photo_path_is_safe(path: &str) -> bool {
use std::path::{Component, Path};
!path.is_empty()
&& Path::new(path)
.components()
.all(|c| matches!(c, Component::Normal(_) | Component::CurDir))
}
/// A tool's centre moment: `timestamp` (unix seconds) wins, else
/// `datetime` read as a wall clock in the owner's display zone. A bare
/// date centres on midday.
fn parse_moment(args: &serde_json::Value, zone: chrono_tz::Tz) -> Option<i64> {
if let Some(ts) = args.get("timestamp").and_then(|v| v.as_i64()) {
return Some(ts);
}
let raw = args.get("datetime").and_then(|v| v.as_str())?.trim();
let naive = chrono::NaiveDateTime::parse_from_str(raw, "%Y-%m-%d %H:%M")
.or_else(|_| chrono::NaiveDateTime::parse_from_str(raw, "%Y-%m-%dT%H:%M"))
.or_else(|_| chrono::NaiveDateTime::parse_from_str(raw, "%Y-%m-%dT%H:%M:%S"))
.ok()
.or_else(|| {
NaiveDate::parse_from_str(raw, "%Y-%m-%d")
.ok()
.and_then(|d| d.and_hms_opt(12, 0, 0))
})?;
zone.from_local_datetime(&naive)
.earliest()
.map(|dt| dt.timestamp())
}
/// Decode arbitrary image bytes (an MMS attachment), shrink to the
/// vision models' 1024px working size and re-encode as base64 JPEG —
/// the same shape `load_image_as_base64` produces for library files.
fn encode_image_bytes_for_vision(bytes: &[u8]) -> Result<String> {
use image::imageops::FilterType;
let img = image::load_from_memory(bytes)?;
// JPEG has no alpha channel; a PNG screenshot would fail to encode.
let resized =
image::DynamicImage::ImageRgb8(img.resize(1024, 1024, FilterType::Lanczos3).to_rgb8());
let mut buffer = Vec::new();
resized.write_to(&mut Cursor::new(&mut buffer), ImageFormat::Jpeg)?;
Ok(base64::engine::general_purpose::STANDARD.encode(&buffer))
}
/// Tool: get_calendar_events — fetch calendar events near a date
async fn tool_get_calendar_events(&self, args: &serde_json::Value) -> String {
let date_str = match args.get("date").and_then(|v| v.as_str()) {
@@ -3781,6 +4051,77 @@ Return ONLY the summary, nothing else."#,
}),
));
if opts.sms_thread_tools {
tools.push(Tool::function(
"get_thread_messages",
"Read more of the SMS thread this chat was opened from. Pages from a message id in the \
thread (the #numbers in the transcript) — `before` for older messages, `after` for what \
came next, `around` for both sides. Lines carry their own #id, so call again from the \
oldest/newest id to keep paging. \
Example: `{contact_id: 42, message_id: 812, direction: \"before\", limit: 30}`.",
serde_json::json!({
"type": "object",
"required": ["contact_id", "message_id"],
"properties": {
"contact_id": { "type": "integer", "description": "The thread's contact_id (given in the system prompt)." },
"message_id": { "type": "integer", "description": "Message id to page from." },
"direction": { "type": "string", "enum": ["before", "after", "around"], "description": "Which side of message_id to read (default before)." },
"limit": { "type": "integer", "description": "Max messages (default 30, max 80)." }
}
}),
));
tools.push(Tool::function(
"look_at_message_image",
"Look at a photo that was sent in the SMS thread, using the local vision model. Returns a \
visual description. Attachments appear in the transcript as `image #N` on a message line — \
pass that message's id and N as media_index. Videos and audio can't be viewed. \
Example: `{contact_id: 42, message_id: 815, media_index: 0}`.",
serde_json::json!({
"type": "object",
"required": ["contact_id", "message_id"],
"properties": {
"contact_id": { "type": "integer", "description": "The thread's contact_id." },
"message_id": { "type": "integer", "description": "Id of the message carrying the photo." },
"media_index": { "type": "integer", "description": "Which attachment on that message (default 0)." }
}
}),
));
tools.push(Tool::function(
"get_photos_near_time",
"List photos and videos from the user's library taken around a moment, closest first, with \
their user-applied tags and the names of people recognised in them. Use to see what was \
happening around a message (pass its time). Follow up with `look_at_photo` to see one. \
Pass either `timestamp` (unix seconds) or `datetime` (local, \"YYYY-MM-DD HH:MM\" or \
\"YYYY-MM-DD\"). \
Example: `{datetime: \"2019-06-12 18:30\", hours_radius: 4}`.",
serde_json::json!({
"type": "object",
"properties": {
"timestamp": { "type": "integer", "description": "Centre moment, unix seconds." },
"datetime": { "type": "string", "description": "Centre moment in local time, YYYY-MM-DD HH:MM (or YYYY-MM-DD for the whole day)." },
"hours_radius": { "type": "integer", "description": "Hours before and after to include (default 3, max 72)." },
"limit": { "type": "integer", "description": "Max photos (default 15, max 40)." }
}
}),
));
tools.push(Tool::function(
"look_at_photo",
"Look at one photo from the user's library with the local vision model and return a visual \
description — people, place, activity, mood. Pass a file_path from `get_photos_near_time`. \
Example: `{file_path: \"2019/06/IMG_4242.jpg\"}`.",
serde_json::json!({
"type": "object",
"required": ["file_path"],
"properties": {
"file_path": { "type": "string", "description": "Library-relative path of the photo." }
}
}),
));
}
if opts.has_vision {
tools.push(Tool::function(
"describe_photo",
@@ -3915,7 +4256,8 @@ Return ONLY the summary, nothing else."#,
&& let Ok(n) = n_str.parse::<usize>()
{
let kind = match tool_name {
"search_messages" | "get_sms_messages" => "messages",
"search_messages" | "get_sms_messages" | "get_thread_messages" => "messages",
"get_photos_near_time" => "photos",
"get_calendar_events" => "events",
"get_location_history" => "location records",
_ => "results",
@@ -3964,6 +4306,10 @@ Return ONLY the summary, nothing else."#,
.map(|id| format!("stored id={}", id.trim_end_matches(',')))
.unwrap_or_else(|| "stored".to_string()),
"get_current_datetime" => "time noted".to_string(),
"look_at_message_image" | "look_at_photo" => {
let short: String = raw.chars().take(80).collect();
format!("looked: \"{}...\"", short)
}
_ => format!("{} chars", raw.len()),
}
}
@@ -4882,6 +5228,7 @@ mod tests {
location_history_present: false,
faces_present: false,
allow_agent_corrections: false,
sms_thread_tools: false,
};
let tools = InsightGenerator::build_tool_definitions(opts);
let names: Vec<&str> = tools.iter().map(|t| t.function.name.as_str()).collect();
@@ -4908,6 +5255,11 @@ mod tests {
// Agent-correction tools are absent without the gate.
assert!(!names.contains(&"update_fact"));
assert!(!names.contains(&"supersede_fact"));
// SMS-thread tools only exist for an anchored persona chat.
assert!(!names.contains(&"get_thread_messages"));
assert!(!names.contains(&"look_at_message_image"));
assert!(!names.contains(&"get_photos_near_time"));
assert!(!names.contains(&"look_at_photo"));
}
#[test]
@@ -4920,6 +5272,7 @@ mod tests {
location_history_present: true,
faces_present: true,
allow_agent_corrections: true,
sms_thread_tools: true,
};
let tools = InsightGenerator::build_tool_definitions(opts);
let names: Vec<&str> = tools.iter().map(|t| t.function.name.as_str()).collect();
@@ -4931,6 +5284,45 @@ mod tests {
assert!(names.contains(&"get_faces_in_photo"));
assert!(names.contains(&"update_fact"));
assert!(names.contains(&"supersede_fact"));
assert!(names.contains(&"get_thread_messages"));
assert!(names.contains(&"look_at_message_image"));
assert!(names.contains(&"get_photos_near_time"));
assert!(names.contains(&"look_at_photo"));
}
#[test]
fn photo_path_is_safe_rejects_escapes() {
assert!(InsightGenerator::photo_path_is_safe("2019/06/IMG_1.jpg"));
assert!(!InsightGenerator::photo_path_is_safe("../etc/passwd"));
assert!(!InsightGenerator::photo_path_is_safe("2019/../../x.jpg"));
assert!(!InsightGenerator::photo_path_is_safe("/etc/passwd"));
assert!(!InsightGenerator::photo_path_is_safe(""));
}
#[test]
fn parse_moment_accepts_unix_and_local_wall_clock() {
let tz = chrono_tz::Tz::America__New_York;
assert_eq!(
InsightGenerator::parse_moment(&serde_json::json!({"timestamp": 1_700_000_000}), tz),
Some(1_700_000_000)
);
// 2023-11-14 17:13 in New York (EST) = 22:13 UTC.
assert_eq!(
InsightGenerator::parse_moment(
&serde_json::json!({"datetime": "2023-11-14 17:13"}),
tz
),
Some(1_699_999_980)
);
// A bare date means midday, the least-wrong centre for a whole day.
assert_eq!(
InsightGenerator::parse_moment(&serde_json::json!({"datetime": "2023-11-14"}), tz),
Some(1_699_981_200)
);
assert_eq!(
InsightGenerator::parse_moment(&serde_json::json!({}), tz),
None
);
}
fn place(name: &str, description: &str) -> ApolloPlace {