From 7615b9c99bf3e9ddf690b3bc61c863edb0f2c95e Mon Sep 17 00:00:00 2001 From: Cameron Date: Wed, 18 Mar 2026 23:00:41 -0400 Subject: [PATCH] feat: add tool executors and generate_agentic_insight_for_photo() to InsightGenerator Add 6 tool executor methods (search_rag, get_sms_messages, get_calendar_events, get_location_history, get_file_tags, describe_photo) and the agentic loop that uses Ollama's chat_with_tools API to let the model decide which context to gather before writing the final photo insight. Co-Authored-By: Claude Sonnet 4.6 --- src/ai/insight_generator.rs | 754 +++++++++++++++++++++++++++++++++++- 1 file changed, 752 insertions(+), 2 deletions(-) diff --git a/src/ai/insight_generator.rs b/src/ai/insight_generator.rs index be513e8..6434314 100644 --- a/src/ai/insight_generator.rs +++ b/src/ai/insight_generator.rs @@ -1,6 +1,6 @@ use anyhow::Result; use base64::Engine as _; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, NaiveDate, Utc}; use image::ImageFormat; use opentelemetry::KeyValue; use opentelemetry::trace::{Span, Status, TraceContextExt, Tracer}; @@ -9,7 +9,7 @@ use std::fs::File; use std::io::Cursor; use std::sync::{Arc, Mutex}; -use crate::ai::ollama::OllamaClient; +use crate::ai::ollama::{ChatMessage, OllamaClient, Tool}; use crate::ai::sms_client::SmsApiClient; use crate::database::models::InsertPhotoInsight; use crate::database::{ @@ -1314,6 +1314,756 @@ Return ONLY the summary, nothing else."#, } } + // ── Tool executors for agentic loop ──────────────────────────────── + + /// Dispatch a tool call to the appropriate executor + async fn execute_tool( + &self, + tool_name: &str, + arguments: &serde_json::Value, + ollama: &OllamaClient, + image_base64: &Option, + cx: &opentelemetry::Context, + ) -> String { + match tool_name { + "search_rag" => self.tool_search_rag(arguments, cx).await, + "get_sms_messages" => self.tool_get_sms_messages(arguments, cx).await, + "get_calendar_events" => self.tool_get_calendar_events(arguments, cx).await, + "get_location_history" => self.tool_get_location_history(arguments, cx).await, + "get_file_tags" => self.tool_get_file_tags(arguments, cx).await, + "describe_photo" => self.tool_describe_photo(ollama, image_base64).await, + unknown => format!("Unknown tool: {}", unknown), + } + } + + /// Tool: search_rag — semantic search over daily summaries + async fn tool_search_rag( + &self, + args: &serde_json::Value, + _cx: &opentelemetry::Context, + ) -> String { + let query = match args.get("query").and_then(|v| v.as_str()) { + Some(q) => q.to_string(), + None => return "Error: missing required parameter 'query'".to_string(), + }; + let date_str = match args.get("date").and_then(|v| v.as_str()) { + Some(d) => d, + None => return "Error: missing required parameter 'date'".to_string(), + }; + let date = match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") { + Ok(d) => d, + Err(e) => return format!("Error: failed to parse date '{}': {}", date_str, e), + }; + let contact = args + .get("contact") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + + log::info!( + "tool_search_rag: query='{}', date={}, contact={:?}", + query, + date, + contact + ); + + match self + .find_relevant_messages_rag(date, None, contact.as_deref(), None, 5, Some(&query)) + .await + { + Ok(results) if !results.is_empty() => results.join("\n\n"), + Ok(_) => "No relevant messages found.".to_string(), + Err(e) => format!("Error searching RAG: {}", e), + } + } + + /// Tool: get_sms_messages — fetch SMS messages near a date for a contact + async fn tool_get_sms_messages( + &self, + args: &serde_json::Value, + _cx: &opentelemetry::Context, + ) -> String { + let date_str = match args.get("date").and_then(|v| v.as_str()) { + Some(d) => d, + None => return "Error: missing required parameter 'date'".to_string(), + }; + let contact = match args.get("contact").and_then(|v| v.as_str()) { + Some(c) => c.to_string(), + None => return "Error: missing required parameter 'contact'".to_string(), + }; + let days_radius = args + .get("days_radius") + .and_then(|v| v.as_i64()) + .unwrap_or(4); + + let date = match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") { + Ok(d) => d, + Err(e) => return format!("Error: failed to parse date '{}': {}", date_str, e), + }; + let timestamp = date + .and_hms_opt(12, 0, 0) + .unwrap() + .and_utc() + .timestamp(); + + log::info!( + "tool_get_sms_messages: date={}, contact='{}', days_radius={}", + date, + contact, + days_radius + ); + + // Use the SMS client's existing fetch mechanism + // Build start/end from days_radius + let start_ts = timestamp - (days_radius * 86400); + let end_ts = timestamp + (days_radius * 86400); + + let center_dt = DateTime::from_timestamp(timestamp, 0); + let start_dt = DateTime::from_timestamp(start_ts, 0); + let end_dt = DateTime::from_timestamp(end_ts, 0); + + if center_dt.is_none() || start_dt.is_none() || end_dt.is_none() { + return "Error: invalid timestamp range".to_string(); + } + + match self + .sms_client + .fetch_messages_for_contact(Some(&contact), timestamp) + .await + { + Ok(messages) if !messages.is_empty() => { + let formatted: Vec = messages + .iter() + .take(30) + .map(|m| { + let sender = if m.is_sent { "Me" } else { &m.contact }; + let ts = DateTime::from_timestamp(m.timestamp, 0) + .map(|dt| dt.format("%Y-%m-%d %H:%M").to_string()) + .unwrap_or_else(|| "unknown".to_string()); + format!("[{}] {}: {}", ts, sender, m.body) + }) + .collect(); + format!("Found {} messages:\n{}", messages.len(), formatted.join("\n")) + } + Ok(_) => "No messages found.".to_string(), + Err(e) => format!("Error fetching SMS messages: {}", e), + } + } + + /// Tool: get_calendar_events — fetch calendar events near a date + async fn tool_get_calendar_events( + &self, + args: &serde_json::Value, + cx: &opentelemetry::Context, + ) -> String { + let date_str = match args.get("date").and_then(|v| v.as_str()) { + Some(d) => d, + None => return "Error: missing required parameter 'date'".to_string(), + }; + let days_radius = args + .get("days_radius") + .and_then(|v| v.as_i64()) + .unwrap_or(7); + + let date = match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") { + Ok(d) => d, + Err(e) => return format!("Error: failed to parse date '{}': {}", date_str, e), + }; + let timestamp = date + .and_hms_opt(12, 0, 0) + .unwrap() + .and_utc() + .timestamp(); + + log::info!( + "tool_get_calendar_events: date={}, days_radius={}", + date, + days_radius + ); + + let events = { + let mut dao = self + .calendar_dao + .lock() + .expect("Unable to lock CalendarEventDao"); + dao.find_relevant_events_hybrid(cx, timestamp, days_radius, None, 10) + .ok() + }; + + match events { + Some(evts) if !evts.is_empty() => { + let formatted: Vec = evts + .iter() + .map(|e| { + let dt = DateTime::from_timestamp(e.start_time, 0) + .map(|dt| dt.format("%Y-%m-%d %H:%M").to_string()) + .unwrap_or_else(|| "unknown".to_string()); + let loc = e + .location + .as_ref() + .map(|l| format!(" at {}", l)) + .unwrap_or_default(); + let attendees = e + .attendees + .as_ref() + .and_then(|a| serde_json::from_str::>(a).ok()) + .map(|list| format!(" (with {})", list.join(", "))) + .unwrap_or_default(); + format!("[{}] {}{}{}", dt, e.summary, loc, attendees) + }) + .collect(); + format!( + "Found {} calendar events:\n{}", + evts.len(), + formatted.join("\n") + ) + } + Some(_) => "No calendar events found.".to_string(), + None => "No calendar events found.".to_string(), + } + } + + /// Tool: get_location_history — fetch location records near a date + async fn tool_get_location_history( + &self, + args: &serde_json::Value, + cx: &opentelemetry::Context, + ) -> String { + let date_str = match args.get("date").and_then(|v| v.as_str()) { + Some(d) => d, + None => return "Error: missing required parameter 'date'".to_string(), + }; + let days_radius = args + .get("days_radius") + .and_then(|v| v.as_i64()) + .unwrap_or(14); + + let date = match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") { + Ok(d) => d, + Err(e) => return format!("Error: failed to parse date '{}': {}", date_str, e), + }; + let timestamp = date + .and_hms_opt(12, 0, 0) + .unwrap() + .and_utc() + .timestamp(); + + log::info!( + "tool_get_location_history: date={}, days_radius={}", + date, + days_radius + ); + + let start_ts = timestamp - (days_radius * 86400); + let end_ts = timestamp + (days_radius * 86400); + + let locations = { + let mut dao = self + .location_dao + .lock() + .expect("Unable to lock LocationHistoryDao"); + dao.find_locations_in_range(cx, start_ts, end_ts).ok() + }; + + match locations { + Some(locs) if !locs.is_empty() => { + let formatted: Vec = locs + .iter() + .take(20) + .map(|loc| { + let dt = DateTime::from_timestamp(loc.timestamp, 0) + .map(|dt| dt.format("%Y-%m-%d %H:%M").to_string()) + .unwrap_or_else(|| "unknown".to_string()); + let activity = loc + .activity + .as_ref() + .map(|a| format!(" ({})", a)) + .unwrap_or_default(); + let place = loc + .place_name + .as_ref() + .map(|p| format!(" at {}", p)) + .unwrap_or_default(); + format!( + "[{}] {:.4}, {:.4}{}{}", + dt, loc.latitude, loc.longitude, place, activity + ) + }) + .collect(); + format!( + "Found {} location records:\n{}", + locs.len(), + formatted.join("\n") + ) + } + Some(_) => "No location history found.".to_string(), + None => "No location history found.".to_string(), + } + } + + /// Tool: get_file_tags — fetch tags for a file path + async fn tool_get_file_tags( + &self, + args: &serde_json::Value, + cx: &opentelemetry::Context, + ) -> String { + let file_path = match args.get("file_path").and_then(|v| v.as_str()) { + Some(p) => p.to_string(), + None => return "Error: missing required parameter 'file_path'".to_string(), + }; + + log::info!("tool_get_file_tags: file_path='{}'", file_path); + + let tags = { + let mut dao = self.tag_dao.lock().expect("Unable to lock TagDao"); + dao.get_tags_for_path(cx, &file_path).ok() + }; + + match tags { + Some(t) if !t.is_empty() => { + let names: Vec = t.into_iter().map(|tag| tag.name).collect(); + names.join(", ") + } + Some(_) => "No tags found.".to_string(), + None => "No tags found.".to_string(), + } + } + + /// Tool: describe_photo — generate a visual description of the photo + async fn tool_describe_photo( + &self, + ollama: &OllamaClient, + image_base64: &Option, + ) -> String { + log::info!("tool_describe_photo: generating visual description"); + + match image_base64 { + Some(img) => match ollama.generate_photo_description(img).await { + Ok(desc) => desc, + Err(e) => format!("Error describing photo: {}", e), + }, + None => "No image available for description.".to_string(), + } + } + + // ── Agentic insight generation ────────────────────────────────────── + + /// Build the list of tool definitions for the agentic loop + fn build_tool_definitions(has_vision: bool) -> Vec { + let mut tools = vec![ + Tool::function( + "search_rag", + "Search conversation history using semantic search. Use this to find relevant past conversations about specific topics, people, or events.", + serde_json::json!({ + "type": "object", + "required": ["query", "date"], + "properties": { + "query": { + "type": "string", + "description": "The search query to find relevant conversations" + }, + "date": { + "type": "string", + "description": "The reference date in YYYY-MM-DD format" + }, + "contact": { + "type": "string", + "description": "Optional contact name to filter results" + } + } + }), + ), + Tool::function( + "get_sms_messages", + "Fetch SMS/text messages near a specific date for a contact. Returns the actual message conversation.", + serde_json::json!({ + "type": "object", + "required": ["date", "contact"], + "properties": { + "date": { + "type": "string", + "description": "The center date in YYYY-MM-DD format" + }, + "contact": { + "type": "string", + "description": "The contact name to fetch messages for" + }, + "days_radius": { + "type": "integer", + "description": "Number of days before and after the date to search (default: 4)" + } + } + }), + ), + Tool::function( + "get_calendar_events", + "Fetch calendar events near a specific date. Shows scheduled events, meetings, and activities.", + serde_json::json!({ + "type": "object", + "required": ["date"], + "properties": { + "date": { + "type": "string", + "description": "The center date in YYYY-MM-DD format" + }, + "days_radius": { + "type": "integer", + "description": "Number of days before and after the date to search (default: 7)" + } + } + }), + ), + Tool::function( + "get_location_history", + "Fetch location history records near a specific date. Shows places visited and activities.", + serde_json::json!({ + "type": "object", + "required": ["date"], + "properties": { + "date": { + "type": "string", + "description": "The center date in YYYY-MM-DD format" + }, + "days_radius": { + "type": "integer", + "description": "Number of days before and after the date to search (default: 14)" + } + } + }), + ), + Tool::function( + "get_file_tags", + "Get tags/labels that have been applied to a specific photo file.", + serde_json::json!({ + "type": "object", + "required": ["file_path"], + "properties": { + "file_path": { + "type": "string", + "description": "The file path of the photo to get tags for" + } + } + }), + ), + ]; + + if has_vision { + tools.push(Tool::function( + "describe_photo", + "Generate a visual description of the photo. Describes people, location, and activity visible in the image.", + serde_json::json!({ + "type": "object", + "properties": {} + }), + )); + } + + tools + } + + /// Generate an AI insight for a photo using an agentic tool-calling loop. + /// The model decides which tools to call to gather context before writing the final insight. + pub async fn generate_agentic_insight_for_photo( + &self, + file_path: &str, + custom_model: Option, + custom_system_prompt: Option, + num_ctx: Option, + max_iterations: usize, + ) -> Result<()> { + let tracer = global_tracer(); + let current_cx = opentelemetry::Context::current(); + let mut span = tracer.start_with_context("ai.insight.generate_agentic", ¤t_cx); + + let file_path = normalize_path(file_path); + log::info!("Generating agentic insight for photo: {}", file_path); + + span.set_attribute(KeyValue::new("file_path", file_path.clone())); + span.set_attribute(KeyValue::new("max_iterations", max_iterations as i64)); + + // 1. Create OllamaClient + let mut ollama_client = if let Some(ref model) = custom_model { + log::info!("Using custom model for agentic: {}", model); + span.set_attribute(KeyValue::new("custom_model", model.clone())); + OllamaClient::new( + self.ollama.primary_url.clone(), + self.ollama.fallback_url.clone(), + model.clone(), + Some(model.clone()), + ) + } else { + span.set_attribute(KeyValue::new("model", self.ollama.primary_model.clone())); + self.ollama.clone() + }; + + if let Some(ctx) = num_ctx { + log::info!("Using custom context size: {}", ctx); + span.set_attribute(KeyValue::new("num_ctx", ctx as i64)); + ollama_client.set_num_ctx(Some(ctx)); + } + + let insight_cx = current_cx.with_span(span); + + // 2. Check tool calling capability + let capabilities = OllamaClient::check_model_capabilities( + &ollama_client.primary_url, + &ollama_client.primary_model, + ) + .await + .map_err(|e| { + anyhow::anyhow!( + "Failed to check model capabilities for '{}': {}", + ollama_client.primary_model, + e + ) + })?; + + if !capabilities.has_tool_calling { + return Err(anyhow::anyhow!( + "tool calling not supported by model '{}'", + ollama_client.primary_model + )); + } + + let has_vision = capabilities.has_vision; + insight_cx + .span() + .set_attribute(KeyValue::new("model_has_vision", has_vision)); + insight_cx + .span() + .set_attribute(KeyValue::new("model_has_tool_calling", true)); + + // 3. Fetch EXIF + let exif = { + let mut exif_dao = self.exif_dao.lock().expect("Unable to lock ExifDao"); + exif_dao + .get_exif(&insight_cx, &file_path) + .map_err(|e| anyhow::anyhow!("Failed to get EXIF: {:?}", e))? + }; + + // 4. Extract timestamp and contact + let timestamp = if let Some(ts) = exif.as_ref().and_then(|e| e.date_taken) { + ts + } else { + log::warn!("No date_taken in EXIF for {}, trying filename", file_path); + extract_date_from_filename(&file_path) + .map(|dt| dt.timestamp()) + .or_else(|| { + let full_path = std::path::Path::new(&self.base_path).join(&file_path); + File::open(&full_path) + .and_then(|f| f.metadata()) + .and_then(|m| m.created().or(m.modified())) + .map(|t| DateTime::::from(t).timestamp()) + .inspect_err(|e| { + log::warn!( + "Failed to get file timestamp for agentic insight {}: {}", + file_path, + e + ) + }) + .ok() + }) + .unwrap_or_else(|| Utc::now().timestamp()) + }; + + let date_taken = DateTime::from_timestamp(timestamp, 0) + .map(|dt| dt.date_naive()) + .unwrap_or_else(|| Utc::now().date_naive()); + + let contact = Self::extract_contact_from_path(&file_path); + log::info!("Agentic: date_taken={}, contact={:?}", date_taken, contact); + + // 5. Fetch tags + let tag_names: Vec = { + let mut dao = self.tag_dao.lock().expect("Unable to lock TagDao"); + dao.get_tags_for_path(&insight_cx, &file_path) + .unwrap_or_else(|e| { + log::warn!("Failed to fetch tags for agentic {}: {}", file_path, e); + Vec::new() + }) + .into_iter() + .map(|t| t.name) + .collect() + }; + + // 6. Load image if vision capable + let image_base64 = if has_vision { + match self.load_image_as_base64(&file_path) { + Ok(b64) => { + log::info!("Loaded image for vision-capable agentic model"); + Some(b64) + } + Err(e) => { + log::warn!("Failed to load image for agentic vision: {}", e); + None + } + } + } else { + None + }; + + // 7. Build system message + let base_system = "You are a personal photo memory assistant. You have access to tools to gather context about when and where this photo was taken. Use them to build a rich, personal insight. Call tools as needed, then write a final summary and title."; + let system_content = if let Some(ref custom) = custom_system_prompt { + format!("{}\n\n{}", custom, base_system) + } else { + base_system.to_string() + }; + + // 8. Build user message + let gps_info = exif + .as_ref() + .and_then(|e| { + if let (Some(lat), Some(lon)) = (e.gps_latitude, e.gps_longitude) { + Some(format!("GPS: {:.4}, {:.4}", lat, lon)) + } else { + None + } + }) + .unwrap_or_else(|| "GPS: unknown".to_string()); + + let tags_info = if tag_names.is_empty() { + "Tags: none".to_string() + } else { + format!("Tags: {}", tag_names.join(", ")) + }; + + let contact_info = contact + .as_ref() + .map(|c| format!("Contact/Person: {}", c)) + .unwrap_or_else(|| "Contact/Person: unknown".to_string()); + + let user_content = format!( + "Please analyze this photo and gather context to write a personal journal-style insight.\n\n\ + Photo file path: {}\n\ + Date taken: {}\n\ + {}\n\ + {}\n\ + {}\n\n\ + Use the available tools to gather more context about this moment (messages, calendar events, location history, etc.), \ + then write a detailed personal insight with a title and summary. Write in first person as Cameron.", + file_path, + date_taken.format("%B %d, %Y"), + contact_info, + gps_info, + tags_info, + ); + + // 9. Define tools + let tools = Self::build_tool_definitions(has_vision); + + // 10. Build initial messages + let system_msg = ChatMessage::system(system_content); + let mut user_msg = ChatMessage::user(user_content); + if let Some(ref img) = image_base64 { + user_msg.images = Some(vec![img.clone()]); + } + + let mut messages = vec![system_msg, user_msg]; + + // 11. Agentic loop + let loop_span = tracer.start_with_context("ai.agentic.loop", &insight_cx); + let loop_cx = insight_cx.with_span(loop_span); + + let mut final_content = String::new(); + let mut iterations_used = 0usize; + + for iteration in 0..max_iterations { + iterations_used = iteration + 1; + log::info!("Agentic iteration {}/{}", iteration + 1, max_iterations); + + let response = ollama_client + .chat_with_tools(messages.clone(), tools.clone()) + .await?; + + messages.push(response.clone()); + + if let Some(ref tool_calls) = response.tool_calls { + if !tool_calls.is_empty() { + for tool_call in tool_calls { + log::info!( + "Agentic tool call [{}]: {} {:?}", + iteration, + tool_call.function.name, + tool_call.function.arguments + ); + let result = self + .execute_tool( + &tool_call.function.name, + &tool_call.function.arguments, + &ollama_client, + &image_base64, + &loop_cx, + ) + .await; + messages.push(ChatMessage::tool_result(result)); + } + continue; + } + } + + // No tool calls — this is the final answer + final_content = response.content; + break; + } + + // If loop exhausted without final answer, ask for one + if final_content.is_empty() { + log::info!("Agentic loop exhausted after {} iterations, requesting final answer", iterations_used); + messages.push(ChatMessage::user( + "Based on the context gathered, please write the final photo insight: a title and a detailed personal summary. Write in first person as Cameron.", + )); + let final_response = ollama_client + .chat_with_tools(messages, vec![]) + .await?; + final_content = final_response.content; + } + + loop_cx + .span() + .set_attribute(KeyValue::new("iterations_used", iterations_used as i64)); + loop_cx.span().set_status(Status::Ok); + + // 12. Generate title + let title = ollama_client + .generate_photo_title(&final_content, custom_system_prompt.as_deref()) + .await?; + + log::info!("Agentic generated title: {}", title); + log::info!( + "Agentic generated summary ({} chars): {}", + final_content.len(), + &final_content[..final_content.len().min(200)] + ); + + // 13. Store + let insight = InsertPhotoInsight { + file_path: file_path.to_string(), + title, + summary: final_content, + generated_at: Utc::now().timestamp(), + model_version: ollama_client.primary_model.clone(), + }; + + let mut dao = self.insight_dao.lock().expect("Unable to lock InsightDao"); + let result = dao + .store_insight(&insight_cx, insight) + .map_err(|e| anyhow::anyhow!("Failed to store agentic insight: {:?}", e)); + + match &result { + Ok(_) => { + log::info!("Successfully stored agentic insight for {}", file_path); + insight_cx.span().set_status(Status::Ok); + } + Err(e) => { + log::error!("Failed to store agentic insight: {:?}", e); + insight_cx.span().set_status(Status::error(e.to_string())); + } + } + + result?; + Ok(()) + } + /// Reverse geocode GPS coordinates to human-readable place names async fn reverse_geocode(&self, lat: f64, lon: f64) -> Option { let url = format!(