Apps/ImageApi
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

feat: add tool executors and generate_agentic_insight_for_photo() to InsightGenerator

Browse Source 
Add 6 tool executor methods (search_rag, get_sms_messages, get_calendar_events,
get_location_history, get_file_tags, describe_photo) and the agentic loop that
uses Ollama's chat_with_tools API to let the model decide which context to gather
before writing the final photo insight.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Cameron
2026-03-18 23:00:41 -04:00
parent 5e5a2a3167
commit 7615b9c99b
1 changed files with 752 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

754
src/ai/insight_generator.rs
View File

@@ -1,6 +1,6 @@
use anyhow::Result;
use base64::Engine as _;
use chrono::{DateTime, Utc};
use chrono::{DateTime, NaiveDate, Utc};
use image::ImageFormat;
use opentelemetry::KeyValue;
use opentelemetry::trace::{Span, Status, TraceContextExt, Tracer};
@@ -9,7 +9,7 @@ use std::fs::File;
use std::io::Cursor;
use std::sync::{Arc, Mutex};
use crate::ai::ollama::OllamaClient;
use crate::ai::ollama::{ChatMessage, OllamaClient, Tool};
use crate::ai::sms_client::SmsApiClient;
use crate::database::models::InsertPhotoInsight;
use crate::database::{
@@ -1314,6 +1314,756 @@ Return ONLY the summary, nothing else."#,
}
}
// ── Tool executors for agentic loop ────────────────────────────────
/// Dispatch a tool call to the appropriate executor
async fn execute_tool(
&self,
tool_name: &str,
arguments: &serde_json::Value,
ollama: &OllamaClient,
image_base64: &Option<String>,
cx: &opentelemetry::Context,
) -> String {
match tool_name {
"search_rag" => self.tool_search_rag(arguments, cx).await,
"get_sms_messages" => self.tool_get_sms_messages(arguments, cx).await,
"get_calendar_events" => self.tool_get_calendar_events(arguments, cx).await,
"get_location_history" => self.tool_get_location_history(arguments, cx).await,
"get_file_tags" => self.tool_get_file_tags(arguments, cx).await,
"describe_photo" => self.tool_describe_photo(ollama, image_base64).await,
unknown => format!("Unknown tool: {}", unknown),
}
}
/// Tool: search_rag — semantic search over daily summaries
async fn tool_search_rag(
&self,
args: &serde_json::Value,
_cx: &opentelemetry::Context,
) -> String {
let query = match args.get("query").and_then(|v| v.as_str()) {
Some(q) => q.to_string(),
None => return "Error: missing required parameter 'query'".to_string(),
};
let date_str = match args.get("date").and_then(|v| v.as_str()) {
Some(d) => d,
None => return "Error: missing required parameter 'date'".to_string(),
};
let date = match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") {
Ok(d) => d,
Err(e) => return format!("Error: failed to parse date '{}': {}", date_str, e),
};
let contact = args
.get("contact")
.and_then(|v| v.as_str())
.map(|s| s.to_string());
log::info!(
"tool_search_rag: query='{}', date={}, contact={:?}",
query,
date,
contact
);
match self
.find_relevant_messages_rag(date, None, contact.as_deref(), None, 5, Some(&query))
.await
{
Ok(results) if !results.is_empty() => results.join("\n\n"),
Ok(_) => "No relevant messages found.".to_string(),
Err(e) => format!("Error searching RAG: {}", e),
}
}
/// Tool: get_sms_messages — fetch SMS messages near a date for a contact
async fn tool_get_sms_messages(
&self,
args: &serde_json::Value,
_cx: &opentelemetry::Context,
) -> String {
let date_str = match args.get("date").and_then(|v| v.as_str()) {
Some(d) => d,
None => return "Error: missing required parameter 'date'".to_string(),
};
let contact = match args.get("contact").and_then(|v| v.as_str()) {
Some(c) => c.to_string(),
None => return "Error: missing required parameter 'contact'".to_string(),
};
let days_radius = args
.get("days_radius")
.and_then(|v| v.as_i64())
.unwrap_or(4);
let date = match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") {
Ok(d) => d,
Err(e) => return format!("Error: failed to parse date '{}': {}", date_str, e),
};
let timestamp = date
.and_hms_opt(12, 0, 0)
.unwrap()
.and_utc()
.timestamp();
log::info!(
"tool_get_sms_messages: date={}, contact='{}', days_radius={}",
date,
contact,
days_radius
);
// Use the SMS client's existing fetch mechanism
// Build start/end from days_radius
let start_ts = timestamp - (days_radius * 86400);
let end_ts = timestamp + (days_radius * 86400);
let center_dt = DateTime::from_timestamp(timestamp, 0);
let start_dt = DateTime::from_timestamp(start_ts, 0);
let end_dt = DateTime::from_timestamp(end_ts, 0);
if center_dt.is_none() || start_dt.is_none() || end_dt.is_none() {
return "Error: invalid timestamp range".to_string();
}
match self
.sms_client
.fetch_messages_for_contact(Some(&contact), timestamp)
.await
{
Ok(messages) if !messages.is_empty() => {
let formatted: Vec<String> = messages
.iter()
.take(30)
.map(|m| {
let sender = if m.is_sent { "Me" } else { &m.contact };
let ts = DateTime::from_timestamp(m.timestamp, 0)
.map(|dt| dt.format("%Y-%m-%d %H:%M").to_string())
.unwrap_or_else(|| "unknown".to_string());
format!("[{}] {}: {}", ts, sender, m.body)
})
.collect();
format!("Found {} messages:\n{}", messages.len(), formatted.join("\n"))
}
Ok(_) => "No messages found.".to_string(),
Err(e) => format!("Error fetching SMS messages: {}", e),
}
}
/// Tool: get_calendar_events — fetch calendar events near a date
async fn tool_get_calendar_events(
&self,
args: &serde_json::Value,
cx: &opentelemetry::Context,
) -> String {
let date_str = match args.get("date").and_then(|v| v.as_str()) {
Some(d) => d,
None => return "Error: missing required parameter 'date'".to_string(),
};
let days_radius = args
.get("days_radius")
.and_then(|v| v.as_i64())
.unwrap_or(7);
let date = match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") {
Ok(d) => d,
Err(e) => return format!("Error: failed to parse date '{}': {}", date_str, e),
};
let timestamp = date
.and_hms_opt(12, 0, 0)
.unwrap()
.and_utc()
.timestamp();
log::info!(
"tool_get_calendar_events: date={}, days_radius={}",
date,
days_radius
);
let events = {
let mut dao = self
.calendar_dao
.lock()
.expect("Unable to lock CalendarEventDao");
dao.find_relevant_events_hybrid(cx, timestamp, days_radius, None, 10)
.ok()
};
match events {
Some(evts) if !evts.is_empty() => {
let formatted: Vec<String> = evts
.iter()
.map(|e| {
let dt = DateTime::from_timestamp(e.start_time, 0)
.map(|dt| dt.format("%Y-%m-%d %H:%M").to_string())
.unwrap_or_else(|| "unknown".to_string());
let loc = e
.location
.as_ref()
.map(|l| format!(" at {}", l))
.unwrap_or_default();
let attendees = e
.attendees
.as_ref()
.and_then(|a| serde_json::from_str::<Vec<String>>(a).ok())
.map(|list| format!(" (with {})", list.join(", ")))
.unwrap_or_default();
format!("[{}] {}{}{}", dt, e.summary, loc, attendees)
})
.collect();
format!(
"Found {} calendar events:\n{}",
evts.len(),
formatted.join("\n")
)
}
Some(_) => "No calendar events found.".to_string(),
None => "No calendar events found.".to_string(),
}
}
/// Tool: get_location_history — fetch location records near a date
async fn tool_get_location_history(
&self,
args: &serde_json::Value,
cx: &opentelemetry::Context,
) -> String {
let date_str = match args.get("date").and_then(|v| v.as_str()) {
Some(d) => d,
None => return "Error: missing required parameter 'date'".to_string(),
};
let days_radius = args
.get("days_radius")
.and_then(|v| v.as_i64())
.unwrap_or(14);
let date = match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") {
Ok(d) => d,
Err(e) => return format!("Error: failed to parse date '{}': {}", date_str, e),
};
let timestamp = date
.and_hms_opt(12, 0, 0)
.unwrap()
.and_utc()
.timestamp();
log::info!(
"tool_get_location_history: date={}, days_radius={}",
date,
days_radius
);
let start_ts = timestamp - (days_radius * 86400);
let end_ts = timestamp + (days_radius * 86400);
let locations = {
let mut dao = self
.location_dao
.lock()
.expect("Unable to lock LocationHistoryDao");
dao.find_locations_in_range(cx, start_ts, end_ts).ok()
};
match locations {
Some(locs) if !locs.is_empty() => {
let formatted: Vec<String> = locs
.iter()
.take(20)
.map(|loc| {
let dt = DateTime::from_timestamp(loc.timestamp, 0)
.map(|dt| dt.format("%Y-%m-%d %H:%M").to_string())
.unwrap_or_else(|| "unknown".to_string());
let activity = loc
.activity
.as_ref()
.map(|a| format!(" ({})", a))
.unwrap_or_default();
let place = loc
.place_name
.as_ref()
.map(|p| format!(" at {}", p))
.unwrap_or_default();
format!(
"[{}] {:.4}, {:.4}{}{}",
dt, loc.latitude, loc.longitude, place, activity
)
})
.collect();
format!(
"Found {} location records:\n{}",
locs.len(),
formatted.join("\n")
)
}
Some(_) => "No location history found.".to_string(),
None => "No location history found.".to_string(),
}
}
/// Tool: get_file_tags — fetch tags for a file path
async fn tool_get_file_tags(
&self,
args: &serde_json::Value,
cx: &opentelemetry::Context,
) -> String {
let file_path = match args.get("file_path").and_then(|v| v.as_str()) {
Some(p) => p.to_string(),
None => return "Error: missing required parameter 'file_path'".to_string(),
};
log::info!("tool_get_file_tags: file_path='{}'", file_path);
let tags = {
let mut dao = self.tag_dao.lock().expect("Unable to lock TagDao");
dao.get_tags_for_path(cx, &file_path).ok()
};
match tags {
Some(t) if !t.is_empty() => {
let names: Vec<String> = t.into_iter().map(|tag| tag.name).collect();
names.join(", ")
}
Some(_) => "No tags found.".to_string(),
None => "No tags found.".to_string(),
}
}
/// Tool: describe_photo — generate a visual description of the photo
async fn tool_describe_photo(
&self,
ollama: &OllamaClient,
image_base64: &Option<String>,
) -> String {
log::info!("tool_describe_photo: generating visual description");
match image_base64 {
Some(img) => match ollama.generate_photo_description(img).await {
Ok(desc) => desc,
Err(e) => format!("Error describing photo: {}", e),
},
None => "No image available for description.".to_string(),
}
}
// ── Agentic insight generation ──────────────────────────────────────
/// Build the list of tool definitions for the agentic loop
fn build_tool_definitions(has_vision: bool) -> Vec<Tool> {
let mut tools = vec![
Tool::function(
"search_rag",
"Search conversation history using semantic search. Use this to find relevant past conversations about specific topics, people, or events.",
serde_json::json!({
"type": "object",
"required": ["query", "date"],
"properties": {
"query": {
"type": "string",
"description": "The search query to find relevant conversations"
},
"date": {
"type": "string",
"description": "The reference date in YYYY-MM-DD format"
},
"contact": {
"type": "string",
"description": "Optional contact name to filter results"
}
}
}),
),
Tool::function(
"get_sms_messages",
"Fetch SMS/text messages near a specific date for a contact. Returns the actual message conversation.",
serde_json::json!({
"type": "object",
"required": ["date", "contact"],
"properties": {
"date": {
"type": "string",
"description": "The center date in YYYY-MM-DD format"
},
"contact": {
"type": "string",
"description": "The contact name to fetch messages for"
},
"days_radius": {
"type": "integer",
"description": "Number of days before and after the date to search (default: 4)"
}
}
}),
),
Tool::function(
"get_calendar_events",
"Fetch calendar events near a specific date. Shows scheduled events, meetings, and activities.",
serde_json::json!({
"type": "object",
"required": ["date"],
"properties": {
"date": {
"type": "string",
"description": "The center date in YYYY-MM-DD format"
},
"days_radius": {
"type": "integer",
"description": "Number of days before and after the date to search (default: 7)"
}
}
}),
),
Tool::function(
"get_location_history",
"Fetch location history records near a specific date. Shows places visited and activities.",
serde_json::json!({
"type": "object",
"required": ["date"],
"properties": {
"date": {
"type": "string",
"description": "The center date in YYYY-MM-DD format"
},
"days_radius": {
"type": "integer",
"description": "Number of days before and after the date to search (default: 14)"
}
}
}),
),
Tool::function(
"get_file_tags",
"Get tags/labels that have been applied to a specific photo file.",
serde_json::json!({
"type": "object",
"required": ["file_path"],
"properties": {
"file_path": {
"type": "string",
"description": "The file path of the photo to get tags for"
}
}
}),
),
];
if has_vision {
tools.push(Tool::function(
"describe_photo",
"Generate a visual description of the photo. Describes people, location, and activity visible in the image.",
serde_json::json!({
"type": "object",
"properties": {}
}),
));
}
tools
}
/// Generate an AI insight for a photo using an agentic tool-calling loop.
/// The model decides which tools to call to gather context before writing the final insight.
pub async fn generate_agentic_insight_for_photo(
&self,
file_path: &str,
custom_model: Option<String>,
custom_system_prompt: Option<String>,
num_ctx: Option<i32>,
max_iterations: usize,
) -> Result<()> {
let tracer = global_tracer();
let current_cx = opentelemetry::Context::current();
let mut span = tracer.start_with_context("ai.insight.generate_agentic", &current_cx);
let file_path = normalize_path(file_path);
log::info!("Generating agentic insight for photo: {}", file_path);
span.set_attribute(KeyValue::new("file_path", file_path.clone()));
span.set_attribute(KeyValue::new("max_iterations", max_iterations as i64));
// 1. Create OllamaClient
let mut ollama_client = if let Some(ref model) = custom_model {
log::info!("Using custom model for agentic: {}", model);
span.set_attribute(KeyValue::new("custom_model", model.clone()));
OllamaClient::new(
self.ollama.primary_url.clone(),
self.ollama.fallback_url.clone(),
model.clone(),
Some(model.clone()),
)
} else {
span.set_attribute(KeyValue::new("model", self.ollama.primary_model.clone()));
self.ollama.clone()
};
if let Some(ctx) = num_ctx {
log::info!("Using custom context size: {}", ctx);
span.set_attribute(KeyValue::new("num_ctx", ctx as i64));
ollama_client.set_num_ctx(Some(ctx));
}
let insight_cx = current_cx.with_span(span);
// 2. Check tool calling capability
let capabilities = OllamaClient::check_model_capabilities(
&ollama_client.primary_url,
&ollama_client.primary_model,
)
.await
.map_err(|e| {
anyhow::anyhow!(
"Failed to check model capabilities for '{}': {}",
ollama_client.primary_model,
e
)
})?;
if !capabilities.has_tool_calling {
return Err(anyhow::anyhow!(
"tool calling not supported by model '{}'",
ollama_client.primary_model
));
}
let has_vision = capabilities.has_vision;
insight_cx
.span()
.set_attribute(KeyValue::new("model_has_vision", has_vision));
insight_cx
.span()
.set_attribute(KeyValue::new("model_has_tool_calling", true));
// 3. Fetch EXIF
let exif = {
let mut exif_dao = self.exif_dao.lock().expect("Unable to lock ExifDao");
exif_dao
.get_exif(&insight_cx, &file_path)
.map_err(|e| anyhow::anyhow!("Failed to get EXIF: {:?}", e))?
};
// 4. Extract timestamp and contact
let timestamp = if let Some(ts) = exif.as_ref().and_then(|e| e.date_taken) {
ts
} else {
log::warn!("No date_taken in EXIF for {}, trying filename", file_path);
extract_date_from_filename(&file_path)
.map(|dt| dt.timestamp())
.or_else(|| {
let full_path = std::path::Path::new(&self.base_path).join(&file_path);
File::open(&full_path)
.and_then(|f| f.metadata())
.and_then(|m| m.created().or(m.modified()))
.map(|t| DateTime::<Utc>::from(t).timestamp())
.inspect_err(|e| {
log::warn!(
"Failed to get file timestamp for agentic insight {}: {}",
file_path,
e
)
})
.ok()
})
.unwrap_or_else(|| Utc::now().timestamp())
};
let date_taken = DateTime::from_timestamp(timestamp, 0)
.map(|dt| dt.date_naive())
.unwrap_or_else(|| Utc::now().date_naive());
let contact = Self::extract_contact_from_path(&file_path);
log::info!("Agentic: date_taken={}, contact={:?}", date_taken, contact);
// 5. Fetch tags
let tag_names: Vec<String> = {
let mut dao = self.tag_dao.lock().expect("Unable to lock TagDao");
dao.get_tags_for_path(&insight_cx, &file_path)
.unwrap_or_else(|e| {
log::warn!("Failed to fetch tags for agentic {}: {}", file_path, e);
Vec::new()
})
.into_iter()
.map(|t| t.name)
.collect()
};
// 6. Load image if vision capable
let image_base64 = if has_vision {
match self.load_image_as_base64(&file_path) {
Ok(b64) => {
log::info!("Loaded image for vision-capable agentic model");
Some(b64)
}
Err(e) => {
log::warn!("Failed to load image for agentic vision: {}", e);
None
}
}
} else {
None
};
// 7. Build system message
let base_system = "You are a personal photo memory assistant. You have access to tools to gather context about when and where this photo was taken. Use them to build a rich, personal insight. Call tools as needed, then write a final summary and title.";
let system_content = if let Some(ref custom) = custom_system_prompt {
format!("{}\n\n{}", custom, base_system)
} else {
base_system.to_string()
};
// 8. Build user message
let gps_info = exif
.as_ref()
.and_then(|e| {
if let (Some(lat), Some(lon)) = (e.gps_latitude, e.gps_longitude) {
Some(format!("GPS: {:.4}, {:.4}", lat, lon))
} else {
None
}
})
.unwrap_or_else(|| "GPS: unknown".to_string());
let tags_info = if tag_names.is_empty() {
"Tags: none".to_string()
} else {
format!("Tags: {}", tag_names.join(", "))
};
let contact_info = contact
.as_ref()
.map(|c| format!("Contact/Person: {}", c))
.unwrap_or_else(|| "Contact/Person: unknown".to_string());
let user_content = format!(
"Please analyze this photo and gather context to write a personal journal-style insight.\n\n\
Photo file path: {}\n\
Date taken: {}\n\
{}\n\
{}\n\
{}\n\n\
Use the available tools to gather more context about this moment (messages, calendar events, location history, etc.), \
then write a detailed personal insight with a title and summary. Write in first person as Cameron.",
file_path,
date_taken.format("%B %d, %Y"),
contact_info,
gps_info,
tags_info,
);
// 9. Define tools
let tools = Self::build_tool_definitions(has_vision);
// 10. Build initial messages
let system_msg = ChatMessage::system(system_content);
let mut user_msg = ChatMessage::user(user_content);
if let Some(ref img) = image_base64 {
user_msg.images = Some(vec![img.clone()]);
}
let mut messages = vec![system_msg, user_msg];
// 11. Agentic loop
let loop_span = tracer.start_with_context("ai.agentic.loop", &insight_cx);
let loop_cx = insight_cx.with_span(loop_span);
let mut final_content = String::new();
let mut iterations_used = 0usize;
for iteration in 0..max_iterations {
iterations_used = iteration + 1;
log::info!("Agentic iteration {}/{}", iteration + 1, max_iterations);
let response = ollama_client
.chat_with_tools(messages.clone(), tools.clone())
.await?;
messages.push(response.clone());
if let Some(ref tool_calls) = response.tool_calls {
if !tool_calls.is_empty() {
for tool_call in tool_calls {
log::info!(
"Agentic tool call [{}]: {} {:?}",
iteration,
tool_call.function.name,
tool_call.function.arguments
);
let result = self
.execute_tool(
&tool_call.function.name,
&tool_call.function.arguments,
&ollama_client,
&image_base64,
&loop_cx,
)
.await;
messages.push(ChatMessage::tool_result(result));
}
continue;
}
}
// No tool calls — this is the final answer
final_content = response.content;
break;
}
// If loop exhausted without final answer, ask for one
if final_content.is_empty() {
log::info!("Agentic loop exhausted after {} iterations, requesting final answer", iterations_used);
messages.push(ChatMessage::user(
"Based on the context gathered, please write the final photo insight: a title and a detailed personal summary. Write in first person as Cameron.",
));
let final_response = ollama_client
.chat_with_tools(messages, vec![])
.await?;
final_content = final_response.content;
}
loop_cx
.span()
.set_attribute(KeyValue::new("iterations_used", iterations_used as i64));
loop_cx.span().set_status(Status::Ok);
// 12. Generate title
let title = ollama_client
.generate_photo_title(&final_content, custom_system_prompt.as_deref())
.await?;
log::info!("Agentic generated title: {}", title);
log::info!(
"Agentic generated summary ({} chars): {}",
final_content.len(),
&final_content[..final_content.len().min(200)]
);
// 13. Store
let insight = InsertPhotoInsight {
file_path: file_path.to_string(),
title,
summary: final_content,
generated_at: Utc::now().timestamp(),
model_version: ollama_client.primary_model.clone(),
};
let mut dao = self.insight_dao.lock().expect("Unable to lock InsightDao");
let result = dao
.store_insight(&insight_cx, insight)
.map_err(|e| anyhow::anyhow!("Failed to store agentic insight: {:?}", e));
match &result {
Ok(_) => {
log::info!("Successfully stored agentic insight for {}", file_path);
insight_cx.span().set_status(Status::Ok);
}
Err(e) => {
log::error!("Failed to store agentic insight: {:?}", e);
insight_cx.span().set_status(Status::error(e.to_string()));
}
}
result?;
Ok(())
}
/// Reverse geocode GPS coordinates to human-readable place names
async fn reverse_geocode(&self, lat: f64, lon: f64) -> Option<String> {
let url = format!(