feat: add tool executors and generate_agentic_insight_for_photo() to InsightGenerator
Add 6 tool executor methods (search_rag, get_sms_messages, get_calendar_events, get_location_history, get_file_tags, describe_photo) and the agentic loop that uses Ollama's chat_with_tools API to let the model decide which context to gather before writing the final photo insight. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Cameron
@@ -1,6 +1,6 @@
|
|||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use base64::Engine as _;
|
use base64::Engine as _;
|
||||||
use chrono::{DateTime, Utc};
|
use chrono::{DateTime, NaiveDate, Utc};
|
||||||
use image::ImageFormat;
|
use image::ImageFormat;
|
||||||
use opentelemetry::KeyValue;
|
use opentelemetry::KeyValue;
|
||||||
use opentelemetry::trace::{Span, Status, TraceContextExt, Tracer};
|
use opentelemetry::trace::{Span, Status, TraceContextExt, Tracer};
|
||||||
@@ -9,7 +9,7 @@ use std::fs::File;
|
|||||||
use std::io::Cursor;
|
use std::io::Cursor;
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
use crate::ai::ollama::OllamaClient;
|
use crate::ai::ollama::{ChatMessage, OllamaClient, Tool};
|
||||||
use crate::ai::sms_client::SmsApiClient;
|
use crate::ai::sms_client::SmsApiClient;
|
||||||
use crate::database::models::InsertPhotoInsight;
|
use crate::database::models::InsertPhotoInsight;
|
||||||
use crate::database::{
|
use crate::database::{
|
||||||
@@ -1314,6 +1314,756 @@ Return ONLY the summary, nothing else."#,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Tool executors for agentic loop ────────────────────────────────
|
||||||
|
|
||||||
|
/// Dispatch a tool call to the appropriate executor
|
||||||
|
async fn execute_tool(
|
||||||
|
&self,
|
||||||
|
tool_name: &str,
|
||||||
|
arguments: &serde_json::Value,
|
||||||
|
ollama: &OllamaClient,
|
||||||
|
image_base64: &Option<String>,
|
||||||
|
cx: &opentelemetry::Context,
|
||||||
|
) -> String {
|
||||||
|
match tool_name {
|
||||||
|
"search_rag" => self.tool_search_rag(arguments, cx).await,
|
||||||
|
"get_sms_messages" => self.tool_get_sms_messages(arguments, cx).await,
|
||||||
|
"get_calendar_events" => self.tool_get_calendar_events(arguments, cx).await,
|
||||||
|
"get_location_history" => self.tool_get_location_history(arguments, cx).await,
|
||||||
|
"get_file_tags" => self.tool_get_file_tags(arguments, cx).await,
|
||||||
|
"describe_photo" => self.tool_describe_photo(ollama, image_base64).await,
|
||||||
|
unknown => format!("Unknown tool: {}", unknown),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Tool: search_rag — semantic search over daily summaries
|
||||||
|
async fn tool_search_rag(
|
||||||
|
&self,
|
||||||
|
args: &serde_json::Value,
|
||||||
|
_cx: &opentelemetry::Context,
|
||||||
|
) -> String {
|
||||||
|
let query = match args.get("query").and_then(|v| v.as_str()) {
|
||||||
|
Some(q) => q.to_string(),
|
||||||
|
None => return "Error: missing required parameter 'query'".to_string(),
|
||||||
|
};
|
||||||
|
let date_str = match args.get("date").and_then(|v| v.as_str()) {
|
||||||
|
Some(d) => d,
|
||||||
|
None => return "Error: missing required parameter 'date'".to_string(),
|
||||||
|
};
|
||||||
|
let date = match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") {
|
||||||
|
Ok(d) => d,
|
||||||
|
Err(e) => return format!("Error: failed to parse date '{}': {}", date_str, e),
|
||||||
|
};
|
||||||
|
let contact = args
|
||||||
|
.get("contact")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string());
|
||||||
|
|
||||||
|
log::info!(
|
||||||
|
"tool_search_rag: query='{}', date={}, contact={:?}",
|
||||||
|
query,
|
||||||
|
date,
|
||||||
|
contact
|
||||||
|
);
|
||||||
|
|
||||||
|
match self
|
||||||
|
.find_relevant_messages_rag(date, None, contact.as_deref(), None, 5, Some(&query))
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(results) if !results.is_empty() => results.join("\n\n"),
|
||||||
|
Ok(_) => "No relevant messages found.".to_string(),
|
||||||
|
Err(e) => format!("Error searching RAG: {}", e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Tool: get_sms_messages — fetch SMS messages near a date for a contact
|
||||||
|
async fn tool_get_sms_messages(
|
||||||
|
&self,
|
||||||
|
args: &serde_json::Value,
|
||||||
|
_cx: &opentelemetry::Context,
|
||||||
|
) -> String {
|
||||||
|
let date_str = match args.get("date").and_then(|v| v.as_str()) {
|
||||||
|
Some(d) => d,
|
||||||
|
None => return "Error: missing required parameter 'date'".to_string(),
|
||||||
|
};
|
||||||
|
let contact = match args.get("contact").and_then(|v| v.as_str()) {
|
||||||
|
Some(c) => c.to_string(),
|
||||||
|
None => return "Error: missing required parameter 'contact'".to_string(),
|
||||||
|
};
|
||||||
|
let days_radius = args
|
||||||
|
.get("days_radius")
|
||||||
|
.and_then(|v| v.as_i64())
|
||||||
|
.unwrap_or(4);
|
||||||
|
|
||||||
|
let date = match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") {
|
||||||
|
Ok(d) => d,
|
||||||
|
Err(e) => return format!("Error: failed to parse date '{}': {}", date_str, e),
|
||||||
|
};
|
||||||
|
let timestamp = date
|
||||||
|
.and_hms_opt(12, 0, 0)
|
||||||
|
.unwrap()
|
||||||
|
.and_utc()
|
||||||
|
.timestamp();
|
||||||
|
|
||||||
|
log::info!(
|
||||||
|
"tool_get_sms_messages: date={}, contact='{}', days_radius={}",
|
||||||
|
date,
|
||||||
|
contact,
|
||||||
|
days_radius
|
||||||
|
);
|
||||||
|
|
||||||
|
// Use the SMS client's existing fetch mechanism
|
||||||
|
// Build start/end from days_radius
|
||||||
|
let start_ts = timestamp - (days_radius * 86400);
|
||||||
|
let end_ts = timestamp + (days_radius * 86400);
|
||||||
|
|
||||||
|
let center_dt = DateTime::from_timestamp(timestamp, 0);
|
||||||
|
let start_dt = DateTime::from_timestamp(start_ts, 0);
|
||||||
|
let end_dt = DateTime::from_timestamp(end_ts, 0);
|
||||||
|
|
||||||
|
if center_dt.is_none() || start_dt.is_none() || end_dt.is_none() {
|
||||||
|
return "Error: invalid timestamp range".to_string();
|
||||||
|
}
|
||||||
|
|
||||||
|
match self
|
||||||
|
.sms_client
|
||||||
|
.fetch_messages_for_contact(Some(&contact), timestamp)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(messages) if !messages.is_empty() => {
|
||||||
|
let formatted: Vec<String> = messages
|
||||||
|
.iter()
|
||||||
|
.take(30)
|
||||||
|
.map(|m| {
|
||||||
|
let sender = if m.is_sent { "Me" } else { &m.contact };
|
||||||
|
let ts = DateTime::from_timestamp(m.timestamp, 0)
|
||||||
|
.map(|dt| dt.format("%Y-%m-%d %H:%M").to_string())
|
||||||
|
.unwrap_or_else(|| "unknown".to_string());
|
||||||
|
format!("[{}] {}: {}", ts, sender, m.body)
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
format!("Found {} messages:\n{}", messages.len(), formatted.join("\n"))
|
||||||
|
}
|
||||||
|
Ok(_) => "No messages found.".to_string(),
|
||||||
|
Err(e) => format!("Error fetching SMS messages: {}", e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Tool: get_calendar_events — fetch calendar events near a date
|
||||||
|
async fn tool_get_calendar_events(
|
||||||
|
&self,
|
||||||
|
args: &serde_json::Value,
|
||||||
|
cx: &opentelemetry::Context,
|
||||||
|
) -> String {
|
||||||
|
let date_str = match args.get("date").and_then(|v| v.as_str()) {
|
||||||
|
Some(d) => d,
|
||||||
|
None => return "Error: missing required parameter 'date'".to_string(),
|
||||||
|
};
|
||||||
|
let days_radius = args
|
||||||
|
.get("days_radius")
|
||||||
|
.and_then(|v| v.as_i64())
|
||||||
|
.unwrap_or(7);
|
||||||
|
|
||||||
|
let date = match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") {
|
||||||
|
Ok(d) => d,
|
||||||
|
Err(e) => return format!("Error: failed to parse date '{}': {}", date_str, e),
|
||||||
|
};
|
||||||
|
let timestamp = date
|
||||||
|
.and_hms_opt(12, 0, 0)
|
||||||
|
.unwrap()
|
||||||
|
.and_utc()
|
||||||
|
.timestamp();
|
||||||
|
|
||||||
|
log::info!(
|
||||||
|
"tool_get_calendar_events: date={}, days_radius={}",
|
||||||
|
date,
|
||||||
|
days_radius
|
||||||
|
);
|
||||||
|
|
||||||
|
let events = {
|
||||||
|
let mut dao = self
|
||||||
|
.calendar_dao
|
||||||
|
.lock()
|
||||||
|
.expect("Unable to lock CalendarEventDao");
|
||||||
|
dao.find_relevant_events_hybrid(cx, timestamp, days_radius, None, 10)
|
||||||
|
.ok()
|
||||||
|
};
|
||||||
|
|
||||||
|
match events {
|
||||||
|
Some(evts) if !evts.is_empty() => {
|
||||||
|
let formatted: Vec<String> = evts
|
||||||
|
.iter()
|
||||||
|
.map(|e| {
|
||||||
|
let dt = DateTime::from_timestamp(e.start_time, 0)
|
||||||
|
.map(|dt| dt.format("%Y-%m-%d %H:%M").to_string())
|
||||||
|
.unwrap_or_else(|| "unknown".to_string());
|
||||||
|
let loc = e
|
||||||
|
.location
|
||||||
|
.as_ref()
|
||||||
|
.map(|l| format!(" at {}", l))
|
||||||
|
.unwrap_or_default();
|
||||||
|
let attendees = e
|
||||||
|
.attendees
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|a| serde_json::from_str::<Vec<String>>(a).ok())
|
||||||
|
.map(|list| format!(" (with {})", list.join(", ")))
|
||||||
|
.unwrap_or_default();
|
||||||
|
format!("[{}] {}{}{}", dt, e.summary, loc, attendees)
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
format!(
|
||||||
|
"Found {} calendar events:\n{}",
|
||||||
|
evts.len(),
|
||||||
|
formatted.join("\n")
|
||||||
|
)
|
||||||
|
}
|
||||||
|
Some(_) => "No calendar events found.".to_string(),
|
||||||
|
None => "No calendar events found.".to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Tool: get_location_history — fetch location records near a date
|
||||||
|
async fn tool_get_location_history(
|
||||||
|
&self,
|
||||||
|
args: &serde_json::Value,
|
||||||
|
cx: &opentelemetry::Context,
|
||||||
|
) -> String {
|
||||||
|
let date_str = match args.get("date").and_then(|v| v.as_str()) {
|
||||||
|
Some(d) => d,
|
||||||
|
None => return "Error: missing required parameter 'date'".to_string(),
|
||||||
|
};
|
||||||
|
let days_radius = args
|
||||||
|
.get("days_radius")
|
||||||
|
.and_then(|v| v.as_i64())
|
||||||
|
.unwrap_or(14);
|
||||||
|
|
||||||
|
let date = match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") {
|
||||||
|
Ok(d) => d,
|
||||||
|
Err(e) => return format!("Error: failed to parse date '{}': {}", date_str, e),
|
||||||
|
};
|
||||||
|
let timestamp = date
|
||||||
|
.and_hms_opt(12, 0, 0)
|
||||||
|
.unwrap()
|
||||||
|
.and_utc()
|
||||||
|
.timestamp();
|
||||||
|
|
||||||
|
log::info!(
|
||||||
|
"tool_get_location_history: date={}, days_radius={}",
|
||||||
|
date,
|
||||||
|
days_radius
|
||||||
|
);
|
||||||
|
|
||||||
|
let start_ts = timestamp - (days_radius * 86400);
|
||||||
|
let end_ts = timestamp + (days_radius * 86400);
|
||||||
|
|
||||||
|
let locations = {
|
||||||
|
let mut dao = self
|
||||||
|
.location_dao
|
||||||
|
.lock()
|
||||||
|
.expect("Unable to lock LocationHistoryDao");
|
||||||
|
dao.find_locations_in_range(cx, start_ts, end_ts).ok()
|
||||||
|
};
|
||||||
|
|
||||||
|
match locations {
|
||||||
|
Some(locs) if !locs.is_empty() => {
|
||||||
|
let formatted: Vec<String> = locs
|
||||||
|
.iter()
|
||||||
|
.take(20)
|
||||||
|
.map(|loc| {
|
||||||
|
let dt = DateTime::from_timestamp(loc.timestamp, 0)
|
||||||
|
.map(|dt| dt.format("%Y-%m-%d %H:%M").to_string())
|
||||||
|
.unwrap_or_else(|| "unknown".to_string());
|
||||||
|
let activity = loc
|
||||||
|
.activity
|
||||||
|
.as_ref()
|
||||||
|
.map(|a| format!(" ({})", a))
|
||||||
|
.unwrap_or_default();
|
||||||
|
let place = loc
|
||||||
|
.place_name
|
||||||
|
.as_ref()
|
||||||
|
.map(|p| format!(" at {}", p))
|
||||||
|
.unwrap_or_default();
|
||||||
|
format!(
|
||||||
|
"[{}] {:.4}, {:.4}{}{}",
|
||||||
|
dt, loc.latitude, loc.longitude, place, activity
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
format!(
|
||||||
|
"Found {} location records:\n{}",
|
||||||
|
locs.len(),
|
||||||
|
formatted.join("\n")
|
||||||
|
)
|
||||||
|
}
|
||||||
|
Some(_) => "No location history found.".to_string(),
|
||||||
|
None => "No location history found.".to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Tool: get_file_tags — fetch tags for a file path
|
||||||
|
async fn tool_get_file_tags(
|
||||||
|
&self,
|
||||||
|
args: &serde_json::Value,
|
||||||
|
cx: &opentelemetry::Context,
|
||||||
|
) -> String {
|
||||||
|
let file_path = match args.get("file_path").and_then(|v| v.as_str()) {
|
||||||
|
Some(p) => p.to_string(),
|
||||||
|
None => return "Error: missing required parameter 'file_path'".to_string(),
|
||||||
|
};
|
||||||
|
|
||||||
|
log::info!("tool_get_file_tags: file_path='{}'", file_path);
|
||||||
|
|
||||||
|
let tags = {
|
||||||
|
let mut dao = self.tag_dao.lock().expect("Unable to lock TagDao");
|
||||||
|
dao.get_tags_for_path(cx, &file_path).ok()
|
||||||
|
};
|
||||||
|
|
||||||
|
match tags {
|
||||||
|
Some(t) if !t.is_empty() => {
|
||||||
|
let names: Vec<String> = t.into_iter().map(|tag| tag.name).collect();
|
||||||
|
names.join(", ")
|
||||||
|
}
|
||||||
|
Some(_) => "No tags found.".to_string(),
|
||||||
|
None => "No tags found.".to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Tool: describe_photo — generate a visual description of the photo
|
||||||
|
async fn tool_describe_photo(
|
||||||
|
&self,
|
||||||
|
ollama: &OllamaClient,
|
||||||
|
image_base64: &Option<String>,
|
||||||
|
) -> String {
|
||||||
|
log::info!("tool_describe_photo: generating visual description");
|
||||||
|
|
||||||
|
match image_base64 {
|
||||||
|
Some(img) => match ollama.generate_photo_description(img).await {
|
||||||
|
Ok(desc) => desc,
|
||||||
|
Err(e) => format!("Error describing photo: {}", e),
|
||||||
|
},
|
||||||
|
None => "No image available for description.".to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Agentic insight generation ──────────────────────────────────────
|
||||||
|
|
||||||
|
/// Build the list of tool definitions for the agentic loop
|
||||||
|
fn build_tool_definitions(has_vision: bool) -> Vec<Tool> {
|
||||||
|
let mut tools = vec![
|
||||||
|
Tool::function(
|
||||||
|
"search_rag",
|
||||||
|
"Search conversation history using semantic search. Use this to find relevant past conversations about specific topics, people, or events.",
|
||||||
|
serde_json::json!({
|
||||||
|
"type": "object",
|
||||||
|
"required": ["query", "date"],
|
||||||
|
"properties": {
|
||||||
|
"query": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The search query to find relevant conversations"
|
||||||
|
},
|
||||||
|
"date": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The reference date in YYYY-MM-DD format"
|
||||||
|
},
|
||||||
|
"contact": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Optional contact name to filter results"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
Tool::function(
|
||||||
|
"get_sms_messages",
|
||||||
|
"Fetch SMS/text messages near a specific date for a contact. Returns the actual message conversation.",
|
||||||
|
serde_json::json!({
|
||||||
|
"type": "object",
|
||||||
|
"required": ["date", "contact"],
|
||||||
|
"properties": {
|
||||||
|
"date": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The center date in YYYY-MM-DD format"
|
||||||
|
},
|
||||||
|
"contact": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The contact name to fetch messages for"
|
||||||
|
},
|
||||||
|
"days_radius": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "Number of days before and after the date to search (default: 4)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
Tool::function(
|
||||||
|
"get_calendar_events",
|
||||||
|
"Fetch calendar events near a specific date. Shows scheduled events, meetings, and activities.",
|
||||||
|
serde_json::json!({
|
||||||
|
"type": "object",
|
||||||
|
"required": ["date"],
|
||||||
|
"properties": {
|
||||||
|
"date": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The center date in YYYY-MM-DD format"
|
||||||
|
},
|
||||||
|
"days_radius": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "Number of days before and after the date to search (default: 7)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
Tool::function(
|
||||||
|
"get_location_history",
|
||||||
|
"Fetch location history records near a specific date. Shows places visited and activities.",
|
||||||
|
serde_json::json!({
|
||||||
|
"type": "object",
|
||||||
|
"required": ["date"],
|
||||||
|
"properties": {
|
||||||
|
"date": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The center date in YYYY-MM-DD format"
|
||||||
|
},
|
||||||
|
"days_radius": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "Number of days before and after the date to search (default: 14)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
Tool::function(
|
||||||
|
"get_file_tags",
|
||||||
|
"Get tags/labels that have been applied to a specific photo file.",
|
||||||
|
serde_json::json!({
|
||||||
|
"type": "object",
|
||||||
|
"required": ["file_path"],
|
||||||
|
"properties": {
|
||||||
|
"file_path": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The file path of the photo to get tags for"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
];
|
||||||
|
|
||||||
|
if has_vision {
|
||||||
|
tools.push(Tool::function(
|
||||||
|
"describe_photo",
|
||||||
|
"Generate a visual description of the photo. Describes people, location, and activity visible in the image.",
|
||||||
|
serde_json::json!({
|
||||||
|
"type": "object",
|
||||||
|
"properties": {}
|
||||||
|
}),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
tools
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Generate an AI insight for a photo using an agentic tool-calling loop.
|
||||||
|
/// The model decides which tools to call to gather context before writing the final insight.
|
||||||
|
pub async fn generate_agentic_insight_for_photo(
|
||||||
|
&self,
|
||||||
|
file_path: &str,
|
||||||
|
custom_model: Option<String>,
|
||||||
|
custom_system_prompt: Option<String>,
|
||||||
|
num_ctx: Option<i32>,
|
||||||
|
max_iterations: usize,
|
||||||
|
) -> Result<()> {
|
||||||
|
let tracer = global_tracer();
|
||||||
|
let current_cx = opentelemetry::Context::current();
|
||||||
|
let mut span = tracer.start_with_context("ai.insight.generate_agentic", ¤t_cx);
|
||||||
|
|
||||||
|
let file_path = normalize_path(file_path);
|
||||||
|
log::info!("Generating agentic insight for photo: {}", file_path);
|
||||||
|
|
||||||
|
span.set_attribute(KeyValue::new("file_path", file_path.clone()));
|
||||||
|
span.set_attribute(KeyValue::new("max_iterations", max_iterations as i64));
|
||||||
|
|
||||||
|
// 1. Create OllamaClient
|
||||||
|
let mut ollama_client = if let Some(ref model) = custom_model {
|
||||||
|
log::info!("Using custom model for agentic: {}", model);
|
||||||
|
span.set_attribute(KeyValue::new("custom_model", model.clone()));
|
||||||
|
OllamaClient::new(
|
||||||
|
self.ollama.primary_url.clone(),
|
||||||
|
self.ollama.fallback_url.clone(),
|
||||||
|
model.clone(),
|
||||||
|
Some(model.clone()),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
span.set_attribute(KeyValue::new("model", self.ollama.primary_model.clone()));
|
||||||
|
self.ollama.clone()
|
||||||
|
};
|
||||||
|
|
||||||
|
if let Some(ctx) = num_ctx {
|
||||||
|
log::info!("Using custom context size: {}", ctx);
|
||||||
|
span.set_attribute(KeyValue::new("num_ctx", ctx as i64));
|
||||||
|
ollama_client.set_num_ctx(Some(ctx));
|
||||||
|
}
|
||||||
|
|
||||||
|
let insight_cx = current_cx.with_span(span);
|
||||||
|
|
||||||
|
// 2. Check tool calling capability
|
||||||
|
let capabilities = OllamaClient::check_model_capabilities(
|
||||||
|
&ollama_client.primary_url,
|
||||||
|
&ollama_client.primary_model,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
anyhow::anyhow!(
|
||||||
|
"Failed to check model capabilities for '{}': {}",
|
||||||
|
ollama_client.primary_model,
|
||||||
|
e
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
|
||||||
|
if !capabilities.has_tool_calling {
|
||||||
|
return Err(anyhow::anyhow!(
|
||||||
|
"tool calling not supported by model '{}'",
|
||||||
|
ollama_client.primary_model
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
let has_vision = capabilities.has_vision;
|
||||||
|
insight_cx
|
||||||
|
.span()
|
||||||
|
.set_attribute(KeyValue::new("model_has_vision", has_vision));
|
||||||
|
insight_cx
|
||||||
|
.span()
|
||||||
|
.set_attribute(KeyValue::new("model_has_tool_calling", true));
|
||||||
|
|
||||||
|
// 3. Fetch EXIF
|
||||||
|
let exif = {
|
||||||
|
let mut exif_dao = self.exif_dao.lock().expect("Unable to lock ExifDao");
|
||||||
|
exif_dao
|
||||||
|
.get_exif(&insight_cx, &file_path)
|
||||||
|
.map_err(|e| anyhow::anyhow!("Failed to get EXIF: {:?}", e))?
|
||||||
|
};
|
||||||
|
|
||||||
|
// 4. Extract timestamp and contact
|
||||||
|
let timestamp = if let Some(ts) = exif.as_ref().and_then(|e| e.date_taken) {
|
||||||
|
ts
|
||||||
|
} else {
|
||||||
|
log::warn!("No date_taken in EXIF for {}, trying filename", file_path);
|
||||||
|
extract_date_from_filename(&file_path)
|
||||||
|
.map(|dt| dt.timestamp())
|
||||||
|
.or_else(|| {
|
||||||
|
let full_path = std::path::Path::new(&self.base_path).join(&file_path);
|
||||||
|
File::open(&full_path)
|
||||||
|
.and_then(|f| f.metadata())
|
||||||
|
.and_then(|m| m.created().or(m.modified()))
|
||||||
|
.map(|t| DateTime::<Utc>::from(t).timestamp())
|
||||||
|
.inspect_err(|e| {
|
||||||
|
log::warn!(
|
||||||
|
"Failed to get file timestamp for agentic insight {}: {}",
|
||||||
|
file_path,
|
||||||
|
e
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.ok()
|
||||||
|
})
|
||||||
|
.unwrap_or_else(|| Utc::now().timestamp())
|
||||||
|
};
|
||||||
|
|
||||||
|
let date_taken = DateTime::from_timestamp(timestamp, 0)
|
||||||
|
.map(|dt| dt.date_naive())
|
||||||
|
.unwrap_or_else(|| Utc::now().date_naive());
|
||||||
|
|
||||||
|
let contact = Self::extract_contact_from_path(&file_path);
|
||||||
|
log::info!("Agentic: date_taken={}, contact={:?}", date_taken, contact);
|
||||||
|
|
||||||
|
// 5. Fetch tags
|
||||||
|
let tag_names: Vec<String> = {
|
||||||
|
let mut dao = self.tag_dao.lock().expect("Unable to lock TagDao");
|
||||||
|
dao.get_tags_for_path(&insight_cx, &file_path)
|
||||||
|
.unwrap_or_else(|e| {
|
||||||
|
log::warn!("Failed to fetch tags for agentic {}: {}", file_path, e);
|
||||||
|
Vec::new()
|
||||||
|
})
|
||||||
|
.into_iter()
|
||||||
|
.map(|t| t.name)
|
||||||
|
.collect()
|
||||||
|
};
|
||||||
|
|
||||||
|
// 6. Load image if vision capable
|
||||||
|
let image_base64 = if has_vision {
|
||||||
|
match self.load_image_as_base64(&file_path) {
|
||||||
|
Ok(b64) => {
|
||||||
|
log::info!("Loaded image for vision-capable agentic model");
|
||||||
|
Some(b64)
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
log::warn!("Failed to load image for agentic vision: {}", e);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
|
// 7. Build system message
|
||||||
|
let base_system = "You are a personal photo memory assistant. You have access to tools to gather context about when and where this photo was taken. Use them to build a rich, personal insight. Call tools as needed, then write a final summary and title.";
|
||||||
|
let system_content = if let Some(ref custom) = custom_system_prompt {
|
||||||
|
format!("{}\n\n{}", custom, base_system)
|
||||||
|
} else {
|
||||||
|
base_system.to_string()
|
||||||
|
};
|
||||||
|
|
||||||
|
// 8. Build user message
|
||||||
|
let gps_info = exif
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|e| {
|
||||||
|
if let (Some(lat), Some(lon)) = (e.gps_latitude, e.gps_longitude) {
|
||||||
|
Some(format!("GPS: {:.4}, {:.4}", lat, lon))
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.unwrap_or_else(|| "GPS: unknown".to_string());
|
||||||
|
|
||||||
|
let tags_info = if tag_names.is_empty() {
|
||||||
|
"Tags: none".to_string()
|
||||||
|
} else {
|
||||||
|
format!("Tags: {}", tag_names.join(", "))
|
||||||
|
};
|
||||||
|
|
||||||
|
let contact_info = contact
|
||||||
|
.as_ref()
|
||||||
|
.map(|c| format!("Contact/Person: {}", c))
|
||||||
|
.unwrap_or_else(|| "Contact/Person: unknown".to_string());
|
||||||
|
|
||||||
|
let user_content = format!(
|
||||||
|
"Please analyze this photo and gather context to write a personal journal-style insight.\n\n\
|
||||||
|
Photo file path: {}\n\
|
||||||
|
Date taken: {}\n\
|
||||||
|
{}\n\
|
||||||
|
{}\n\
|
||||||
|
{}\n\n\
|
||||||
|
Use the available tools to gather more context about this moment (messages, calendar events, location history, etc.), \
|
||||||
|
then write a detailed personal insight with a title and summary. Write in first person as Cameron.",
|
||||||
|
file_path,
|
||||||
|
date_taken.format("%B %d, %Y"),
|
||||||
|
contact_info,
|
||||||
|
gps_info,
|
||||||
|
tags_info,
|
||||||
|
);
|
||||||
|
|
||||||
|
// 9. Define tools
|
||||||
|
let tools = Self::build_tool_definitions(has_vision);
|
||||||
|
|
||||||
|
// 10. Build initial messages
|
||||||
|
let system_msg = ChatMessage::system(system_content);
|
||||||
|
let mut user_msg = ChatMessage::user(user_content);
|
||||||
|
if let Some(ref img) = image_base64 {
|
||||||
|
user_msg.images = Some(vec![img.clone()]);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut messages = vec![system_msg, user_msg];
|
||||||
|
|
||||||
|
// 11. Agentic loop
|
||||||
|
let loop_span = tracer.start_with_context("ai.agentic.loop", &insight_cx);
|
||||||
|
let loop_cx = insight_cx.with_span(loop_span);
|
||||||
|
|
||||||
|
let mut final_content = String::new();
|
||||||
|
let mut iterations_used = 0usize;
|
||||||
|
|
||||||
|
for iteration in 0..max_iterations {
|
||||||
|
iterations_used = iteration + 1;
|
||||||
|
log::info!("Agentic iteration {}/{}", iteration + 1, max_iterations);
|
||||||
|
|
||||||
|
let response = ollama_client
|
||||||
|
.chat_with_tools(messages.clone(), tools.clone())
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
messages.push(response.clone());
|
||||||
|
|
||||||
|
if let Some(ref tool_calls) = response.tool_calls {
|
||||||
|
if !tool_calls.is_empty() {
|
||||||
|
for tool_call in tool_calls {
|
||||||
|
log::info!(
|
||||||
|
"Agentic tool call [{}]: {} {:?}",
|
||||||
|
iteration,
|
||||||
|
tool_call.function.name,
|
||||||
|
tool_call.function.arguments
|
||||||
|
);
|
||||||
|
let result = self
|
||||||
|
.execute_tool(
|
||||||
|
&tool_call.function.name,
|
||||||
|
&tool_call.function.arguments,
|
||||||
|
&ollama_client,
|
||||||
|
&image_base64,
|
||||||
|
&loop_cx,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
messages.push(ChatMessage::tool_result(result));
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// No tool calls — this is the final answer
|
||||||
|
final_content = response.content;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// If loop exhausted without final answer, ask for one
|
||||||
|
if final_content.is_empty() {
|
||||||
|
log::info!("Agentic loop exhausted after {} iterations, requesting final answer", iterations_used);
|
||||||
|
messages.push(ChatMessage::user(
|
||||||
|
"Based on the context gathered, please write the final photo insight: a title and a detailed personal summary. Write in first person as Cameron.",
|
||||||
|
));
|
||||||
|
let final_response = ollama_client
|
||||||
|
.chat_with_tools(messages, vec![])
|
||||||
|
.await?;
|
||||||
|
final_content = final_response.content;
|
||||||
|
}
|
||||||
|
|
||||||
|
loop_cx
|
||||||
|
.span()
|
||||||
|
.set_attribute(KeyValue::new("iterations_used", iterations_used as i64));
|
||||||
|
loop_cx.span().set_status(Status::Ok);
|
||||||
|
|
||||||
|
// 12. Generate title
|
||||||
|
let title = ollama_client
|
||||||
|
.generate_photo_title(&final_content, custom_system_prompt.as_deref())
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
log::info!("Agentic generated title: {}", title);
|
||||||
|
log::info!(
|
||||||
|
"Agentic generated summary ({} chars): {}",
|
||||||
|
final_content.len(),
|
||||||
|
&final_content[..final_content.len().min(200)]
|
||||||
|
);
|
||||||
|
|
||||||
|
// 13. Store
|
||||||
|
let insight = InsertPhotoInsight {
|
||||||
|
file_path: file_path.to_string(),
|
||||||
|
title,
|
||||||
|
summary: final_content,
|
||||||
|
generated_at: Utc::now().timestamp(),
|
||||||
|
model_version: ollama_client.primary_model.clone(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut dao = self.insight_dao.lock().expect("Unable to lock InsightDao");
|
||||||
|
let result = dao
|
||||||
|
.store_insight(&insight_cx, insight)
|
||||||
|
.map_err(|e| anyhow::anyhow!("Failed to store agentic insight: {:?}", e));
|
||||||
|
|
||||||
|
match &result {
|
||||||
|
Ok(_) => {
|
||||||
|
log::info!("Successfully stored agentic insight for {}", file_path);
|
||||||
|
insight_cx.span().set_status(Status::Ok);
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
log::error!("Failed to store agentic insight: {:?}", e);
|
||||||
|
insight_cx.span().set_status(Status::error(e.to_string()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
result?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
/// Reverse geocode GPS coordinates to human-readable place names
|
/// Reverse geocode GPS coordinates to human-readable place names
|
||||||
async fn reverse_geocode(&self, lat: f64, lon: f64) -> Option<String> {
|
async fn reverse_geocode(&self, lat: f64, lon: f64) -> Option<String> {
|
||||||
let url = format!(
|
let url = format!(
|
||||||
|
|||||||
Reference in New Issue
Block a user