6f0c15d0c5
- Drop redundant `use anyhow::Context` inside has_any_faces (already
imported at the module level).
- Drop dead `.unwrap_or("?")` on bound faces — the vec is filtered to
is_some() so the fallback can never fire.
- Reorder the face_dao constructor param + initializer to match the
struct declaration (between tag_dao and knowledge_dao). Update both
state.rs call sites and populate_knowledge.rs to match.
- Hold face_dao lock once across the library-resolver loop instead of
reacquiring per iteration.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
4255 lines
163 KiB
Rust
4255 lines
163 KiB
Rust
use anyhow::Result;
|
||
use base64::Engine as _;
|
||
use chrono::{DateTime, Local, NaiveDate, Utc};
|
||
use image::ImageFormat;
|
||
use opentelemetry::KeyValue;
|
||
use opentelemetry::trace::{Span, Status, TraceContextExt, Tracer};
|
||
use serde::Deserialize;
|
||
use std::fs::File;
|
||
use std::io::Cursor;
|
||
use std::sync::{Arc, Mutex};
|
||
|
||
use crate::ai::apollo_client::{ApolloClient, ApolloPlace};
|
||
use crate::ai::llm_client::LlmClient;
|
||
use crate::ai::ollama::{ChatMessage, OllamaClient, Tool};
|
||
use crate::ai::openrouter::OpenRouterClient;
|
||
use crate::ai::sms_client::SmsApiClient;
|
||
use crate::ai::user_display_name;
|
||
use crate::database::models::InsertPhotoInsight;
|
||
use crate::database::{
|
||
CalendarEventDao, DailySummaryDao, ExifDao, InsightDao, KnowledgeDao, LocationHistoryDao,
|
||
SearchHistoryDao,
|
||
};
|
||
use crate::libraries::Library;
|
||
use crate::memories::extract_date_from_filename;
|
||
use crate::otel::global_tracer;
|
||
use crate::tags::TagDao;
|
||
use crate::utils::{earliest_fs_time, normalize_path};
|
||
|
||
/// Combine an optional personal Apollo Place with an optional Nominatim
|
||
/// reverse-geocoded city, falling back to bare coordinates when neither
|
||
/// resolves. Free function so we can test it cheaply without spinning up
|
||
/// the whole InsightGenerator.
|
||
fn compose_location_string(
|
||
apollo: Option<ApolloPlace>,
|
||
nominatim: Option<String>,
|
||
lat: f64,
|
||
lon: f64,
|
||
) -> String {
|
||
match (apollo, nominatim) {
|
||
(Some(p), Some(n)) if !p.description.is_empty() => {
|
||
format!("{} ({}) — near {}", p.name, p.description, n)
|
||
}
|
||
(Some(p), Some(n)) => format!("{} — near {}", p.name, n),
|
||
(Some(p), None) if !p.description.is_empty() => format!("{} ({})", p.name, p.description),
|
||
(Some(p), None) => p.name,
|
||
(None, Some(n)) => n,
|
||
(None, None) => format!("{lat:.4}, {lon:.4}"),
|
||
}
|
||
}
|
||
|
||
#[derive(Deserialize)]
|
||
struct NominatimResponse {
|
||
display_name: Option<String>,
|
||
address: Option<NominatimAddress>,
|
||
}
|
||
|
||
#[derive(Deserialize)]
|
||
struct NominatimAddress {
|
||
city: Option<String>,
|
||
town: Option<String>,
|
||
village: Option<String>,
|
||
state: Option<String>,
|
||
}
|
||
|
||
#[derive(Clone)]
|
||
pub struct InsightGenerator {
|
||
ollama: OllamaClient,
|
||
/// Optional OpenRouter client, used when `backend=hybrid` is requested.
|
||
/// `None` when `OPENROUTER_API_KEY` is not configured.
|
||
openrouter: Option<Arc<OpenRouterClient>>,
|
||
sms_client: SmsApiClient,
|
||
/// Optional integration with Apollo's user-defined Places. When the
|
||
/// integration is disabled (`APOLLO_API_BASE_URL` unset), every
|
||
/// query returns empty and the legacy Nominatim path is used as-is.
|
||
apollo_client: ApolloClient,
|
||
insight_dao: Arc<Mutex<Box<dyn InsightDao>>>,
|
||
exif_dao: Arc<Mutex<Box<dyn ExifDao>>>,
|
||
daily_summary_dao: Arc<Mutex<Box<dyn DailySummaryDao>>>,
|
||
|
||
// Google Takeout data sources
|
||
calendar_dao: Arc<Mutex<Box<dyn CalendarEventDao>>>,
|
||
location_dao: Arc<Mutex<Box<dyn LocationHistoryDao>>>,
|
||
search_dao: Arc<Mutex<Box<dyn SearchHistoryDao>>>,
|
||
tag_dao: Arc<Mutex<Box<dyn TagDao>>>,
|
||
|
||
// Face detections (used by the get_faces_in_photo agentic tool)
|
||
face_dao: Arc<Mutex<Box<dyn crate::faces::FaceDao>>>,
|
||
|
||
// Knowledge memory
|
||
knowledge_dao: Arc<Mutex<Box<dyn KnowledgeDao>>>,
|
||
|
||
libraries: Vec<Library>,
|
||
}
|
||
|
||
/// Per-call gating flags for `build_tool_definitions`. Tools whose backing
|
||
/// data is empty (or whose env-var guard is unset) are dropped from the
|
||
/// catalog so the LLM doesn't reach for a tool that always returns "No
|
||
/// results found." — that wastes iteration budget.
|
||
#[derive(Debug, Clone, Copy, Default)]
|
||
pub struct ToolGateOpts {
|
||
pub has_vision: bool,
|
||
pub apollo_enabled: bool,
|
||
pub daily_summaries_present: bool,
|
||
pub calendar_present: bool,
|
||
pub location_history_present: bool,
|
||
pub faces_present: bool,
|
||
}
|
||
|
||
impl InsightGenerator {
|
||
pub fn new(
|
||
ollama: OllamaClient,
|
||
openrouter: Option<Arc<OpenRouterClient>>,
|
||
sms_client: SmsApiClient,
|
||
apollo_client: ApolloClient,
|
||
insight_dao: Arc<Mutex<Box<dyn InsightDao>>>,
|
||
exif_dao: Arc<Mutex<Box<dyn ExifDao>>>,
|
||
daily_summary_dao: Arc<Mutex<Box<dyn DailySummaryDao>>>,
|
||
calendar_dao: Arc<Mutex<Box<dyn CalendarEventDao>>>,
|
||
location_dao: Arc<Mutex<Box<dyn LocationHistoryDao>>>,
|
||
search_dao: Arc<Mutex<Box<dyn SearchHistoryDao>>>,
|
||
tag_dao: Arc<Mutex<Box<dyn TagDao>>>,
|
||
face_dao: Arc<Mutex<Box<dyn crate::faces::FaceDao>>>,
|
||
knowledge_dao: Arc<Mutex<Box<dyn KnowledgeDao>>>,
|
||
libraries: Vec<Library>,
|
||
) -> Self {
|
||
Self {
|
||
ollama,
|
||
openrouter,
|
||
sms_client,
|
||
apollo_client,
|
||
insight_dao,
|
||
exif_dao,
|
||
daily_summary_dao,
|
||
calendar_dao,
|
||
location_dao,
|
||
search_dao,
|
||
tag_dao,
|
||
face_dao,
|
||
knowledge_dao,
|
||
libraries,
|
||
}
|
||
}
|
||
|
||
/// Whether the optional Apollo Places integration is wired up. Drives
|
||
/// tool-definition gating (no point offering `get_personal_place_at`
|
||
/// when Apollo is unreachable) — exposed publicly so `insight_chat`
|
||
/// can call `build_tool_definitions` with the same flag.
|
||
pub fn apollo_enabled(&self) -> bool {
|
||
self.apollo_client.is_enabled()
|
||
}
|
||
|
||
/// Compute the per-call tool gate options by probing each backing
|
||
/// table for presence. `daily_summaries_present` uses a `LIMIT 1`
|
||
/// existence probe; `calendar_present` and `location_history_present`
|
||
/// use the existing `get_event_count` / `get_location_count`
|
||
/// methods (small enough that a full `COUNT(*)` is fine). Meant to
|
||
/// be called once per chat turn / generation. `has_vision` is
|
||
/// supplied by the caller because it depends on the model selected
|
||
/// for this turn, not on persistent state.
|
||
pub fn current_gate_opts(&self, has_vision: bool) -> ToolGateOpts {
|
||
let cx = opentelemetry::Context::new();
|
||
let calendar_present = {
|
||
let mut dao = self
|
||
.calendar_dao
|
||
.lock()
|
||
.expect("Unable to lock CalendarEventDao");
|
||
dao.get_event_count(&cx).map(|n| n > 0).unwrap_or(false)
|
||
};
|
||
let location_history_present = {
|
||
let mut dao = self
|
||
.location_dao
|
||
.lock()
|
||
.expect("Unable to lock LocationHistoryDao");
|
||
dao.get_location_count(&cx).map(|n| n > 0).unwrap_or(false)
|
||
};
|
||
let daily_summaries_present = {
|
||
let mut dao = self
|
||
.daily_summary_dao
|
||
.lock()
|
||
.expect("Unable to lock DailySummaryDao");
|
||
dao.has_any_summaries(&cx).unwrap_or(false)
|
||
};
|
||
let faces_present = {
|
||
let mut dao = self
|
||
.face_dao
|
||
.lock()
|
||
.expect("Unable to lock FaceDao");
|
||
dao.has_any_faces(&cx).unwrap_or(false)
|
||
};
|
||
ToolGateOpts {
|
||
has_vision,
|
||
apollo_enabled: self.apollo_enabled(),
|
||
daily_summaries_present,
|
||
calendar_present,
|
||
location_history_present,
|
||
faces_present,
|
||
}
|
||
}
|
||
|
||
/// Resolve `rel_path` against the configured libraries, returning the
|
||
/// first root under which the file exists. Insights may be generated
|
||
/// for any library — the generator itself doesn't know which — so we
|
||
/// probe each root rather than trust a single `base_path`.
|
||
pub(crate) fn resolve_full_path(&self, rel_path: &str) -> Option<std::path::PathBuf> {
|
||
use std::path::Path;
|
||
for lib in &self.libraries {
|
||
let candidate = Path::new(&lib.root_path).join(rel_path);
|
||
if candidate.exists() {
|
||
return Some(candidate);
|
||
}
|
||
}
|
||
None
|
||
}
|
||
|
||
/// Extract contact name from file path
|
||
/// e.g., "Sarah/img.jpeg" -> Some("Sarah")
|
||
/// e.g., "img.jpeg" -> None
|
||
fn extract_contact_from_path(file_path: &str) -> Option<String> {
|
||
use std::path::Path;
|
||
|
||
let path = Path::new(file_path);
|
||
let components: Vec<_> = path.components().collect();
|
||
|
||
// If path has at least 2 components (directory + file), extract first directory
|
||
if components.len() >= 2
|
||
&& let Some(component) = components.first()
|
||
&& let Some(os_str) = component.as_os_str().to_str()
|
||
{
|
||
return Some(os_str.to_string());
|
||
}
|
||
|
||
None
|
||
}
|
||
|
||
/// Load image file, resize it, and encode as base64 for vision models
|
||
/// Resizes to max 1024px on longest edge to reduce context usage
|
||
pub(crate) fn load_image_as_base64(&self, file_path: &str) -> Result<String> {
|
||
use image::imageops::FilterType;
|
||
|
||
let full_path = self.resolve_full_path(file_path).ok_or_else(|| {
|
||
anyhow::anyhow!(
|
||
"File '{}' not found under any configured library",
|
||
file_path
|
||
)
|
||
})?;
|
||
|
||
log::debug!("Loading image for vision model: {:?}", full_path);
|
||
|
||
// Open and decode the image
|
||
let img = image::open(&full_path)
|
||
.map_err(|e| anyhow::anyhow!("Failed to open image file: {}", e))?;
|
||
|
||
let (original_width, original_height) = (img.width(), img.height());
|
||
|
||
// Resize to max 1024px on longest edge
|
||
let resized = img.resize(1024, 1024, FilterType::Lanczos3);
|
||
|
||
log::debug!(
|
||
"Resized image from {}x{} to {}x{}",
|
||
original_width,
|
||
original_height,
|
||
resized.width(),
|
||
resized.height()
|
||
);
|
||
|
||
// Encode as JPEG at 85% quality
|
||
let mut buffer = Vec::new();
|
||
let mut cursor = Cursor::new(&mut buffer);
|
||
resized
|
||
.write_to(&mut cursor, ImageFormat::Jpeg)
|
||
.map_err(|e| anyhow::anyhow!("Failed to encode image as JPEG: {}", e))?;
|
||
|
||
let base64_string = base64::engine::general_purpose::STANDARD.encode(&buffer);
|
||
|
||
log::debug!(
|
||
"Encoded image as base64 ({} bytes -> {} chars)",
|
||
buffer.len(),
|
||
base64_string.len()
|
||
);
|
||
|
||
Ok(base64_string)
|
||
}
|
||
|
||
/// Find relevant messages using RAG, excluding recent messages (>30 days ago)
|
||
/// This prevents RAG from returning messages already in the immediate time window
|
||
async fn find_relevant_messages_rag_historical(
|
||
&self,
|
||
parent_cx: &opentelemetry::Context,
|
||
date: chrono::NaiveDate,
|
||
location: Option<&str>,
|
||
contact: Option<&str>,
|
||
topics: Option<&[String]>,
|
||
limit: usize,
|
||
extra_context: Option<&str>,
|
||
) -> Result<Vec<String>> {
|
||
let tracer = global_tracer();
|
||
let span = tracer.start_with_context("ai.rag.filter_historical", parent_cx);
|
||
let filter_cx = parent_cx.with_span(span);
|
||
|
||
filter_cx
|
||
.span()
|
||
.set_attribute(KeyValue::new("date", date.to_string()));
|
||
filter_cx
|
||
.span()
|
||
.set_attribute(KeyValue::new("limit", limit as i64));
|
||
filter_cx
|
||
.span()
|
||
.set_attribute(KeyValue::new("exclusion_window_days", 30));
|
||
if let Some(t) = topics {
|
||
filter_cx
|
||
.span()
|
||
.set_attribute(KeyValue::new("topics", t.join(", ")));
|
||
}
|
||
|
||
let query_results = self
|
||
.find_relevant_messages_rag(date, location, contact, topics, limit * 2, extra_context)
|
||
.await?;
|
||
|
||
filter_cx.span().set_attribute(KeyValue::new(
|
||
"rag_results_count",
|
||
query_results.len() as i64,
|
||
));
|
||
|
||
// Filter out messages from within 30 days of the photo date
|
||
let photo_timestamp = date
|
||
.and_hms_opt(12, 0, 0)
|
||
.ok_or_else(|| anyhow::anyhow!("Invalid date"))?
|
||
.and_utc()
|
||
.timestamp();
|
||
let exclusion_window = 30 * 86400; // 30 days in seconds
|
||
|
||
let historical_only: Vec<String> = query_results
|
||
.into_iter()
|
||
.filter(|msg| {
|
||
// Extract date from formatted daily summary "[2024-08-15] Contact ..."
|
||
if let Some(bracket_end) = msg.find(']')
|
||
&& let Some(date_str) = msg.get(1..bracket_end)
|
||
{
|
||
// Parse just the date (daily summaries don't have time)
|
||
if let Ok(msg_date) = chrono::NaiveDate::parse_from_str(date_str, "%Y-%m-%d") {
|
||
let msg_timestamp = msg_date
|
||
.and_hms_opt(12, 0, 0)
|
||
.unwrap()
|
||
.and_utc()
|
||
.timestamp();
|
||
let time_diff = (photo_timestamp - msg_timestamp).abs();
|
||
return time_diff > exclusion_window;
|
||
}
|
||
}
|
||
false
|
||
})
|
||
.take(limit)
|
||
.collect();
|
||
|
||
log::info!(
|
||
"Found {} historical messages (>30 days from photo date)",
|
||
historical_only.len()
|
||
);
|
||
|
||
filter_cx.span().set_attribute(KeyValue::new(
|
||
"historical_results_count",
|
||
historical_only.len() as i64,
|
||
));
|
||
filter_cx.span().set_status(Status::Ok);
|
||
|
||
Ok(historical_only)
|
||
}
|
||
|
||
/// Find relevant daily summaries using RAG (semantic search)
|
||
/// Returns formatted daily summary strings for LLM context
|
||
async fn find_relevant_messages_rag(
|
||
&self,
|
||
date: chrono::NaiveDate,
|
||
location: Option<&str>,
|
||
contact: Option<&str>,
|
||
topics: Option<&[String]>,
|
||
limit: usize,
|
||
extra_context: Option<&str>,
|
||
) -> Result<Vec<String>> {
|
||
let tracer = global_tracer();
|
||
let current_cx = opentelemetry::Context::current();
|
||
let mut span = tracer.start_with_context("ai.rag.search_daily_summaries", ¤t_cx);
|
||
span.set_attribute(KeyValue::new("date", date.to_string()));
|
||
span.set_attribute(KeyValue::new("limit", limit as i64));
|
||
if let Some(loc) = location {
|
||
span.set_attribute(KeyValue::new("location", loc.to_string()));
|
||
}
|
||
if let Some(c) = contact {
|
||
span.set_attribute(KeyValue::new("contact", c.to_string()));
|
||
}
|
||
|
||
// Build query string - prioritize topics if available (semantically meaningful)
|
||
let base_query = if let Some(topics) = topics {
|
||
if !topics.is_empty() {
|
||
// Use topics for semantic search - these are actual content keywords
|
||
let topic_str = topics.join(", ");
|
||
if let Some(c) = contact {
|
||
format!("Conversations about {} with {}", topic_str, c)
|
||
} else {
|
||
format!("Conversations about {}", topic_str)
|
||
}
|
||
} else {
|
||
// Fallback to metadata-based query
|
||
Self::build_metadata_query(date, location, contact)
|
||
}
|
||
} else {
|
||
// Fallback to metadata-based query
|
||
Self::build_metadata_query(date, location, contact)
|
||
};
|
||
|
||
let query = if let Some(extra) = extra_context {
|
||
format!("{}. {}", base_query, extra)
|
||
} else {
|
||
base_query
|
||
};
|
||
|
||
span.set_attribute(KeyValue::new("query", query.clone()));
|
||
|
||
// Create context with this span for child operations
|
||
let search_cx = current_cx.with_span(span);
|
||
|
||
log::info!("========================================");
|
||
log::info!("RAG QUERY: {}", query);
|
||
log::info!("========================================");
|
||
|
||
// Generate embedding for the query
|
||
let query_embedding = self.ollama.generate_embedding(&query).await?;
|
||
|
||
// Search for similar daily summaries with time-based weighting
|
||
// This prioritizes summaries temporally close to the query date
|
||
let mut summary_dao = self
|
||
.daily_summary_dao
|
||
.lock()
|
||
.expect("Unable to lock DailySummaryDao");
|
||
|
||
let date_str = date.format("%Y-%m-%d").to_string();
|
||
let similar_summaries = summary_dao
|
||
.find_similar_summaries_with_time_weight(&search_cx, &query_embedding, &date_str, limit)
|
||
.map_err(|e| anyhow::anyhow!("Failed to find similar summaries: {:?}", e))?;
|
||
|
||
log::info!(
|
||
"Found {} relevant daily summaries via RAG",
|
||
similar_summaries.len()
|
||
);
|
||
|
||
search_cx.span().set_attribute(KeyValue::new(
|
||
"results_count",
|
||
similar_summaries.len() as i64,
|
||
));
|
||
|
||
// Format daily summaries for LLM context
|
||
let formatted = similar_summaries
|
||
.into_iter()
|
||
.map(|s| {
|
||
format!(
|
||
"[{}] {} ({} messages):\n{}",
|
||
s.date, s.contact, s.message_count, s.summary
|
||
)
|
||
})
|
||
.collect();
|
||
|
||
search_cx.span().set_status(Status::Ok);
|
||
|
||
Ok(formatted)
|
||
}
|
||
|
||
/// Build a metadata-based query (fallback when no topics available)
|
||
fn build_metadata_query(
|
||
date: chrono::NaiveDate,
|
||
location: Option<&str>,
|
||
contact: Option<&str>,
|
||
) -> String {
|
||
let mut query_parts = Vec::new();
|
||
|
||
// Add temporal context
|
||
query_parts.push(format!("On {}", date.format("%B %d, %Y")));
|
||
|
||
// Add location if available
|
||
if let Some(loc) = location {
|
||
query_parts.push(format!("at {}", loc));
|
||
}
|
||
|
||
// Add contact context if available
|
||
if let Some(c) = contact {
|
||
query_parts.push(format!("conversation with {}", c));
|
||
}
|
||
|
||
// Add day of week for temporal context
|
||
let weekday = date.format("%A");
|
||
query_parts.push(format!("it was a {}", weekday));
|
||
|
||
query_parts.join(", ")
|
||
}
|
||
|
||
/// Haversine distance calculation for GPS proximity (in kilometers)
|
||
fn haversine_distance(lat1: f64, lon1: f64, lat2: f64, lon2: f64) -> f64 {
|
||
const R: f64 = 6371.0; // Earth radius in km
|
||
let d_lat = (lat2 - lat1).to_radians();
|
||
let d_lon = (lon2 - lon1).to_radians();
|
||
let a = (d_lat / 2.0).sin().powi(2)
|
||
+ lat1.to_radians().cos() * lat2.to_radians().cos() * (d_lon / 2.0).sin().powi(2);
|
||
R * 2.0 * a.sqrt().atan2((1.0 - a).sqrt())
|
||
}
|
||
|
||
/// Gather calendar context for photo timestamp
|
||
/// Uses hybrid time + semantic search (±7 days, ranked by relevance)
|
||
async fn gather_calendar_context(
|
||
&self,
|
||
parent_cx: &opentelemetry::Context,
|
||
timestamp: i64,
|
||
location: Option<&str>,
|
||
) -> Result<Option<String>> {
|
||
let tracer = global_tracer();
|
||
let span = tracer.start_with_context("ai.context.calendar", parent_cx);
|
||
let calendar_cx = parent_cx.with_span(span);
|
||
|
||
let query_embedding = if let Some(loc) = location {
|
||
match self.ollama.generate_embedding(loc).await {
|
||
Ok(emb) => Some(emb),
|
||
Err(e) => {
|
||
log::warn!("Failed to generate embedding for location '{}': {}", loc, e);
|
||
None
|
||
}
|
||
}
|
||
} else {
|
||
None
|
||
};
|
||
|
||
let events = {
|
||
let mut dao = self
|
||
.calendar_dao
|
||
.lock()
|
||
.expect("Unable to lock CalendarEventDao");
|
||
dao.find_relevant_events_hybrid(
|
||
&calendar_cx,
|
||
timestamp,
|
||
7, // ±7 days window
|
||
query_embedding.as_deref(),
|
||
5, // Top 5 events
|
||
)
|
||
.ok()
|
||
};
|
||
|
||
calendar_cx.span().set_status(Status::Ok);
|
||
|
||
if let Some(events) = events {
|
||
if events.is_empty() {
|
||
return Ok(None);
|
||
}
|
||
|
||
let formatted = events
|
||
.iter()
|
||
.map(|e| {
|
||
let date = DateTime::from_timestamp(e.start_time, 0)
|
||
.map(|dt| {
|
||
dt.with_timezone(&Local)
|
||
.format("%Y-%m-%d %H:%M")
|
||
.to_string()
|
||
})
|
||
.unwrap_or_else(|| "unknown".to_string());
|
||
|
||
let attendees = e
|
||
.attendees
|
||
.as_ref()
|
||
.and_then(|a| serde_json::from_str::<Vec<String>>(a).ok())
|
||
.map(|list| format!(" (with {})", list.join(", ")))
|
||
.unwrap_or_default();
|
||
|
||
format!("[{}] {}{}", date, e.summary, attendees)
|
||
})
|
||
.collect::<Vec<_>>()
|
||
.join("\n");
|
||
|
||
Ok(Some(format!("Calendar events:\n{}", formatted)))
|
||
} else {
|
||
Ok(None)
|
||
}
|
||
}
|
||
|
||
/// Gather location context for photo timestamp
|
||
/// Finds nearest location record (±30 minutes)
|
||
async fn gather_location_context(
|
||
&self,
|
||
parent_cx: &opentelemetry::Context,
|
||
timestamp: i64,
|
||
exif_gps: Option<(f64, f64)>,
|
||
) -> Result<Option<String>> {
|
||
let tracer = global_tracer();
|
||
let span = tracer.start_with_context("ai.context.location", parent_cx);
|
||
let location_cx = parent_cx.with_span(span);
|
||
|
||
let nearest = {
|
||
let mut dao = self
|
||
.location_dao
|
||
.lock()
|
||
.expect("Unable to lock LocationHistoryDao");
|
||
dao.find_nearest_location(
|
||
&location_cx,
|
||
timestamp,
|
||
10800, // ±3 hours (more realistic for photo timing)
|
||
)
|
||
.ok()
|
||
.flatten()
|
||
};
|
||
|
||
location_cx.span().set_status(Status::Ok);
|
||
|
||
if let Some(loc) = nearest {
|
||
// Check if this adds NEW information compared to EXIF
|
||
if let Some((exif_lat, exif_lon)) = exif_gps {
|
||
let distance =
|
||
Self::haversine_distance(exif_lat, exif_lon, loc.latitude, loc.longitude);
|
||
|
||
// Skip only if very close AND no useful activity/place info
|
||
// Allow activity context even if coordinates match
|
||
if distance < 0.5 && loc.activity.is_none() && loc.place_name.is_none() {
|
||
log::debug!(
|
||
"Location history matches EXIF GPS ({}m) with no extra context, skipping",
|
||
(distance * 1000.0) as i32
|
||
);
|
||
return Ok(None);
|
||
} else if distance < 0.5 {
|
||
log::debug!(
|
||
"Location history close to EXIF ({}m) but has activity/place info",
|
||
(distance * 1000.0) as i32
|
||
);
|
||
}
|
||
}
|
||
|
||
let activity = loc
|
||
.activity
|
||
.as_ref()
|
||
.map(|a| format!(" ({})", a))
|
||
.unwrap_or_default();
|
||
|
||
let place = loc
|
||
.place_name
|
||
.as_ref()
|
||
.map(|p| format!(" at {}", p))
|
||
.unwrap_or_default();
|
||
|
||
Ok(Some(format!(
|
||
"Location history: You were{}{}{}",
|
||
activity,
|
||
place,
|
||
if activity.is_empty() && place.is_empty() {
|
||
format!(" near {:.4}, {:.4}", loc.latitude, loc.longitude)
|
||
} else {
|
||
String::new()
|
||
}
|
||
)))
|
||
} else {
|
||
Ok(None)
|
||
}
|
||
}
|
||
|
||
/// Gather search context for photo date
|
||
/// Uses semantic search on queries (±30 days, top 5 relevant)
|
||
async fn gather_search_context(
|
||
&self,
|
||
parent_cx: &opentelemetry::Context,
|
||
timestamp: i64,
|
||
location: Option<&str>,
|
||
contact: Option<&str>,
|
||
enrichment: Option<&str>,
|
||
) -> Result<Option<String>> {
|
||
let tracer = global_tracer();
|
||
let span = tracer.start_with_context("ai.context.search", parent_cx);
|
||
let search_cx = parent_cx.with_span(span);
|
||
|
||
// Use enrichment (topics + photo description + tags) if available;
|
||
// fall back to generic temporal query.
|
||
let query_text = if let Some(enriched) = enrichment {
|
||
enriched.to_string()
|
||
} else {
|
||
// Fallback: generic temporal query
|
||
format!(
|
||
"searches about {} {} {}",
|
||
DateTime::from_timestamp(timestamp, 0)
|
||
.map(|dt| dt.format("%B %Y").to_string())
|
||
.unwrap_or_default(),
|
||
location.unwrap_or(""),
|
||
contact
|
||
.map(|c| format!("involving {}", c))
|
||
.unwrap_or_default()
|
||
)
|
||
};
|
||
|
||
let query_embedding = match self.ollama.generate_embedding(&query_text).await {
|
||
Ok(emb) => emb,
|
||
Err(e) => {
|
||
log::warn!("Failed to generate search embedding: {}", e);
|
||
search_cx.span().set_status(Status::Error {
|
||
description: e.to_string().into(),
|
||
});
|
||
return Ok(None);
|
||
}
|
||
};
|
||
|
||
let searches = {
|
||
let mut dao = self
|
||
.search_dao
|
||
.lock()
|
||
.expect("Unable to lock SearchHistoryDao");
|
||
dao.find_relevant_searches_hybrid(
|
||
&search_cx,
|
||
timestamp,
|
||
30, // ±30 days (wider window than calendar)
|
||
Some(&query_embedding),
|
||
5, // Top 5 searches
|
||
)
|
||
.ok()
|
||
};
|
||
|
||
search_cx.span().set_status(Status::Ok);
|
||
|
||
if let Some(searches) = searches {
|
||
if searches.is_empty() {
|
||
log::warn!(
|
||
"No relevant searches found for photo timestamp {}",
|
||
timestamp
|
||
);
|
||
return Ok(None);
|
||
}
|
||
|
||
let formatted = searches
|
||
.iter()
|
||
.map(|s| {
|
||
let date = DateTime::from_timestamp(s.timestamp, 0)
|
||
.map(|dt| dt.format("%Y-%m-%d").to_string())
|
||
.unwrap_or_else(|| "unknown".to_string());
|
||
format!("[{}] \"{}\"", date, s.query)
|
||
})
|
||
.collect::<Vec<_>>()
|
||
.join("\n");
|
||
|
||
Ok(Some(format!("Search history:\n{}", formatted)))
|
||
} else {
|
||
Ok(None)
|
||
}
|
||
}
|
||
|
||
/// Combine all context sources with equal weight
|
||
fn combine_contexts(
|
||
sms: Option<String>,
|
||
calendar: Option<String>,
|
||
location: Option<String>,
|
||
search: Option<String>,
|
||
tags: Option<String>,
|
||
) -> String {
|
||
let mut parts = Vec::new();
|
||
|
||
if let Some(s) = sms {
|
||
parts.push(format!("## Messages\n{}", s));
|
||
}
|
||
if let Some(c) = calendar {
|
||
parts.push(format!("## Calendar\n{}", c));
|
||
}
|
||
if let Some(l) = location {
|
||
parts.push(format!("## Location\n{}", l));
|
||
}
|
||
if let Some(s) = search {
|
||
parts.push(format!("## Searches\n{}", s));
|
||
}
|
||
if let Some(t) = tags {
|
||
parts.push(format!("## Tags\n{}", t));
|
||
}
|
||
|
||
if parts.is_empty() {
|
||
"No additional context available".to_string()
|
||
} else {
|
||
parts.join("\n\n")
|
||
}
|
||
}
|
||
|
||
/// Generate AI insight for a single photo with custom configuration
|
||
pub async fn generate_insight_for_photo_with_config(
|
||
&self,
|
||
file_path: &str,
|
||
custom_model: Option<String>,
|
||
custom_system_prompt: Option<String>,
|
||
num_ctx: Option<i32>,
|
||
temperature: Option<f32>,
|
||
top_p: Option<f32>,
|
||
top_k: Option<i32>,
|
||
min_p: Option<f32>,
|
||
) -> Result<()> {
|
||
let tracer = global_tracer();
|
||
let current_cx = opentelemetry::Context::current();
|
||
let mut span = tracer.start_with_context("ai.insight.generate", ¤t_cx);
|
||
|
||
// Normalize path to ensure consistent forward slashes in database
|
||
let file_path = normalize_path(file_path);
|
||
log::info!("Generating insight for photo: {}", file_path);
|
||
|
||
span.set_attribute(KeyValue::new("file_path", file_path.clone()));
|
||
|
||
// Create custom Ollama client if model is specified
|
||
let mut ollama_client = if let Some(model) = custom_model {
|
||
log::info!("Using custom model: {}", model);
|
||
span.set_attribute(KeyValue::new("custom_model", model.clone()));
|
||
OllamaClient::new(
|
||
self.ollama.primary_url.clone(),
|
||
self.ollama.fallback_url.clone(),
|
||
model.clone(),
|
||
Some(model), // Use the same custom model for fallback server
|
||
)
|
||
} else {
|
||
span.set_attribute(KeyValue::new("model", self.ollama.primary_model.clone()));
|
||
self.ollama.clone()
|
||
};
|
||
|
||
// Set context size if specified
|
||
if let Some(ctx) = num_ctx {
|
||
log::info!("Using custom context size: {}", ctx);
|
||
span.set_attribute(KeyValue::new("num_ctx", ctx as i64));
|
||
ollama_client.set_num_ctx(Some(ctx));
|
||
}
|
||
|
||
// Apply sampling parameters if any were provided
|
||
if temperature.is_some() || top_p.is_some() || top_k.is_some() || min_p.is_some() {
|
||
log::info!(
|
||
"Using sampling params — temperature: {:?}, top_p: {:?}, top_k: {:?}, min_p: {:?}",
|
||
temperature,
|
||
top_p,
|
||
top_k,
|
||
min_p
|
||
);
|
||
if let Some(t) = temperature {
|
||
span.set_attribute(KeyValue::new("temperature", t as f64));
|
||
}
|
||
if let Some(p) = top_p {
|
||
span.set_attribute(KeyValue::new("top_p", p as f64));
|
||
}
|
||
if let Some(k) = top_k {
|
||
span.set_attribute(KeyValue::new("top_k", k as i64));
|
||
}
|
||
if let Some(m) = min_p {
|
||
span.set_attribute(KeyValue::new("min_p", m as f64));
|
||
}
|
||
ollama_client.set_sampling_params(temperature, top_p, top_k, min_p);
|
||
}
|
||
|
||
// Create context with this span for child operations
|
||
let insight_cx = current_cx.with_span(span);
|
||
|
||
// 1. Get EXIF data for the photo
|
||
let exif = {
|
||
let mut exif_dao = self.exif_dao.lock().expect("Unable to lock ExifDao");
|
||
exif_dao
|
||
.get_exif(&insight_cx, &file_path)
|
||
.map_err(|e| anyhow::anyhow!("Failed to get EXIF: {:?}", e))?
|
||
};
|
||
|
||
// Get full timestamp for proximity-based message filtering
|
||
let timestamp = if let Some(ts) = exif.as_ref().and_then(|e| e.date_taken) {
|
||
ts
|
||
} else {
|
||
log::warn!("No date_taken in EXIF for {}, trying filename", file_path);
|
||
|
||
extract_date_from_filename(&file_path)
|
||
.map(|dt| dt.timestamp())
|
||
.or_else(|| {
|
||
let full_path = self.resolve_full_path(&file_path)?;
|
||
File::open(&full_path)
|
||
.and_then(|f| f.metadata())
|
||
.inspect_err(|e| {
|
||
log::warn!(
|
||
"Failed to get file timestamp for insight {}: {}",
|
||
file_path,
|
||
e
|
||
)
|
||
})
|
||
.ok()
|
||
.and_then(|m| earliest_fs_time(&m))
|
||
.map(|t| DateTime::<Utc>::from(t).timestamp())
|
||
})
|
||
.unwrap_or_else(|| Utc::now().timestamp())
|
||
};
|
||
|
||
let date_taken = DateTime::from_timestamp(timestamp, 0)
|
||
.map(|dt| dt.date_naive())
|
||
.unwrap_or_else(|| Utc::now().date_naive());
|
||
|
||
// 3. Extract contact name from file path
|
||
let contact = Self::extract_contact_from_path(&file_path);
|
||
log::info!("Extracted contact from path: {:?}", contact);
|
||
|
||
insight_cx
|
||
.span()
|
||
.set_attribute(KeyValue::new("date_taken", date_taken.to_string()));
|
||
if let Some(ref c) = contact {
|
||
insight_cx
|
||
.span()
|
||
.set_attribute(KeyValue::new("contact", c.clone()));
|
||
}
|
||
|
||
// Fetch file tags (used to enrich RAG and final context)
|
||
let tag_names: Vec<String> = {
|
||
let mut dao = self.tag_dao.lock().expect("Unable to lock TagDao");
|
||
dao.get_tags_for_path(&insight_cx, &file_path)
|
||
.unwrap_or_else(|e| {
|
||
log::warn!("Failed to fetch tags for insight {}: {}", file_path, e);
|
||
Vec::new()
|
||
})
|
||
.into_iter()
|
||
.map(|t| t.name)
|
||
.collect()
|
||
};
|
||
log::info!(
|
||
"Fetched {} tags for photo: {:?}",
|
||
tag_names.len(),
|
||
tag_names
|
||
);
|
||
|
||
// 4. Get location name from GPS coordinates (needed for RAG query).
|
||
// Personal Apollo Place wins over Nominatim when both apply —
|
||
// "Home (My house in Cambridge) — near Cambridge, MA" is more
|
||
// grounding for the LLM than the city name alone.
|
||
let location = match exif {
|
||
Some(ref exif) => {
|
||
if let (Some(lat), Some(lon)) = (exif.gps_latitude, exif.gps_longitude) {
|
||
let lat = lat as f64;
|
||
let lon = lon as f64;
|
||
let nominatim = self.reverse_geocode(lat, lon).await;
|
||
let apollo_primary = self
|
||
.apollo_client
|
||
.places_containing(lat, lon)
|
||
.await
|
||
.into_iter()
|
||
.next();
|
||
let combined = compose_location_string(apollo_primary, nominatim, lat, lon);
|
||
insight_cx
|
||
.span()
|
||
.set_attribute(KeyValue::new("location", combined.clone()));
|
||
Some(combined)
|
||
} else {
|
||
None
|
||
}
|
||
}
|
||
None => None,
|
||
};
|
||
|
||
// Check if the model has vision capabilities
|
||
let model_to_check = ollama_client.primary_model.clone();
|
||
let has_vision = match OllamaClient::check_model_capabilities(
|
||
&ollama_client.primary_url,
|
||
&model_to_check,
|
||
)
|
||
.await
|
||
{
|
||
Ok(capabilities) => {
|
||
log::info!(
|
||
"Model '{}' vision capability: {}",
|
||
model_to_check,
|
||
capabilities.has_vision
|
||
);
|
||
capabilities.has_vision
|
||
}
|
||
Err(e) => {
|
||
log::warn!(
|
||
"Failed to check vision capabilities for model '{}', assuming no vision support: {}",
|
||
model_to_check,
|
||
e
|
||
);
|
||
false
|
||
}
|
||
};
|
||
|
||
insight_cx
|
||
.span()
|
||
.set_attribute(KeyValue::new("model_has_vision", has_vision));
|
||
|
||
// Load image and encode as base64 only if model supports vision
|
||
let image_base64 = if has_vision {
|
||
match self.load_image_as_base64(&file_path) {
|
||
Ok(b64) => {
|
||
log::info!(
|
||
"Successfully loaded image for vision-capable model '{}'",
|
||
model_to_check
|
||
);
|
||
Some(b64)
|
||
}
|
||
Err(e) => {
|
||
log::warn!("Failed to load image for vision model: {}", e);
|
||
None
|
||
}
|
||
}
|
||
} else {
|
||
log::info!(
|
||
"Model '{}' does not support vision, skipping image processing",
|
||
model_to_check
|
||
);
|
||
None
|
||
};
|
||
|
||
// Generate brief photo description for RAG enrichment (vision models only)
|
||
let photo_description: Option<String> = if let Some(ref img_b64) = image_base64 {
|
||
match ollama_client.generate_photo_description(img_b64).await {
|
||
Ok(desc) => {
|
||
log::info!("Photo description for RAG enrichment: {}", desc);
|
||
Some(desc)
|
||
}
|
||
Err(e) => {
|
||
log::warn!(
|
||
"Failed to generate photo description for RAG enrichment: {}",
|
||
e
|
||
);
|
||
None
|
||
}
|
||
}
|
||
} else {
|
||
None
|
||
};
|
||
|
||
// Build enriched context string for RAG: photo description + tags
|
||
// (SMS topics are passed separately to RAG functions)
|
||
let enriched_query: Option<String> = {
|
||
let mut parts: Vec<String> = Vec::new();
|
||
if let Some(ref desc) = photo_description {
|
||
parts.push(desc.clone());
|
||
}
|
||
if !tag_names.is_empty() {
|
||
parts.push(format!("tags: {}", tag_names.join(", ")));
|
||
}
|
||
if parts.is_empty() {
|
||
None
|
||
} else {
|
||
Some(parts.join(". "))
|
||
}
|
||
};
|
||
|
||
let mut search_enrichment: Option<String> = enriched_query.clone();
|
||
|
||
// 5. Intelligent retrieval: Hybrid approach for better context
|
||
let mut sms_summary = None;
|
||
let mut used_rag = false;
|
||
|
||
// TEMPORARY: Set to true to disable RAG and use only time-based retrieval for testing
|
||
let disable_rag_for_testing = false;
|
||
|
||
if disable_rag_for_testing {
|
||
log::warn!("RAG DISABLED FOR TESTING - Using only time-based retrieval (±4 days)");
|
||
// Skip directly to fallback
|
||
} else {
|
||
// ALWAYS use Strategy B: Expanded immediate context + historical RAG
|
||
// This is more reliable than pure semantic search which can match irrelevant messages
|
||
log::info!("Using expanded immediate context + historical RAG approach");
|
||
|
||
// Step 1: Get FULL immediate temporal context (±4 days, ALL messages)
|
||
let immediate_messages = self
|
||
.sms_client
|
||
.fetch_messages_for_contact(contact.as_deref(), timestamp, 4)
|
||
.await
|
||
.unwrap_or_else(|e| {
|
||
log::error!("Failed to fetch immediate messages: {}", e);
|
||
Vec::new()
|
||
});
|
||
|
||
log::info!(
|
||
"Fetched {} messages from ±4 days window (using ALL for immediate context)",
|
||
immediate_messages.len()
|
||
);
|
||
|
||
if !immediate_messages.is_empty() {
|
||
// Step 2: Extract topics from immediate messages to enrich RAG query
|
||
let topics = self
|
||
.extract_topics_from_messages(&immediate_messages, &ollama_client)
|
||
.await;
|
||
|
||
log::info!("Extracted topics for query enrichment: {:?}", topics);
|
||
|
||
// Build full search enrichment: SMS topics + photo description + tag names
|
||
search_enrichment = {
|
||
let mut parts: Vec<String> = Vec::new();
|
||
if !topics.is_empty() {
|
||
parts.push(topics.join(", "));
|
||
}
|
||
if let Some(ref desc) = photo_description {
|
||
parts.push(desc.clone());
|
||
}
|
||
if !tag_names.is_empty() {
|
||
parts.push(format!("tags: {}", tag_names.join(", ")));
|
||
}
|
||
if parts.is_empty() {
|
||
None
|
||
} else {
|
||
Some(parts.join(". "))
|
||
}
|
||
};
|
||
|
||
// Step 3: Try historical RAG (>30 days ago) using extracted topics
|
||
let topics_slice = if topics.is_empty() {
|
||
None
|
||
} else {
|
||
Some(topics.as_slice())
|
||
};
|
||
match self
|
||
.find_relevant_messages_rag_historical(
|
||
&insight_cx,
|
||
date_taken,
|
||
None,
|
||
contact.as_deref(),
|
||
topics_slice,
|
||
10, // Top 10 historical matches
|
||
enriched_query.as_deref(),
|
||
)
|
||
.await
|
||
{
|
||
Ok(historical_messages) if !historical_messages.is_empty() => {
|
||
log::info!(
|
||
"Two-context approach: {} immediate (full conversation) + {} historical (similar past moments)",
|
||
immediate_messages.len(),
|
||
historical_messages.len()
|
||
);
|
||
used_rag = true;
|
||
|
||
// Step 4: Summarize contexts separately, then combine
|
||
let immediate_summary = self
|
||
.summarize_context_from_messages(
|
||
&immediate_messages,
|
||
&ollama_client,
|
||
custom_system_prompt.as_deref(),
|
||
)
|
||
.await
|
||
.unwrap_or_else(|| String::from("No immediate context"));
|
||
|
||
let historical_summary = self
|
||
.summarize_messages(
|
||
&historical_messages,
|
||
&ollama_client,
|
||
custom_system_prompt.as_deref(),
|
||
)
|
||
.await
|
||
.unwrap_or_else(|| String::from("No historical context"));
|
||
|
||
// Combine summaries
|
||
sms_summary = Some(format!(
|
||
"Immediate context (±4 days): {}\n\nSimilar moments from the past: {}",
|
||
immediate_summary, historical_summary
|
||
));
|
||
}
|
||
Ok(_) => {
|
||
// RAG found no historical matches, just use immediate context
|
||
log::info!("No historical RAG matches, using immediate context only");
|
||
sms_summary = self
|
||
.summarize_context_from_messages(
|
||
&immediate_messages,
|
||
&ollama_client,
|
||
custom_system_prompt.as_deref(),
|
||
)
|
||
.await;
|
||
}
|
||
Err(e) => {
|
||
log::warn!("Historical RAG failed, using immediate context only: {}", e);
|
||
sms_summary = self
|
||
.summarize_context_from_messages(
|
||
&immediate_messages,
|
||
&ollama_client,
|
||
custom_system_prompt.as_deref(),
|
||
)
|
||
.await;
|
||
}
|
||
}
|
||
} else {
|
||
log::info!("No immediate messages found, trying basic RAG as fallback");
|
||
// Fallback to basic RAG even without strong query
|
||
match self
|
||
.find_relevant_messages_rag(
|
||
date_taken,
|
||
None,
|
||
contact.as_deref(),
|
||
None,
|
||
20,
|
||
enriched_query.as_deref(),
|
||
)
|
||
.await
|
||
{
|
||
Ok(rag_messages) if !rag_messages.is_empty() => {
|
||
used_rag = true;
|
||
sms_summary = self
|
||
.summarize_messages(
|
||
&rag_messages,
|
||
&ollama_client,
|
||
custom_system_prompt.as_deref(),
|
||
)
|
||
.await;
|
||
}
|
||
_ => {}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 6. Fallback to traditional time-based message retrieval if RAG didn't work
|
||
if !used_rag {
|
||
log::info!("Using traditional time-based message retrieval (±4 days)");
|
||
let sms_messages = self
|
||
.sms_client
|
||
.fetch_messages_for_contact(contact.as_deref(), timestamp, 4)
|
||
.await
|
||
.unwrap_or_else(|e| {
|
||
log::error!("Failed to fetch SMS messages: {}", e);
|
||
Vec::new()
|
||
});
|
||
|
||
log::info!(
|
||
"Fetched {} SMS messages closest to {}",
|
||
sms_messages.len(),
|
||
DateTime::from_timestamp(timestamp, 0)
|
||
.map(|dt| dt.format("%Y-%m-%d %H:%M:%S").to_string())
|
||
.unwrap_or_else(|| "unknown time".to_string())
|
||
);
|
||
|
||
// Summarize time-based messages
|
||
if !sms_messages.is_empty() {
|
||
match self
|
||
.sms_client
|
||
.summarize_context(&sms_messages, &ollama_client)
|
||
.await
|
||
{
|
||
Ok(summary) => {
|
||
sms_summary = Some(summary);
|
||
}
|
||
Err(e) => {
|
||
log::warn!("Failed to summarize SMS context: {}", e);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
let retrieval_method = if used_rag { "RAG" } else { "time-based" };
|
||
insight_cx
|
||
.span()
|
||
.set_attribute(KeyValue::new("retrieval_method", retrieval_method));
|
||
insight_cx
|
||
.span()
|
||
.set_attribute(KeyValue::new("has_sms_context", sms_summary.is_some()));
|
||
|
||
log::info!(
|
||
"Photo context: date={}, location={:?}, retrieval_method={}",
|
||
date_taken,
|
||
location,
|
||
retrieval_method
|
||
);
|
||
|
||
// 6. Gather Google Takeout context from all sources
|
||
let calendar_context = self
|
||
.gather_calendar_context(&insight_cx, timestamp, location.as_deref())
|
||
.await
|
||
.ok()
|
||
.flatten();
|
||
|
||
let exif_gps = exif.as_ref().and_then(|e| {
|
||
if let (Some(lat), Some(lon)) = (e.gps_latitude, e.gps_longitude) {
|
||
Some((lat as f64, lon as f64))
|
||
} else {
|
||
None
|
||
}
|
||
});
|
||
|
||
let location_context = self
|
||
.gather_location_context(&insight_cx, timestamp, exif_gps)
|
||
.await
|
||
.ok()
|
||
.flatten();
|
||
|
||
let search_context = self
|
||
.gather_search_context(
|
||
&insight_cx,
|
||
timestamp,
|
||
location.as_deref(),
|
||
contact.as_deref(),
|
||
search_enrichment.as_deref(),
|
||
)
|
||
.await
|
||
.ok()
|
||
.flatten();
|
||
|
||
// 7. Combine all context sources with equal weight
|
||
let tags_context = if tag_names.is_empty() {
|
||
None
|
||
} else {
|
||
Some(tag_names.join(", "))
|
||
};
|
||
|
||
let combined_context = Self::combine_contexts(
|
||
sms_summary,
|
||
calendar_context,
|
||
location_context,
|
||
search_context,
|
||
tags_context,
|
||
);
|
||
|
||
log::info!(
|
||
"Combined context from all sources ({} chars)",
|
||
combined_context.len()
|
||
);
|
||
|
||
// 10. Generate summary first, then derive title from the summary
|
||
let summary = ollama_client
|
||
.generate_photo_summary(
|
||
date_taken,
|
||
location.as_deref(),
|
||
contact.as_deref(),
|
||
Some(&combined_context),
|
||
custom_system_prompt.as_deref(),
|
||
image_base64.clone(),
|
||
)
|
||
.await?;
|
||
|
||
let title = ollama_client
|
||
.generate_photo_title(&summary, custom_system_prompt.as_deref())
|
||
.await?;
|
||
|
||
log::info!("Generated title: {}", title);
|
||
log::info!("Generated summary: {}", summary);
|
||
|
||
insight_cx
|
||
.span()
|
||
.set_attribute(KeyValue::new("title_length", title.len() as i64));
|
||
insight_cx
|
||
.span()
|
||
.set_attribute(KeyValue::new("summary_length", summary.len() as i64));
|
||
|
||
// 11. Store in database. content_hash is None here — store_insight
|
||
// looks it up from image_exif before persisting; reconciliation
|
||
// backfills if the hash isn't known yet.
|
||
let insight = InsertPhotoInsight {
|
||
library_id: crate::libraries::PRIMARY_LIBRARY_ID,
|
||
file_path: file_path.to_string(),
|
||
title,
|
||
summary,
|
||
generated_at: Utc::now().timestamp(),
|
||
model_version: ollama_client.primary_model.clone(),
|
||
is_current: true,
|
||
training_messages: None,
|
||
backend: "local".to_string(),
|
||
fewshot_source_ids: None,
|
||
content_hash: None,
|
||
};
|
||
|
||
let mut dao = self.insight_dao.lock().expect("Unable to lock InsightDao");
|
||
let result = dao
|
||
.store_insight(&insight_cx, insight)
|
||
.map_err(|e| anyhow::anyhow!("Failed to store insight: {:?}", e));
|
||
|
||
match &result {
|
||
Ok(_) => {
|
||
log::info!("Successfully stored insight for {}", file_path);
|
||
insight_cx.span().set_status(Status::Ok);
|
||
}
|
||
Err(e) => {
|
||
log::error!("Failed to store insight: {:?}", e);
|
||
insight_cx.span().set_status(Status::error(e.to_string()));
|
||
}
|
||
}
|
||
|
||
result?;
|
||
Ok(())
|
||
}
|
||
|
||
/// Extract key topics/entities from messages using LLM for query enrichment
|
||
async fn extract_topics_from_messages(
|
||
&self,
|
||
messages: &[crate::ai::SmsMessage],
|
||
ollama: &OllamaClient,
|
||
) -> Vec<String> {
|
||
if messages.is_empty() {
|
||
return Vec::new();
|
||
}
|
||
|
||
// Format a sample of messages for topic extraction
|
||
let sample_size = messages.len().min(20);
|
||
let user_name = user_display_name();
|
||
let sample_text: Vec<String> = messages
|
||
.iter()
|
||
.take(sample_size)
|
||
.map(|m| {
|
||
let sender: &str = if m.is_sent { &user_name } else { &m.contact };
|
||
format!("{}: {}", sender, m.body)
|
||
})
|
||
.collect();
|
||
|
||
let prompt = format!(
|
||
r#"Extract important entities from these messages that provide context about what was happening. Focus on:
|
||
|
||
1. **People**: Names of specific people mentioned (first names, nicknames)
|
||
2. **Places**: Locations, cities, buildings, workplaces, parks, restaurants, venues
|
||
3. **Activities**: Specific events, hobbies, groups, organizations (e.g., "drum corps", "auditions")
|
||
4. **Unique terms**: Domain-specific words or phrases that might need explanation (e.g., "Hyland", "Vanguard", "DCI")
|
||
|
||
Messages:
|
||
{}
|
||
|
||
Return a comma-separated list of 3-7 specific entities (people, places, activities, unique terms).
|
||
Focus on proper nouns and specific terms that provide context.
|
||
Return ONLY the comma-separated list, nothing else."#,
|
||
sample_text.join("\n")
|
||
);
|
||
|
||
match ollama
|
||
.generate(&prompt, Some("You are an entity extraction assistant. Extract proper nouns, people, places, and domain-specific terms that provide context."))
|
||
.await
|
||
{
|
||
Ok(response) => {
|
||
log::debug!("Topic extraction raw response: {}", response);
|
||
|
||
// Parse comma-separated topics
|
||
let topics: Vec<String> = response
|
||
.split(',')
|
||
.map(|s| s.trim().to_string())
|
||
.filter(|s| !s.is_empty() && s.len() > 1) // Filter out single chars
|
||
.take(7) // Increased from 5 to 7
|
||
.collect();
|
||
|
||
if topics.is_empty() {
|
||
log::warn!("Topic extraction returned empty list from {} messages", messages.len());
|
||
} else {
|
||
log::info!("Extracted {} topics from {} messages: {}", topics.len(), messages.len(), topics.join(", "));
|
||
}
|
||
|
||
topics
|
||
}
|
||
Err(e) => {
|
||
log::warn!("Failed to extract topics from messages: {}", e);
|
||
Vec::new()
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Summarize pre-formatted message strings using LLM (concise version for historical context)
|
||
async fn summarize_messages(
|
||
&self,
|
||
messages: &[String],
|
||
ollama: &OllamaClient,
|
||
custom_system: Option<&str>,
|
||
) -> Option<String> {
|
||
if messages.is_empty() {
|
||
return None;
|
||
}
|
||
|
||
let messages_text = messages.join("\n");
|
||
|
||
let prompt = format!(
|
||
r#"Summarize the context from these messages in 2-3 sentences. Focus on activities, locations, events, and relationships mentioned.
|
||
|
||
Messages:
|
||
{}
|
||
|
||
Return ONLY the summary, nothing else."#,
|
||
messages_text
|
||
);
|
||
|
||
let system = custom_system
|
||
.unwrap_or("You are a context summarization assistant. Be concise and factual.");
|
||
|
||
match ollama.generate(&prompt, Some(system)).await {
|
||
Ok(summary) => Some(summary),
|
||
Err(e) => {
|
||
log::warn!("Failed to summarize messages: {}", e);
|
||
None
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Convert SmsMessage objects to formatted strings and summarize with more detail
|
||
/// This is used for immediate context (±2 days) to preserve conversation details
|
||
async fn summarize_context_from_messages(
|
||
&self,
|
||
messages: &[crate::ai::SmsMessage],
|
||
ollama: &OllamaClient,
|
||
custom_system: Option<&str>,
|
||
) -> Option<String> {
|
||
if messages.is_empty() {
|
||
return None;
|
||
}
|
||
|
||
// Format messages
|
||
let user_name = user_display_name();
|
||
let formatted: Vec<String> = messages
|
||
.iter()
|
||
.map(|m| {
|
||
let sender: &str = if m.is_sent { &user_name } else { &m.contact };
|
||
let timestamp = chrono::DateTime::from_timestamp(m.timestamp, 0)
|
||
.map(|dt| {
|
||
dt.with_timezone(&Local)
|
||
.format("%Y-%m-%d %H:%M")
|
||
.to_string()
|
||
})
|
||
.unwrap_or_else(|| "unknown time".to_string());
|
||
format!("[{}] {}: {}", timestamp, sender, m.body)
|
||
})
|
||
.collect();
|
||
|
||
let messages_text = formatted.join("\n");
|
||
|
||
// Use a more detailed prompt for immediate context
|
||
let prompt = format!(
|
||
r#"Provide a detailed summary of the conversation context from these messages. Include:
|
||
- Key activities, events, and plans discussed
|
||
- Important locations or places mentioned
|
||
- Emotional tone and relationship dynamics
|
||
- Any significant details that provide context about what was happening
|
||
|
||
Be thorough but organized.
|
||
|
||
Messages:
|
||
{}
|
||
|
||
Return ONLY the summary, nothing else."#,
|
||
messages_text
|
||
);
|
||
|
||
let system = custom_system.unwrap_or(
|
||
"You are a context summarization assistant. Be detailed and factual, preserving important context.",
|
||
);
|
||
|
||
match ollama.generate(&prompt, Some(system)).await {
|
||
Ok(summary) => Some(summary),
|
||
Err(e) => {
|
||
log::warn!("Failed to summarize immediate context: {}", e);
|
||
None
|
||
}
|
||
}
|
||
}
|
||
|
||
// ── Tool executors for agentic loop ────────────────────────────────
|
||
|
||
/// Dispatch a tool call to the appropriate executor
|
||
pub(crate) async fn execute_tool(
|
||
&self,
|
||
tool_name: &str,
|
||
arguments: &serde_json::Value,
|
||
ollama: &OllamaClient,
|
||
image_base64: &Option<String>,
|
||
file_path: &str,
|
||
cx: &opentelemetry::Context,
|
||
) -> String {
|
||
let result = match tool_name {
|
||
"search_rag" => self.tool_search_rag(arguments, ollama, cx).await,
|
||
"search_messages" => self.tool_search_messages(arguments).await,
|
||
"get_sms_messages" => self.tool_get_sms_messages(arguments, cx).await,
|
||
"get_calendar_events" => self.tool_get_calendar_events(arguments, cx).await,
|
||
"get_location_history" => self.tool_get_location_history(arguments, cx).await,
|
||
"get_file_tags" => self.tool_get_file_tags(arguments, cx).await,
|
||
"get_faces_in_photo" => self.tool_get_faces_in_photo(arguments, cx).await,
|
||
"describe_photo" => self.tool_describe_photo(ollama, image_base64).await,
|
||
"reverse_geocode" => self.tool_reverse_geocode(arguments).await,
|
||
"get_personal_place_at" => self.tool_get_personal_place_at(arguments).await,
|
||
"recall_entities" => self.tool_recall_entities(arguments, cx).await,
|
||
"recall_facts_for_photo" => self.tool_recall_facts_for_photo(arguments, cx).await,
|
||
"store_entity" => self.tool_store_entity(arguments, ollama, cx).await,
|
||
"store_fact" => self.tool_store_fact(arguments, file_path, cx).await,
|
||
"get_current_datetime" => Self::tool_get_current_datetime(),
|
||
unknown => format!("Unknown tool: {}", unknown),
|
||
};
|
||
if result.starts_with("Error") || result.starts_with("No ") {
|
||
log::warn!("Tool '{}' result: {}", tool_name, result);
|
||
} else {
|
||
log::info!("Tool '{}' result: {} chars", tool_name, result.len());
|
||
}
|
||
result
|
||
}
|
||
|
||
/// Tool: search_rag — semantic search over daily summaries
|
||
async fn tool_search_rag(
|
||
&self,
|
||
args: &serde_json::Value,
|
||
ollama: &OllamaClient,
|
||
_cx: &opentelemetry::Context,
|
||
) -> String {
|
||
let query = match args.get("query").and_then(|v| v.as_str()) {
|
||
Some(q) => q.to_string(),
|
||
None => return "Error: missing required parameter 'query'".to_string(),
|
||
};
|
||
let date_str = match args.get("date").and_then(|v| v.as_str()) {
|
||
Some(d) => d,
|
||
None => return "Error: missing required parameter 'date'".to_string(),
|
||
};
|
||
let date = match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") {
|
||
Ok(d) => d,
|
||
Err(e) => return format!("Error: failed to parse date '{}': {}", date_str, e),
|
||
};
|
||
let contact = args
|
||
.get("contact")
|
||
.and_then(|v| v.as_str())
|
||
.map(|s| s.to_string());
|
||
let limit = args
|
||
.get("limit")
|
||
.and_then(|v| v.as_i64())
|
||
.unwrap_or(10)
|
||
.clamp(1, 25) as usize;
|
||
|
||
log::info!(
|
||
"tool_search_rag: query='{}', date={}, contact={:?}, limit={}",
|
||
query,
|
||
date,
|
||
contact,
|
||
limit
|
||
);
|
||
|
||
// Pull a wider candidate pool than the final limit so the LLM
|
||
// reranker has room to promote less-obvious hits. Candidates_factor
|
||
// is capped so a big `limit` doesn't blow past what the reranker
|
||
// can sensibly judge in one prompt.
|
||
let rerank_enabled = std::env::var("SEARCH_RAG_RERANK")
|
||
.ok()
|
||
.map(|v| v.to_lowercase() != "off" && v != "0")
|
||
.unwrap_or(true);
|
||
let candidate_limit = if rerank_enabled {
|
||
(limit * 3).min(40)
|
||
} else {
|
||
limit
|
||
};
|
||
|
||
let results = match self
|
||
.find_relevant_messages_rag(
|
||
date,
|
||
None,
|
||
contact.as_deref(),
|
||
None,
|
||
candidate_limit,
|
||
Some(&query),
|
||
)
|
||
.await
|
||
{
|
||
Ok(results) if !results.is_empty() => results,
|
||
Ok(_) => return "No relevant messages found.".to_string(),
|
||
Err(e) => return format!("Error searching RAG: {}", e),
|
||
};
|
||
|
||
let final_results = if rerank_enabled && results.len() > limit {
|
||
match self.rerank_with_llm(&query, &results, limit, ollama).await {
|
||
Ok(reordered) => reordered,
|
||
Err(e) => {
|
||
log::warn!("rerank failed, using vector order: {}", e);
|
||
results.into_iter().take(limit).collect()
|
||
}
|
||
}
|
||
} else {
|
||
results.into_iter().take(limit).collect::<Vec<_>>()
|
||
};
|
||
|
||
final_results.join("\n\n")
|
||
}
|
||
|
||
/// LLM-based reranker: ask the local model to pick the top-`limit`
|
||
/// passages from `candidates` that are most relevant to `query`.
|
||
/// Returns the reordered subset.
|
||
///
|
||
/// Cheap-ish because the reranker prompt and output live outside the
|
||
/// agent's visible context — only the final selection lands in the
|
||
/// tool_result. On parse failure we fall back to the input order.
|
||
async fn rerank_with_llm(
|
||
&self,
|
||
query: &str,
|
||
candidates: &[String],
|
||
limit: usize,
|
||
ollama: &OllamaClient,
|
||
) -> Result<Vec<String>> {
|
||
let query_preview: String = query.chars().take(60).collect();
|
||
log::info!(
|
||
"rerank: {} candidates -> top {} (query=\"{}\")",
|
||
candidates.len(),
|
||
limit,
|
||
query_preview
|
||
);
|
||
|
||
// Build numbered list (1-based for readability). Cap each passage
|
||
// at ~1000 chars so very long summaries don't eat the prompt.
|
||
let numbered: String = candidates
|
||
.iter()
|
||
.enumerate()
|
||
.map(|(i, c)| {
|
||
let trimmed = if c.len() > 1000 {
|
||
format!("{}…", &c[..1000])
|
||
} else {
|
||
c.clone()
|
||
};
|
||
format!("[{}] {}", i + 1, trimmed)
|
||
})
|
||
.collect::<Vec<_>>()
|
||
.join("\n\n");
|
||
|
||
let prompt = format!(
|
||
"You are ranking search results. From the numbered passages below, \
|
||
select the {} most relevant to the query. Respond with ONLY a \
|
||
comma-separated list of passage numbers in order from most to \
|
||
least relevant. No explanation, no other text.\n\n\
|
||
Query: {}\n\n\
|
||
Passages:\n{}\n\n\
|
||
Top {} passage numbers:",
|
||
limit, query, numbered, limit
|
||
);
|
||
|
||
let started = std::time::Instant::now();
|
||
let response = ollama
|
||
.generate_no_think(
|
||
&prompt,
|
||
Some(
|
||
"You are a terse relevance ranker. You output only numbers separated by commas.",
|
||
),
|
||
)
|
||
.await?;
|
||
log::info!(
|
||
"rerank: finished in {} ms (prompt={} chars)",
|
||
started.elapsed().as_millis(),
|
||
prompt.len()
|
||
);
|
||
|
||
// Extract indices from the response. Accept "3, 1, 7" and also
|
||
// tolerate "[3, 1, 7]" or "3,1,7,..." with trailing junk.
|
||
let picks: Vec<usize> = response
|
||
.split(|c: char| !c.is_ascii_digit())
|
||
.filter_map(|s| s.parse::<usize>().ok())
|
||
.filter(|&n| n >= 1 && n <= candidates.len())
|
||
.collect();
|
||
|
||
if picks.is_empty() {
|
||
return Err(anyhow::anyhow!(
|
||
"reranker returned no usable indices (raw: {})",
|
||
response.chars().take(120).collect::<String>()
|
||
));
|
||
}
|
||
|
||
let mut seen = std::collections::HashSet::new();
|
||
let mut reordered: Vec<String> = Vec::with_capacity(limit);
|
||
let mut final_indices: Vec<usize> = Vec::with_capacity(limit);
|
||
for n in picks {
|
||
if seen.insert(n) {
|
||
reordered.push(candidates[n - 1].clone());
|
||
final_indices.push(n);
|
||
if reordered.len() >= limit {
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
// Top-up from original order if the reranker returned fewer than
|
||
// `limit` distinct entries.
|
||
if reordered.len() < limit {
|
||
for (i, c) in candidates.iter().enumerate() {
|
||
if !seen.contains(&(i + 1)) {
|
||
reordered.push(c.clone());
|
||
final_indices.push(i + 1);
|
||
if reordered.len() >= limit {
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Debug snapshot: show what the reranker changed. Position p holds
|
||
// the 1-based index of the candidate that now sits at position p.
|
||
// A value that equals its position means "no change at that slot".
|
||
let swapped = final_indices
|
||
.iter()
|
||
.enumerate()
|
||
.filter(|(pos, idx)| **idx != pos + 1)
|
||
.count();
|
||
log::info!(
|
||
"rerank: final indices (1-based): {:?} — {} of top {} swapped from vector order",
|
||
final_indices,
|
||
swapped,
|
||
final_indices.len()
|
||
);
|
||
let show = final_indices.len().min(5);
|
||
log::debug!("rerank: vector-order top {}:", show);
|
||
for (i, c) in candidates.iter().enumerate().take(show) {
|
||
let preview: String = c.chars().take(100).collect();
|
||
log::debug!("rerank: [{}] {}", i + 1, preview);
|
||
}
|
||
log::debug!("rerank: reranked top {}:", show);
|
||
for (pos, idx) in final_indices.iter().enumerate().take(show) {
|
||
let preview: String = candidates[*idx - 1].chars().take(100).collect();
|
||
log::debug!("rerank: [{}] (orig #{}) {}", pos + 1, idx, preview);
|
||
}
|
||
|
||
Ok(reordered)
|
||
}
|
||
|
||
/// Tool: search_messages — keyword / semantic / hybrid search over all
|
||
/// SMS message bodies via the Django FTS5 + embeddings pipeline. Now
|
||
/// supports optional `contact_id`, `start_ts`, `end_ts` filters.
|
||
async fn tool_search_messages(&self, args: &serde_json::Value) -> String {
|
||
let query = match args.get("query").and_then(|v| v.as_str()) {
|
||
Some(q) if !q.trim().is_empty() => q.trim(),
|
||
_ => {
|
||
let has_date = args.get("date").is_some()
|
||
|| args.get("start_ts").is_some()
|
||
|| args.get("end_ts").is_some();
|
||
let has_contact = args.get("contact").is_some() || args.get("contact_id").is_some();
|
||
if has_date || has_contact {
|
||
return "Error: search_messages needs a 'query' (keywords/phrase). \
|
||
To fetch messages around a date or from a contact without keywords, \
|
||
call get_sms_messages with { date, contact? } instead."
|
||
.to_string();
|
||
}
|
||
return "Error: missing required parameter 'query'".to_string();
|
||
}
|
||
};
|
||
if query.len() < 3 {
|
||
return "Error: query must be at least 3 characters".to_string();
|
||
}
|
||
let mode = args
|
||
.get("mode")
|
||
.and_then(|v| v.as_str())
|
||
.map(|s| s.to_lowercase())
|
||
.unwrap_or_else(|| "hybrid".to_string());
|
||
if !matches!(mode.as_str(), "fts5" | "semantic" | "hybrid") {
|
||
return format!(
|
||
"Error: unknown mode '{}'; expected one of: fts5, semantic, hybrid",
|
||
mode
|
||
);
|
||
}
|
||
let user_limit = args
|
||
.get("limit")
|
||
.and_then(|v| v.as_i64())
|
||
.unwrap_or(20)
|
||
.clamp(1, 50) as usize;
|
||
let contact_id = args.get("contact_id").and_then(|v| v.as_i64());
|
||
let start_ts = args.get("start_ts").and_then(|v| v.as_i64());
|
||
let end_ts = args.get("end_ts").and_then(|v| v.as_i64());
|
||
let has_date_filter = start_ts.is_some() || end_ts.is_some();
|
||
|
||
// When a date filter is supplied, fetch a larger pool from SMS-API
|
||
// so in-window matches that ranked lower than out-of-window ones
|
||
// aren't lost.
|
||
let fetch_limit = if has_date_filter { 100 } else { user_limit };
|
||
|
||
log::info!(
|
||
"tool_search_messages: query='{}', mode={}, contact_id={:?}, range=[{:?}, {:?}], user_limit={}, fetch_limit={}",
|
||
query,
|
||
mode,
|
||
contact_id,
|
||
start_ts,
|
||
end_ts,
|
||
user_limit,
|
||
fetch_limit
|
||
);
|
||
|
||
let hits = match self
|
||
.sms_client
|
||
.search_messages_with_contact(query, &mode, fetch_limit, contact_id)
|
||
.await
|
||
{
|
||
Ok(h) => h,
|
||
Err(e) => return format!("Error searching messages: {}", e),
|
||
};
|
||
|
||
// Date-range post-filter on the client side. SMS-API's /search/
|
||
// doesn't accept date params; mirroring Apollo's pattern here.
|
||
let filtered: Vec<_> = hits
|
||
.into_iter()
|
||
.filter(|h| {
|
||
if let Some(s) = start_ts
|
||
&& h.date < s
|
||
{
|
||
return false;
|
||
}
|
||
if let Some(e) = end_ts
|
||
&& h.date > e
|
||
{
|
||
return false;
|
||
}
|
||
true
|
||
})
|
||
.take(user_limit)
|
||
.collect();
|
||
|
||
if filtered.is_empty() {
|
||
return "No messages matched.".to_string();
|
||
}
|
||
|
||
let user_name = user_display_name();
|
||
let mut out = String::new();
|
||
out.push_str(&format!(
|
||
"Found {} messages (mode: {}{}):\n\n",
|
||
filtered.len(),
|
||
mode,
|
||
if has_date_filter {
|
||
", date-filtered"
|
||
} else {
|
||
""
|
||
}
|
||
));
|
||
for h in filtered {
|
||
let date = chrono::DateTime::from_timestamp(h.date, 0)
|
||
.map(|dt| dt.format("%Y-%m-%d").to_string())
|
||
.unwrap_or_else(|| h.date.to_string());
|
||
let direction: &str = if h.type_ == 2 {
|
||
&user_name
|
||
} else {
|
||
&h.contact_name
|
||
};
|
||
let score = h
|
||
.similarity_score
|
||
.map(|s| format!(" [score {:.2}]", s))
|
||
.unwrap_or_default();
|
||
out.push_str(&format!(
|
||
"[{}]{} {} — {}\n\n",
|
||
date, score, direction, h.body
|
||
));
|
||
}
|
||
out
|
||
}
|
||
|
||
/// Tool: get_sms_messages — fetch SMS messages near a date for a contact
|
||
async fn tool_get_sms_messages(
|
||
&self,
|
||
args: &serde_json::Value,
|
||
_cx: &opentelemetry::Context,
|
||
) -> String {
|
||
let date_str = match args.get("date").and_then(|v| v.as_str()) {
|
||
Some(d) => d,
|
||
None => return "Error: missing required parameter 'date'".to_string(),
|
||
};
|
||
let contact = args
|
||
.get("contact")
|
||
.and_then(|v| v.as_str())
|
||
.map(|s| s.to_string());
|
||
let days_radius = args
|
||
.get("days_radius")
|
||
.and_then(|v| v.as_i64())
|
||
.unwrap_or(4)
|
||
.clamp(1, 30);
|
||
let limit = args
|
||
.get("limit")
|
||
.and_then(|v| v.as_i64())
|
||
.unwrap_or(60)
|
||
.clamp(1, 150) as usize;
|
||
|
||
let date = match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") {
|
||
Ok(d) => d,
|
||
Err(e) => return format!("Error: failed to parse date '{}': {}", date_str, e),
|
||
};
|
||
let timestamp = date.and_hms_opt(12, 0, 0).unwrap().and_utc().timestamp();
|
||
|
||
log::info!(
|
||
"tool_get_sms_messages: date={}, contact={:?}, days_radius={}, limit={}",
|
||
date,
|
||
contact,
|
||
days_radius,
|
||
limit
|
||
);
|
||
|
||
match self
|
||
.sms_client
|
||
.fetch_messages_for_contact(contact.as_deref(), timestamp, days_radius)
|
||
.await
|
||
{
|
||
Ok(messages) if !messages.is_empty() => {
|
||
let user_name = user_display_name();
|
||
let formatted: Vec<String> = messages
|
||
.iter()
|
||
.take(limit)
|
||
.map(|m| {
|
||
let sender: &str = if m.is_sent { &user_name } else { &m.contact };
|
||
let ts = DateTime::from_timestamp(m.timestamp, 0)
|
||
.map(|dt| {
|
||
dt.with_timezone(&Local)
|
||
.format("%Y-%m-%d %H:%M")
|
||
.to_string()
|
||
})
|
||
.unwrap_or_else(|| "unknown".to_string());
|
||
format!("[{}] {}: {}", ts, sender, m.body)
|
||
})
|
||
.collect();
|
||
format!(
|
||
"Found {} messages:\n{}",
|
||
messages.len(),
|
||
formatted.join("\n")
|
||
)
|
||
}
|
||
Ok(_) => "No messages found.".to_string(),
|
||
Err(e) => {
|
||
log::warn!("tool_get_sms_messages failed: {}", e);
|
||
format!("Error fetching SMS messages: {}", e)
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Tool: get_calendar_events — fetch calendar events near a date
|
||
async fn tool_get_calendar_events(
|
||
&self,
|
||
args: &serde_json::Value,
|
||
cx: &opentelemetry::Context,
|
||
) -> String {
|
||
let date_str = match args.get("date").and_then(|v| v.as_str()) {
|
||
Some(d) => d,
|
||
None => return "Error: missing required parameter 'date'".to_string(),
|
||
};
|
||
let days_radius = args
|
||
.get("days_radius")
|
||
.and_then(|v| v.as_i64())
|
||
.unwrap_or(7);
|
||
let limit = args
|
||
.get("limit")
|
||
.and_then(|v| v.as_i64())
|
||
.unwrap_or(20)
|
||
.clamp(1, 50) as usize;
|
||
|
||
let date = match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") {
|
||
Ok(d) => d,
|
||
Err(e) => return format!("Error: failed to parse date '{}': {}", date_str, e),
|
||
};
|
||
let timestamp = date.and_hms_opt(12, 0, 0).unwrap().and_utc().timestamp();
|
||
|
||
log::info!(
|
||
"tool_get_calendar_events: date={}, days_radius={}, limit={}",
|
||
date,
|
||
days_radius,
|
||
limit
|
||
);
|
||
|
||
let events = {
|
||
let mut dao = self
|
||
.calendar_dao
|
||
.lock()
|
||
.expect("Unable to lock CalendarEventDao");
|
||
dao.find_relevant_events_hybrid(cx, timestamp, days_radius, None, limit)
|
||
.ok()
|
||
};
|
||
|
||
match events {
|
||
Some(evts) if !evts.is_empty() => {
|
||
let formatted: Vec<String> = evts
|
||
.iter()
|
||
.map(|e| {
|
||
let dt = DateTime::from_timestamp(e.start_time, 0)
|
||
.map(|dt| {
|
||
dt.with_timezone(&Local)
|
||
.format("%Y-%m-%d %H:%M")
|
||
.to_string()
|
||
})
|
||
.unwrap_or_else(|| "unknown".to_string());
|
||
let loc = e
|
||
.location
|
||
.as_ref()
|
||
.map(|l| format!(" at {}", l))
|
||
.unwrap_or_default();
|
||
let attendees = e
|
||
.attendees
|
||
.as_ref()
|
||
.and_then(|a| serde_json::from_str::<Vec<String>>(a).ok())
|
||
.map(|list| format!(" (with {})", list.join(", ")))
|
||
.unwrap_or_default();
|
||
format!("[{}] {}{}{}", dt, e.summary, loc, attendees)
|
||
})
|
||
.collect();
|
||
format!(
|
||
"Found {} calendar events:\n{}",
|
||
evts.len(),
|
||
formatted.join("\n")
|
||
)
|
||
}
|
||
Some(_) => "No calendar events found.".to_string(),
|
||
None => "No calendar events found.".to_string(),
|
||
}
|
||
}
|
||
|
||
/// Tool: get_location_history — fetch location records near a date
|
||
async fn tool_get_location_history(
|
||
&self,
|
||
args: &serde_json::Value,
|
||
cx: &opentelemetry::Context,
|
||
) -> String {
|
||
let date_str = match args.get("date").and_then(|v| v.as_str()) {
|
||
Some(d) => d,
|
||
None => return "Error: missing required parameter 'date'".to_string(),
|
||
};
|
||
let days_radius = args
|
||
.get("days_radius")
|
||
.and_then(|v| v.as_i64())
|
||
.unwrap_or(14);
|
||
|
||
let date = match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") {
|
||
Ok(d) => d,
|
||
Err(e) => return format!("Error: failed to parse date '{}': {}", date_str, e),
|
||
};
|
||
let timestamp = date.and_hms_opt(12, 0, 0).unwrap().and_utc().timestamp();
|
||
|
||
log::info!(
|
||
"tool_get_location_history: date={}, days_radius={}",
|
||
date,
|
||
days_radius
|
||
);
|
||
|
||
let start_ts = timestamp - (days_radius * 86400);
|
||
let end_ts = timestamp + (days_radius * 86400);
|
||
|
||
let locations = {
|
||
let mut dao = self
|
||
.location_dao
|
||
.lock()
|
||
.expect("Unable to lock LocationHistoryDao");
|
||
dao.find_locations_in_range(cx, start_ts, end_ts).ok()
|
||
};
|
||
|
||
match locations {
|
||
Some(locs) if !locs.is_empty() => {
|
||
let formatted: Vec<String> = locs
|
||
.iter()
|
||
.take(20)
|
||
.map(|loc| {
|
||
let dt = DateTime::from_timestamp(loc.timestamp, 0)
|
||
.map(|dt| {
|
||
dt.with_timezone(&Local)
|
||
.format("%Y-%m-%d %H:%M")
|
||
.to_string()
|
||
})
|
||
.unwrap_or_else(|| "unknown".to_string());
|
||
let activity = loc
|
||
.activity
|
||
.as_ref()
|
||
.map(|a| format!(" ({})", a))
|
||
.unwrap_or_default();
|
||
let place = loc
|
||
.place_name
|
||
.as_ref()
|
||
.map(|p| format!(" at {}", p))
|
||
.unwrap_or_default();
|
||
format!(
|
||
"[{}] {:.4}, {:.4}{}{}",
|
||
dt, loc.latitude, loc.longitude, place, activity
|
||
)
|
||
})
|
||
.collect();
|
||
format!(
|
||
"Found {} location records:\n{}",
|
||
locs.len(),
|
||
formatted.join("\n")
|
||
)
|
||
}
|
||
Some(_) => "No location history found.".to_string(),
|
||
None => "No location history found.".to_string(),
|
||
}
|
||
}
|
||
|
||
/// Tool: get_file_tags — fetch tags for a file path
|
||
async fn tool_get_file_tags(
|
||
&self,
|
||
args: &serde_json::Value,
|
||
cx: &opentelemetry::Context,
|
||
) -> String {
|
||
let file_path = match args.get("file_path").and_then(|v| v.as_str()) {
|
||
Some(p) => p.to_string(),
|
||
None => return "Error: missing required parameter 'file_path'".to_string(),
|
||
};
|
||
|
||
log::info!("tool_get_file_tags: file_path='{}'", file_path);
|
||
|
||
let tags = {
|
||
let mut dao = self.tag_dao.lock().expect("Unable to lock TagDao");
|
||
dao.get_tags_for_path(cx, &file_path).ok()
|
||
};
|
||
|
||
match tags {
|
||
Some(t) if !t.is_empty() => {
|
||
let names: Vec<String> = t.into_iter().map(|tag| tag.name).collect();
|
||
names.join(", ")
|
||
}
|
||
Some(_) => "No tags found.".to_string(),
|
||
None => "No tags found.".to_string(),
|
||
}
|
||
}
|
||
|
||
/// Tool: get_faces_in_photo — list face detections + person names for
|
||
/// the given file path. Resolves rel_path → content_hash via FaceDao,
|
||
/// then queries face_detections joined with persons (status='detected'
|
||
/// only). Returns a compact bullet list keyed for human-LLM readability.
|
||
async fn tool_get_faces_in_photo(
|
||
&self,
|
||
args: &serde_json::Value,
|
||
cx: &opentelemetry::Context,
|
||
) -> String {
|
||
let file_path = match args.get("file_path").and_then(|v| v.as_str()) {
|
||
Some(p) if !p.trim().is_empty() => p.trim().to_string(),
|
||
_ => return "Error: missing required parameter 'file_path'".to_string(),
|
||
};
|
||
log::info!("tool_get_faces_in_photo: file_path='{}'", file_path);
|
||
|
||
// Resolve content_hash from any library that has this rel_path.
|
||
// Hold the FaceDao lock once across all libraries — resolve_content_hash
|
||
// is synchronous and there's no await in the loop body.
|
||
let content_hash: Option<String> = {
|
||
let mut dao = self.face_dao.lock().expect("Unable to lock FaceDao");
|
||
self.libraries.iter().find_map(|lib| {
|
||
dao.resolve_content_hash(cx, lib.id, &file_path)
|
||
.ok()
|
||
.flatten()
|
||
})
|
||
};
|
||
let Some(content_hash) = content_hash else {
|
||
return "No content_hash found for that file path (the photo may not be indexed yet, \
|
||
or the path doesn't match any library)."
|
||
.to_string();
|
||
};
|
||
|
||
let faces = {
|
||
let mut dao = self.face_dao.lock().expect("Unable to lock FaceDao");
|
||
match dao.list_for_content_hash(cx, &content_hash) {
|
||
Ok(rows) => rows,
|
||
Err(e) => return format!("Error querying faces: {}", e),
|
||
}
|
||
};
|
||
|
||
if faces.is_empty() {
|
||
return "No faces detected in this photo.".to_string();
|
||
}
|
||
|
||
// Render: bound faces grouped by person first, then unbound. The
|
||
// model uses the bound names directly; the unbound count + bbox
|
||
// helps it count people without naming them.
|
||
let bound: Vec<&_> = faces.iter().filter(|f| f.person_name.is_some()).collect();
|
||
let unbound: Vec<&_> = faces.iter().filter(|f| f.person_name.is_none()).collect();
|
||
|
||
let mut out = format!("Found {} face(s) in this photo:\n", faces.len());
|
||
for f in &bound {
|
||
// Invariant: `bound` is filtered on `person_name.is_some()` above.
|
||
let name = f.person_name.as_deref().expect("bound face must have a name");
|
||
out.push_str(&format!(
|
||
"- {} (confidence {:.2}, bbox x={:.2} y={:.2} w={:.2} h={:.2}, source: {})\n",
|
||
name,
|
||
f.confidence,
|
||
f.bbox_x,
|
||
f.bbox_y,
|
||
f.bbox_w,
|
||
f.bbox_h,
|
||
f.source,
|
||
));
|
||
}
|
||
for f in &unbound {
|
||
out.push_str(&format!(
|
||
"- (unidentified) confidence {:.2}, bbox x={:.2} y={:.2} w={:.2} h={:.2}, source: {}\n",
|
||
f.confidence,
|
||
f.bbox_x,
|
||
f.bbox_y,
|
||
f.bbox_w,
|
||
f.bbox_h,
|
||
f.source,
|
||
));
|
||
}
|
||
out
|
||
}
|
||
|
||
/// Tool: describe_photo — generate a visual description of the photo
|
||
async fn tool_describe_photo(
|
||
&self,
|
||
ollama: &OllamaClient,
|
||
image_base64: &Option<String>,
|
||
) -> String {
|
||
log::info!("tool_describe_photo: generating visual description");
|
||
|
||
match image_base64 {
|
||
Some(img) => match ollama.generate_photo_description(img).await {
|
||
Ok(desc) => desc,
|
||
Err(e) => format!("Error describing photo: {}", e),
|
||
},
|
||
None => "No image available for description.".to_string(),
|
||
}
|
||
}
|
||
|
||
/// Tool: reverse_geocode — convert GPS coordinates to a human-readable place name
|
||
async fn tool_reverse_geocode(&self, args: &serde_json::Value) -> String {
|
||
let lat = match args.get("latitude").and_then(|v| v.as_f64()) {
|
||
Some(v) => v,
|
||
None => return "Error: missing required parameter 'latitude'".to_string(),
|
||
};
|
||
let lon = match args.get("longitude").and_then(|v| v.as_f64()) {
|
||
Some(v) => v,
|
||
None => return "Error: missing required parameter 'longitude'".to_string(),
|
||
};
|
||
|
||
log::info!("tool_reverse_geocode: lat={}, lon={}", lat, lon);
|
||
|
||
match self.reverse_geocode(lat, lon).await {
|
||
Some(place) => place,
|
||
None => "Could not resolve coordinates to a place name.".to_string(),
|
||
}
|
||
}
|
||
|
||
/// Tool: get_personal_place_at — look up the user's named places (Home,
|
||
/// Work, Cabin) at a coordinate. Server-side filter; results sorted
|
||
/// smallest-radius first.
|
||
async fn tool_get_personal_place_at(&self, args: &serde_json::Value) -> String {
|
||
if !self.apollo_client.is_enabled() {
|
||
return "Personal place lookup is disabled.".to_string();
|
||
}
|
||
let lat = match args.get("latitude").and_then(|v| v.as_f64()) {
|
||
Some(v) => v,
|
||
None => return "Error: missing required parameter 'latitude'".to_string(),
|
||
};
|
||
let lon = match args.get("longitude").and_then(|v| v.as_f64()) {
|
||
Some(v) => v,
|
||
None => return "Error: missing required parameter 'longitude'".to_string(),
|
||
};
|
||
log::info!("tool_get_personal_place_at: lat={}, lon={}", lat, lon);
|
||
let places = self.apollo_client.places_containing(lat, lon).await;
|
||
if places.is_empty() {
|
||
return "No personal place contains this coordinate.".to_string();
|
||
}
|
||
places
|
||
.iter()
|
||
.map(|p| {
|
||
let category = p
|
||
.category
|
||
.as_deref()
|
||
.map(|c| format!(" [{c}]"))
|
||
.unwrap_or_default();
|
||
let desc = if p.description.is_empty() {
|
||
"(no description)".to_string()
|
||
} else {
|
||
p.description.clone()
|
||
};
|
||
format!(
|
||
"- {}{}: {} (radius {} m)",
|
||
p.name, category, desc, p.radius_m
|
||
)
|
||
})
|
||
.collect::<Vec<_>>()
|
||
.join("\n")
|
||
}
|
||
|
||
/// Tool: recall_entities — search the knowledge memory for known entities
|
||
async fn tool_recall_entities(
|
||
&self,
|
||
args: &serde_json::Value,
|
||
cx: &opentelemetry::Context,
|
||
) -> String {
|
||
use crate::database::EntityFilter;
|
||
|
||
let name_search = args
|
||
.get("name")
|
||
.and_then(|v| v.as_str())
|
||
.map(|s| s.to_string());
|
||
let entity_type = args
|
||
.get("entity_type")
|
||
.and_then(|v| v.as_str())
|
||
.map(|s| s.to_string());
|
||
let limit = args
|
||
.get("limit")
|
||
.and_then(|v| v.as_i64())
|
||
.unwrap_or(20)
|
||
.clamp(1, 50);
|
||
|
||
log::info!(
|
||
"tool_recall_entities: name={:?}, type={:?}, limit={}",
|
||
name_search,
|
||
entity_type,
|
||
limit
|
||
);
|
||
|
||
let filter = EntityFilter {
|
||
entity_type,
|
||
status: Some("active".to_string()),
|
||
search: name_search,
|
||
limit,
|
||
offset: 0,
|
||
};
|
||
|
||
let mut kdao = self
|
||
.knowledge_dao
|
||
.lock()
|
||
.expect("Unable to lock KnowledgeDao");
|
||
match kdao.list_entities(cx, filter) {
|
||
Ok((entities, _total)) if entities.is_empty() => {
|
||
"No known entities found matching the query.".to_string()
|
||
}
|
||
Ok((entities, _total)) => {
|
||
let lines: Vec<String> = entities
|
||
.iter()
|
||
.map(|e| {
|
||
format!(
|
||
"ID:{} | {} | {} | {} | confidence:{:.2}",
|
||
e.id, e.entity_type, e.name, e.description, e.confidence
|
||
)
|
||
})
|
||
.collect();
|
||
format!("Known entities:\n{}", lines.join("\n"))
|
||
}
|
||
Err(e) => format!("Error recalling entities: {:?}", e),
|
||
}
|
||
}
|
||
|
||
/// Tool: recall_facts_for_photo — retrieve facts linked to a specific photo
|
||
async fn tool_recall_facts_for_photo(
|
||
&self,
|
||
args: &serde_json::Value,
|
||
cx: &opentelemetry::Context,
|
||
) -> String {
|
||
let file_path = match args.get("file_path").and_then(|v| v.as_str()) {
|
||
Some(p) => p.to_string(),
|
||
None => return "Error: missing required parameter 'file_path'".to_string(),
|
||
};
|
||
|
||
log::info!("tool_recall_facts_for_photo: file_path={}", file_path);
|
||
|
||
let mut kdao = self
|
||
.knowledge_dao
|
||
.lock()
|
||
.expect("Unable to lock KnowledgeDao");
|
||
|
||
// Fetch photo links to find which entities appear in this photo
|
||
let links = match kdao.get_links_for_photo(cx, &file_path) {
|
||
Ok(l) => l,
|
||
Err(e) => return format!("Error fetching photo links: {:?}", e),
|
||
};
|
||
|
||
if links.is_empty() {
|
||
return "No knowledge facts found for this photo.".to_string();
|
||
}
|
||
|
||
let mut output_lines = Vec::new();
|
||
let entity_ids: Vec<i32> = links.iter().map(|l| l.entity_id).collect();
|
||
|
||
// For each linked entity, fetch its facts
|
||
for entity_id in entity_ids {
|
||
if let Ok(entity) = kdao.get_entity_by_id(cx, entity_id)
|
||
&& let Some(e) = entity
|
||
{
|
||
let role = links
|
||
.iter()
|
||
.find(|l| l.entity_id == entity_id)
|
||
.map(|l| l.role.as_str())
|
||
.unwrap_or("subject");
|
||
output_lines.push(format!(
|
||
"Entity: {} ({}, role: {})",
|
||
e.name, e.entity_type, role
|
||
));
|
||
if let Ok(facts) = kdao.get_facts_for_entity(cx, entity_id) {
|
||
for f in facts.iter().filter(|f| f.status == "active") {
|
||
let obj = if let Some(ref v) = f.object_value {
|
||
v.clone()
|
||
} else if let Some(oid) = f.object_entity_id {
|
||
kdao.get_entity_by_id(cx, oid)
|
||
.ok()
|
||
.flatten()
|
||
.map(|e| format!("{} (entity ID: {})", e.name, e.id))
|
||
.unwrap_or_else(|| format!("entity:{}", oid))
|
||
} else {
|
||
"(unknown)".to_string()
|
||
};
|
||
output_lines.push(format!(" - {} {}", f.predicate, obj));
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if output_lines.is_empty() {
|
||
"No active knowledge facts found for this photo.".to_string()
|
||
} else {
|
||
format!("Knowledge for this photo:\n{}", output_lines.join("\n"))
|
||
}
|
||
}
|
||
|
||
/// Tool: store_entity — upsert an entity into the knowledge memory
|
||
async fn tool_store_entity(
|
||
&self,
|
||
args: &serde_json::Value,
|
||
ollama: &OllamaClient,
|
||
cx: &opentelemetry::Context,
|
||
) -> String {
|
||
use crate::database::models::InsertEntity;
|
||
|
||
let name = match args.get("name").and_then(|v| v.as_str()) {
|
||
Some(n) => n.to_string(),
|
||
None => return "Error: missing required parameter 'name'".to_string(),
|
||
};
|
||
let entity_type = match args.get("entity_type").and_then(|v| v.as_str()) {
|
||
Some(t) => t.to_string(),
|
||
None => return "Error: missing required parameter 'entity_type'".to_string(),
|
||
};
|
||
let description = args
|
||
.get("description")
|
||
.and_then(|v| v.as_str())
|
||
.unwrap_or("")
|
||
.to_string();
|
||
|
||
log::info!(
|
||
"tool_store_entity: name='{}', type='{}', description='{}'",
|
||
name,
|
||
entity_type,
|
||
description
|
||
);
|
||
|
||
// Pre-flight similarity check — surface near-duplicates to the model
|
||
// before it commits to a new entity. Uses the first name token as the
|
||
// search term so "Sarah" matches when storing "Sarah Johnson" and vice
|
||
// versa. Exact-name matches are excluded (upsert_entity deduplicates
|
||
// those already). Results are appended to the tool response so the
|
||
// model can choose to use an existing entity's ID instead.
|
||
let similar_entities: Vec<String> = {
|
||
use crate::database::EntityFilter;
|
||
use crate::database::knowledge_dao::normalize_entity_type;
|
||
let normalised_type = normalize_entity_type(&entity_type);
|
||
let first_token = name.split_whitespace().next().unwrap_or(&name).to_string();
|
||
let filter = EntityFilter {
|
||
entity_type: None, // search all types, filter client-side to avoid case issues
|
||
status: Some("active".to_string()),
|
||
search: Some(first_token),
|
||
limit: 10,
|
||
offset: 0,
|
||
};
|
||
let mut kdao = self
|
||
.knowledge_dao
|
||
.lock()
|
||
.expect("Unable to lock KnowledgeDao");
|
||
kdao.list_entities(cx, filter)
|
||
.unwrap_or_default()
|
||
.0
|
||
.into_iter()
|
||
.filter(|e| {
|
||
normalize_entity_type(&e.entity_type) == normalised_type
|
||
&& e.name.to_lowercase() != name.to_lowercase()
|
||
})
|
||
.map(|e| format!(" ID:{} | {} | {}", e.id, e.name, e.description))
|
||
.collect()
|
||
};
|
||
|
||
// Generate embedding for name + description (best-effort)
|
||
let embed_text = format!("{} {}", name, description);
|
||
let embedding: Option<Vec<u8>> = match ollama.generate_embedding(&embed_text).await {
|
||
Ok(vec) => {
|
||
let bytes: Vec<u8> = vec.iter().flat_map(|f| f.to_le_bytes()).collect();
|
||
Some(bytes)
|
||
}
|
||
Err(e) => {
|
||
log::warn!("Embedding generation failed for entity '{}': {}", name, e);
|
||
None
|
||
}
|
||
};
|
||
|
||
let now = chrono::Utc::now().timestamp();
|
||
let insert = InsertEntity {
|
||
name,
|
||
entity_type,
|
||
description,
|
||
embedding,
|
||
confidence: 0.6,
|
||
status: "active".to_string(),
|
||
created_at: now,
|
||
updated_at: now,
|
||
};
|
||
|
||
let mut kdao = self
|
||
.knowledge_dao
|
||
.lock()
|
||
.expect("Unable to lock KnowledgeDao");
|
||
match kdao.upsert_entity(cx, insert) {
|
||
Ok(entity) => {
|
||
let mut response = format!(
|
||
"Entity stored: ID:{} | {} | {} | confidence:{:.2}",
|
||
entity.id, entity.entity_type, entity.name, entity.confidence
|
||
);
|
||
if !similar_entities.is_empty() {
|
||
response.push_str(
|
||
"\nSimilar existing entities found — verify this is not a duplicate:\n",
|
||
);
|
||
response.push_str(&similar_entities.join("\n"));
|
||
response.push_str(
|
||
"\nIf one of these is the same entity, use their existing ID in store_fact instead of the newly created one.",
|
||
);
|
||
}
|
||
response
|
||
}
|
||
Err(e) => format!("Error storing entity: {:?}", e),
|
||
}
|
||
}
|
||
|
||
/// Tool: store_fact — record a fact about an entity, linked to the current photo
|
||
async fn tool_store_fact(
|
||
&self,
|
||
args: &serde_json::Value,
|
||
file_path: &str,
|
||
cx: &opentelemetry::Context,
|
||
) -> String {
|
||
use crate::database::models::{InsertEntityFact, InsertEntityPhotoLink};
|
||
|
||
let subject_entity_id = match args.get("subject_entity_id").and_then(|v| v.as_i64()) {
|
||
Some(id) => id as i32,
|
||
None => return "Error: missing required parameter 'subject_entity_id'".to_string(),
|
||
};
|
||
let predicate = match args.get("predicate").and_then(|v| v.as_str()) {
|
||
Some(p) => p.to_string(),
|
||
None => return "Error: missing required parameter 'predicate'".to_string(),
|
||
};
|
||
let object_entity_id = args
|
||
.get("object_entity_id")
|
||
.and_then(|v| v.as_i64())
|
||
.map(|id| id as i32);
|
||
let object_value = args
|
||
.get("object_value")
|
||
.and_then(|v| v.as_str())
|
||
.map(|s| s.to_string());
|
||
|
||
if object_entity_id.is_none() && object_value.is_none() {
|
||
return "Error: provide either object_entity_id or object_value".to_string();
|
||
}
|
||
|
||
let photo_role = args
|
||
.get("photo_role")
|
||
.and_then(|v| v.as_str())
|
||
.unwrap_or("subject")
|
||
.to_string();
|
||
|
||
log::info!(
|
||
"tool_store_fact: entity_id={}, predicate='{}', object_entity_id={:?}, object_value={:?}, photo='{}'",
|
||
subject_entity_id,
|
||
predicate,
|
||
object_entity_id,
|
||
object_value,
|
||
file_path
|
||
);
|
||
|
||
let fact = InsertEntityFact {
|
||
subject_entity_id,
|
||
predicate,
|
||
object_entity_id,
|
||
object_value,
|
||
source_photo: Some(file_path.to_string()),
|
||
source_insight_id: None, // will be backfilled after store_insight
|
||
confidence: 0.6,
|
||
status: "active".to_string(),
|
||
created_at: chrono::Utc::now().timestamp(),
|
||
};
|
||
|
||
let mut kdao = self
|
||
.knowledge_dao
|
||
.lock()
|
||
.expect("Unable to lock KnowledgeDao");
|
||
|
||
// Upsert the fact (corroboration bumps confidence if duplicate)
|
||
let (stored_fact, is_new) = match kdao.upsert_fact(cx, fact) {
|
||
Ok(r) => r,
|
||
Err(e) => return format!("Error storing fact: {:?}", e),
|
||
};
|
||
|
||
// Upsert a photo link so this entity is associated with this photo
|
||
let link = InsertEntityPhotoLink {
|
||
entity_id: subject_entity_id,
|
||
library_id: crate::libraries::PRIMARY_LIBRARY_ID,
|
||
file_path: file_path.to_string(),
|
||
role: photo_role,
|
||
};
|
||
if let Err(e) = kdao.upsert_photo_link(cx, link) {
|
||
log::warn!(
|
||
"Failed to upsert photo link for entity {}: {:?}",
|
||
subject_entity_id,
|
||
e
|
||
);
|
||
}
|
||
|
||
let action = if is_new {
|
||
"Stored new fact"
|
||
} else {
|
||
"Corroborated existing fact"
|
||
};
|
||
format!(
|
||
"{}: ID:{} | confidence:{:.2}",
|
||
action, stored_fact.id, stored_fact.confidence
|
||
)
|
||
}
|
||
|
||
/// Tool: get_current_datetime — returns the current local date and time
|
||
fn tool_get_current_datetime() -> String {
|
||
let now = Local::now();
|
||
format!(
|
||
"Current date/time: {} ({})",
|
||
now.format("%Y-%m-%d %H:%M:%S %Z"),
|
||
now.format("%A")
|
||
)
|
||
}
|
||
|
||
// ── Agentic insight generation ──────────────────────────────────────
|
||
|
||
/// Build the list of tool definitions for the agentic loop, gated by
|
||
/// `opts`. Always-on tools: `search_messages`, `get_sms_messages`,
|
||
/// `get_file_tags`, `reverse_geocode`, `get_current_datetime`, the
|
||
/// four knowledge-memory tools. Conditional: `describe_photo` (vision
|
||
/// model), `get_personal_place_at` (Apollo configured), `search_rag`
|
||
/// (daily_summaries populated), `get_calendar_events` (calendar
|
||
/// populated), `get_location_history` (location history populated).
|
||
pub(crate) fn build_tool_definitions(opts: ToolGateOpts) -> Vec<Tool> {
|
||
let mut tools: Vec<Tool> = Vec::new();
|
||
|
||
if opts.daily_summaries_present {
|
||
tools.push(Tool::function(
|
||
"search_rag",
|
||
"Date-anchored semantic search over the user's daily-summary corpus. \
|
||
Returns up to `limit` summaries most semantically similar to `query`, \
|
||
weighted toward summaries near `date`. For raw message text across all \
|
||
time, prefer `search_messages`. \
|
||
Examples: `{query: \"family dinner\", date: \"2018-12-24\"}` — what \
|
||
daily summaries near Christmas Eve mention family / dinner / gathering. \
|
||
`{query: \"work travel\", date: \"2019-06-15\", contact: \"Alice\"}` — \
|
||
narrowed to summaries that involve Alice.",
|
||
serde_json::json!({
|
||
"type": "object",
|
||
"required": ["query", "date"],
|
||
"properties": {
|
||
"query": { "type": "string", "description": "Free-text query, semantically matched." },
|
||
"date": { "type": "string", "description": "Anchor date, YYYY-MM-DD. Summaries near this date rank higher." },
|
||
"contact": { "type": "string", "description": "Optional contact name to bias toward conversations with that person." },
|
||
"limit": { "type": "integer", "description": "Max summaries to return (default 10, max 25)." }
|
||
}
|
||
}),
|
||
));
|
||
}
|
||
|
||
tools.push(Tool::function(
|
||
"search_messages",
|
||
"Search SMS/MMS message bodies. Modes: `fts5` (keyword + phrase + prefix + AND/OR/NOT + NEAR proximity), \
|
||
`semantic` (embedding similarity, requires generated embeddings), `hybrid` (RRF merge, recommended; \
|
||
degrades to fts5 when embeddings absent). Optional `start_ts` / `end_ts` (real-UTC unix seconds) and \
|
||
`contact_id` filters. For pure date / contact browsing without keywords, prefer `get_sms_messages`. \
|
||
Examples: `{query: \"trader joe's\"}` — phrase across all time. \
|
||
`{query: \"dinner\", contact_id: 42, start_ts: 1700000000, end_ts: 1700604800}` — keyword within a contact and a week. \
|
||
`{query: \"NEAR(meeting work, 5)\"}` — proximity search.",
|
||
serde_json::json!({
|
||
"type": "object",
|
||
"required": ["query"],
|
||
"properties": {
|
||
"query": { "type": "string", "description": "Search query. Min 3 chars. fts5 supports phrase (\"\"), prefix (*), AND/OR/NOT, and NEAR proximity." },
|
||
"mode": { "type": "string", "enum": ["fts5", "semantic", "hybrid"], "description": "Search strategy. Default: hybrid." },
|
||
"limit": { "type": "integer", "description": "Max results (default 20, max 50)." },
|
||
"contact_id": { "type": "integer", "description": "Optional numeric contact id to scope the search." },
|
||
"start_ts": { "type": "integer", "description": "Optional inclusive lower bound, real-UTC unix seconds." },
|
||
"end_ts": { "type": "integer", "description": "Optional inclusive upper bound, real-UTC unix seconds." }
|
||
}
|
||
}),
|
||
));
|
||
|
||
tools.push(Tool::function(
|
||
"get_sms_messages",
|
||
"Fetch SMS/MMS messages near a date (and optionally from a specific contact). Use when you know the date \
|
||
or want context around a photo's timestamp. For keyword search without a date, use `search_messages`. \
|
||
Returns up to `limit` messages within `±days_radius` of `date`, sorted by proximity. \
|
||
Example: `{date: \"2018-08-12\", contact: \"Mom\", days_radius: 2}` — messages from Mom within ±2 days of Aug 12 2018.",
|
||
serde_json::json!({
|
||
"type": "object",
|
||
"required": ["date"],
|
||
"properties": {
|
||
"date": { "type": "string", "description": "Center date, YYYY-MM-DD." },
|
||
"contact": { "type": "string", "description": "Optional contact name (case-insensitive). Falls back to all contacts on no match." },
|
||
"days_radius": { "type": "integer", "description": "Days before and after to include (default 4)." },
|
||
"limit": { "type": "integer", "description": "Max messages to return (default 60, max 150)." }
|
||
}
|
||
}),
|
||
));
|
||
|
||
if opts.calendar_present {
|
||
tools.push(Tool::function(
|
||
"get_calendar_events",
|
||
"Fetch calendar events near a date — meetings, scheduled activities, all-day events. \
|
||
Returns events within `±days_radius` of `date`. \
|
||
Example: `{date: \"2019-03-22\", days_radius: 3}` — events within a week of March 22 2019.",
|
||
serde_json::json!({
|
||
"type": "object",
|
||
"required": ["date"],
|
||
"properties": {
|
||
"date": { "type": "string", "description": "Center date, YYYY-MM-DD." },
|
||
"days_radius": { "type": "integer", "description": "Days before and after to include (default 7)." },
|
||
"limit": { "type": "integer", "description": "Max events to return (default 20, max 50)." }
|
||
}
|
||
}),
|
||
));
|
||
}
|
||
|
||
if opts.location_history_present {
|
||
tools.push(Tool::function(
|
||
"get_location_history",
|
||
"Fetch raw location records (lat/lon/timestamp/activity) near a date. The default 14-day radius is \
|
||
wide because location density varies; tighten to ±1 day for a single-trip query. For a coordinate's \
|
||
named place, use `reverse_geocode` (or `get_personal_place_at` when Apollo is enabled).",
|
||
serde_json::json!({
|
||
"type": "object",
|
||
"required": ["date"],
|
||
"properties": {
|
||
"date": { "type": "string", "description": "Center date, YYYY-MM-DD." },
|
||
"days_radius": { "type": "integer", "description": "Days before and after to include (default 14)." }
|
||
}
|
||
}),
|
||
));
|
||
}
|
||
|
||
tools.push(Tool::function(
|
||
"get_file_tags",
|
||
"Get user-applied tags for a specific photo file path. Tags are user-curated, not auto-detected.",
|
||
serde_json::json!({
|
||
"type": "object",
|
||
"required": ["file_path"],
|
||
"properties": {
|
||
"file_path": { "type": "string", "description": "File path of the photo." }
|
||
}
|
||
}),
|
||
));
|
||
|
||
tools.push(Tool::function(
|
||
"reverse_geocode",
|
||
"Convert GPS lat/lon to a human-readable place name (city, state). Use for any coordinate the LLM has \
|
||
obtained from EXIF or `get_location_history`. When Apollo is configured, prefer `get_personal_place_at` \
|
||
— it returns the user's named places (Home / Work / etc.) which are more specific.",
|
||
serde_json::json!({
|
||
"type": "object",
|
||
"required": ["latitude", "longitude"],
|
||
"properties": {
|
||
"latitude": { "type": "number", "description": "Decimal degrees." },
|
||
"longitude": { "type": "number", "description": "Decimal degrees." }
|
||
}
|
||
}),
|
||
));
|
||
|
||
if opts.apollo_enabled {
|
||
tools.push(Tool::function(
|
||
"get_personal_place_at",
|
||
"Return any of the user's named Places (e.g. Home, Work, Cabin) whose radius contains (latitude, longitude). \
|
||
Smallest radius first — most specific match wins. More specific than `reverse_geocode`; prefer this when \
|
||
both apply. Returns place name, category, free-text description, and radius.",
|
||
serde_json::json!({
|
||
"type": "object",
|
||
"required": ["latitude", "longitude"],
|
||
"properties": {
|
||
"latitude": { "type": "number", "description": "Decimal degrees." },
|
||
"longitude": { "type": "number", "description": "Decimal degrees." }
|
||
}
|
||
}),
|
||
));
|
||
}
|
||
|
||
if opts.faces_present {
|
||
tools.push(Tool::function(
|
||
"get_faces_in_photo",
|
||
"Return the faces detected in this photo with their bounding boxes and assigned person names \
|
||
(when bound). Each face carries `person_name` (string or null), `bbox` ({x, y, w, h} normalized 0–1), \
|
||
`confidence` (0–1), and `source` ('auto' from detector or 'manual' from a user-drawn bbox). \
|
||
More authoritative than `get_file_tags` for counting people in a photo or naming who is present, \
|
||
since it returns detected-but-unbound faces too. \
|
||
Example: `{file_path: \"2019/06/IMG_4242.jpg\"}`.",
|
||
serde_json::json!({
|
||
"type": "object",
|
||
"required": ["file_path"],
|
||
"properties": {
|
||
"file_path": { "type": "string", "description": "File path of the photo." }
|
||
}
|
||
}),
|
||
));
|
||
}
|
||
|
||
tools.push(Tool::function(
|
||
"recall_entities",
|
||
"Search the persistent knowledge memory for previously learned people, places, events, or things. \
|
||
Use BEFORE writing the insight to ground the model on what's already known.",
|
||
serde_json::json!({
|
||
"type": "object",
|
||
"properties": {
|
||
"name": { "type": "string", "description": "Name or partial name (case-insensitive substring match)." },
|
||
"entity_type": { "type": "string", "enum": ["person", "place", "event", "thing"] },
|
||
"limit": { "type": "integer", "description": "Max results (default 20, max 50)." }
|
||
}
|
||
}),
|
||
));
|
||
|
||
tools.push(Tool::function(
|
||
"recall_facts_for_photo",
|
||
"Retrieve all stored facts linked to a specific photo. Call at the start of insight generation to load \
|
||
prior knowledge about subjects in this photo without scanning the whole knowledge base.",
|
||
serde_json::json!({
|
||
"type": "object",
|
||
"required": ["file_path"],
|
||
"properties": {
|
||
"file_path": { "type": "string", "description": "File path of the photo." }
|
||
}
|
||
}),
|
||
));
|
||
|
||
tools.push(Tool::function(
|
||
"store_entity",
|
||
"Upsert a person / place / event / thing into the knowledge memory. Returns the entity id (use it as \
|
||
`subject_entity_id` or `object_entity_id` in `store_fact`). Idempotent on canonical name.",
|
||
serde_json::json!({
|
||
"type": "object",
|
||
"required": ["name", "entity_type"],
|
||
"properties": {
|
||
"name": { "type": "string", "description": "Canonical name (e.g. \"John Smith\", \"Banff National Park\")." },
|
||
"entity_type": { "type": "string", "enum": ["person", "place", "event", "thing"] },
|
||
"description": { "type": "string", "description": "Brief description." }
|
||
}
|
||
}),
|
||
));
|
||
|
||
tools.push(Tool::function(
|
||
"store_fact",
|
||
"Record a fact about an entity in the knowledge memory. Always linked to the current photo. \
|
||
You must provide EITHER `object_entity_id` (when the object is itself a stored entity — e.g. \
|
||
person A is_friend_of person B) OR `object_value` (free-text attribute — e.g. role=\"software engineer\"). \
|
||
`object_entity_id` takes precedence when both are present. \
|
||
Examples: \
|
||
`{subject_entity_id: 7, predicate: \"is_friend_of\", object_entity_id: 12}` — links two known entities. \
|
||
`{subject_entity_id: 7, predicate: \"lives_in\", object_value: \"Portland, Oregon\"}` — free-text attribute.",
|
||
serde_json::json!({
|
||
"type": "object",
|
||
"required": ["subject_entity_id", "predicate"],
|
||
"properties": {
|
||
"subject_entity_id": { "type": "integer", "description": "Entity id this fact is about." },
|
||
"predicate": { "type": "string", "description": "Relationship or attribute (e.g. is_friend_of, located_in, attended_event)." },
|
||
"object_entity_id": { "type": "integer", "description": "Use when the object is itself a stored entity. Takes precedence over object_value." },
|
||
"object_value": { "type": "string", "description": "Use for free-text attributes where the object is not a stored entity." },
|
||
"photo_role": { "type": "string", "description": "How this entity appears in the photo (default \"subject\")." }
|
||
}
|
||
}),
|
||
));
|
||
|
||
tools.push(Tool::function(
|
||
"get_current_datetime",
|
||
"Get the current date and time. Useful when reasoning about how long ago a photo was taken.",
|
||
serde_json::json!({
|
||
"type": "object",
|
||
"properties": {}
|
||
}),
|
||
));
|
||
|
||
if opts.has_vision {
|
||
tools.push(Tool::function(
|
||
"describe_photo",
|
||
"Generate a visual description of the current photo — people, location, objects, activity visible \
|
||
in the image. Only available with vision-capable models.",
|
||
serde_json::json!({
|
||
"type": "object",
|
||
"properties": {}
|
||
}),
|
||
));
|
||
}
|
||
|
||
tools
|
||
}
|
||
|
||
/// Generate an AI insight for a photo using an agentic tool-calling loop.
|
||
/// The model decides which tools to call to gather context before writing the final insight.
|
||
///
|
||
/// `backend` selects the chat provider: `"local"` (default) routes the
|
||
/// agentic loop through the configured Ollama server with the image
|
||
/// attached to the first user message; `"hybrid"` asks the local Ollama
|
||
/// vision model to describe the image once, inlines the description as
|
||
/// text, and runs the loop through OpenRouter (chat only — embeddings
|
||
/// and describe calls stay local in either mode).
|
||
#[allow(clippy::too_many_arguments)]
|
||
/// Render a set of prior-conversation transcripts into a compact
|
||
/// trajectory block for inclusion in the system prompt. Tool results
|
||
/// are summarised to one line each so the prompt stays small.
|
||
fn render_fewshot_examples(examples: &[Vec<ChatMessage>]) -> String {
|
||
if examples.is_empty() {
|
||
return String::new();
|
||
}
|
||
|
||
let mut out = String::from("## Examples of strong context-gathering\n\n");
|
||
out.push_str(
|
||
"The following are compressed trajectories from prior high-quality insights. \
|
||
They show the *pattern* of tool use, not answers to copy.\n\n",
|
||
);
|
||
|
||
for (i, msgs) in examples.iter().enumerate() {
|
||
out.push_str(&format!("### Example {}\n\n", i + 1));
|
||
out.push_str(&Self::render_single_trajectory(msgs));
|
||
out.push('\n');
|
||
}
|
||
|
||
out.push_str("---\n\n");
|
||
out
|
||
}
|
||
|
||
fn render_single_trajectory(msgs: &[ChatMessage]) -> String {
|
||
let mut out = String::new();
|
||
|
||
if let Some(first_user) = msgs.iter().find(|m| m.role == "user") {
|
||
let trimmed = first_user
|
||
.content
|
||
.lines()
|
||
.filter(|l| !l.trim().is_empty())
|
||
.take(8)
|
||
.collect::<Vec<_>>()
|
||
.join("\n");
|
||
out.push_str(&format!("Input:\n{}\n\n", trimmed));
|
||
}
|
||
|
||
out.push_str("Trajectory:\n");
|
||
let mut step = 1;
|
||
let mut final_content: Option<String> = None;
|
||
|
||
for (i, m) in msgs.iter().enumerate() {
|
||
if m.role != "assistant" {
|
||
continue;
|
||
}
|
||
if let Some(ref calls) = m.tool_calls {
|
||
for call in calls {
|
||
let args_brief = Self::brief_json_args(&call.function.arguments);
|
||
let result_summary = msgs
|
||
.get(i + 1)
|
||
.filter(|r| r.role == "tool")
|
||
.map(|r| Self::summarize_tool_result(&call.function.name, &r.content))
|
||
.unwrap_or_else(|| "(no result)".to_string());
|
||
out.push_str(&format!(
|
||
"{}. {}({}) -> {}\n",
|
||
step, call.function.name, args_brief, result_summary
|
||
));
|
||
step += 1;
|
||
}
|
||
} else if !m.content.is_empty() {
|
||
final_content = Some(m.content.clone());
|
||
}
|
||
}
|
||
|
||
if let Some(content) = final_content {
|
||
let short: String = content.chars().take(240).collect();
|
||
out.push_str(&format!("\nFinal insight: {}...\n", short));
|
||
}
|
||
|
||
out
|
||
}
|
||
|
||
fn brief_json_args(v: &serde_json::Value) -> String {
|
||
let Some(obj) = v.as_object() else {
|
||
return v.to_string();
|
||
};
|
||
obj.iter()
|
||
.map(|(k, v)| {
|
||
let rendered = match v {
|
||
serde_json::Value::String(s) if s.len() > 40 => {
|
||
format!("\"{}...\"", &s[..40])
|
||
}
|
||
_ => v.to_string(),
|
||
};
|
||
format!("{}={}", k, rendered)
|
||
})
|
||
.collect::<Vec<_>>()
|
||
.join(", ")
|
||
}
|
||
|
||
/// Collapse a raw tool-result string (the text the model saw) into a
|
||
/// short phrase suitable for a few-shot trajectory. Detects the
|
||
/// "Found N ...", "No ...", and "Error ..." idioms used by the tool
|
||
/// implementations in this file. Unknown shapes fall back to a char
|
||
/// count, which is deliberately visible so drift shows up in output.
|
||
fn summarize_tool_result(tool_name: &str, raw: &str) -> String {
|
||
if raw.starts_with("Error ") {
|
||
return "error".to_string();
|
||
}
|
||
if raw.starts_with("No ") || raw.starts_with("Could not ") {
|
||
return "empty (pivoted)".to_string();
|
||
}
|
||
|
||
if let Some(rest) = raw.strip_prefix("Found ")
|
||
&& let Some(n_str) = rest.split_whitespace().next()
|
||
&& let Ok(n) = n_str.parse::<usize>()
|
||
{
|
||
let kind = match tool_name {
|
||
"search_messages" | "get_sms_messages" => "messages",
|
||
"get_calendar_events" => "events",
|
||
"get_location_history" => "location records",
|
||
_ => "results",
|
||
};
|
||
return format!("{} {}", n, kind);
|
||
}
|
||
|
||
match tool_name {
|
||
"search_rag" => {
|
||
let n = raw.split("\n\n").filter(|s| !s.trim().is_empty()).count();
|
||
format!("{} rag hits", n)
|
||
}
|
||
"get_file_tags" => {
|
||
let n = raw.split(',').filter(|s| !s.trim().is_empty()).count();
|
||
format!("{} tags", n)
|
||
}
|
||
"describe_photo" => {
|
||
let short: String = raw.chars().take(80).collect();
|
||
format!("described: \"{}...\"", short)
|
||
}
|
||
"reverse_geocode" => {
|
||
let short: String = raw.chars().take(60).collect();
|
||
format!("place: {}", short)
|
||
}
|
||
"get_personal_place_at" => {
|
||
if raw.starts_with("No personal place") || raw.starts_with("Personal place lookup")
|
||
{
|
||
"no personal place".to_string()
|
||
} else {
|
||
let n = raw.lines().filter(|l| l.starts_with("- ")).count().max(1);
|
||
format!("{} personal place(s)", n)
|
||
}
|
||
}
|
||
"recall_entities" | "recall_facts_for_photo" => {
|
||
let n = raw.lines().skip(1).filter(|l| !l.trim().is_empty()).count();
|
||
let kind = if tool_name == "recall_entities" {
|
||
"entities"
|
||
} else {
|
||
"facts"
|
||
};
|
||
format!("{} {}", n, kind)
|
||
}
|
||
"store_entity" | "store_fact" => raw
|
||
.split_whitespace()
|
||
.find_map(|tok| tok.strip_prefix("ID:"))
|
||
.map(|id| format!("stored id={}", id.trim_end_matches(',')))
|
||
.unwrap_or_else(|| "stored".to_string()),
|
||
"get_current_datetime" => "time noted".to_string(),
|
||
_ => format!("{} chars", raw.len()),
|
||
}
|
||
}
|
||
|
||
/// Assemble the chat system prompt from two named blocks:
|
||
///
|
||
/// 1. **Identity / voice / format** — `custom_system_prompt` verbatim
|
||
/// when supplied, or a neutral default that doesn't fight a future
|
||
/// persona. The framework never asserts an identity that could
|
||
/// contradict the persona.
|
||
/// 2. **Procedural scaffolding** — tool-use guidance, iteration budget,
|
||
/// contact-filter rule. Identity-free; never asserts voice or shape.
|
||
///
|
||
/// `owner_id_note` and `fewshot_block` are pre-rendered strings (they
|
||
/// already encode their own headers / blank lines). Pass empty / None
|
||
/// to skip.
|
||
pub(crate) fn build_system_content(
|
||
custom_system_prompt: Option<&str>,
|
||
owner_id_note: Option<&str>,
|
||
fewshot_block: &str,
|
||
max_iterations: usize,
|
||
) -> String {
|
||
let identity = match custom_system_prompt {
|
||
Some(s) if !s.trim().is_empty() => s.trim().to_string(),
|
||
_ => String::from(
|
||
"You are reconstructing a memory from a photo. Use the gathered \
|
||
context to write a thoughtful summary; you decide voice, length, and shape.",
|
||
),
|
||
};
|
||
|
||
let owner = owner_id_note.unwrap_or("");
|
||
|
||
let procedural = format!(
|
||
"Tool-use guidance:\n\
|
||
- You have a budget of {max_iterations} tool-calling iterations.\n\
|
||
- Call tools to gather context BEFORE writing your final answer; don't answer after one or two calls.\n\
|
||
- When calling get_sms_messages or search_rag, make at least one call WITHOUT a contact filter \
|
||
— surrounding events matter even when a contact is known.\n\
|
||
- Use recall_facts_for_photo + recall_entities to load any prior knowledge about subjects in the photo.\n\
|
||
- When you identify people / places / events / things, use store_entity + store_fact to grow the persistent memory.\n\
|
||
- A tool returning no results is informative; continue with the others.",
|
||
);
|
||
|
||
let mut out = identity;
|
||
if !owner.is_empty() {
|
||
out.push_str(owner);
|
||
}
|
||
out.push_str("\n\n");
|
||
if !fewshot_block.is_empty() {
|
||
out.push_str(fewshot_block);
|
||
}
|
||
out.push_str(&procedural);
|
||
out
|
||
}
|
||
|
||
pub async fn generate_agentic_insight_for_photo(
|
||
&self,
|
||
file_path: &str,
|
||
custom_model: Option<String>,
|
||
custom_system_prompt: Option<String>,
|
||
num_ctx: Option<i32>,
|
||
temperature: Option<f32>,
|
||
top_p: Option<f32>,
|
||
top_k: Option<i32>,
|
||
min_p: Option<f32>,
|
||
max_iterations: usize,
|
||
backend: Option<String>,
|
||
fewshot_examples: Vec<Vec<ChatMessage>>,
|
||
fewshot_source_ids: Vec<i32>,
|
||
) -> Result<(Option<i32>, Option<i32>)> {
|
||
let tracer = global_tracer();
|
||
let current_cx = opentelemetry::Context::current();
|
||
let mut span = tracer.start_with_context("ai.insight.generate_agentic", ¤t_cx);
|
||
|
||
let file_path = normalize_path(file_path);
|
||
log::info!("Generating agentic insight for photo: {}", file_path);
|
||
|
||
span.set_attribute(KeyValue::new("file_path", file_path.clone()));
|
||
span.set_attribute(KeyValue::new("max_iterations", max_iterations as i64));
|
||
|
||
// 1a. Resolve backend label (defaults to "local").
|
||
let backend_label = backend
|
||
.as_deref()
|
||
.map(|s| s.trim().to_lowercase())
|
||
.filter(|s| !s.is_empty())
|
||
.unwrap_or_else(|| "local".to_string());
|
||
if !matches!(backend_label.as_str(), "local" | "hybrid") {
|
||
return Err(anyhow::anyhow!(
|
||
"unknown backend '{}'; expected 'local' or 'hybrid'",
|
||
backend_label
|
||
));
|
||
}
|
||
span.set_attribute(KeyValue::new("backend", backend_label.clone()));
|
||
let is_hybrid = backend_label == "hybrid";
|
||
|
||
// 1b. Always build an Ollama client. In local mode it owns the chat
|
||
// loop; in hybrid mode it still handles describe_image + any
|
||
// tool-local calls (e.g. if a future tool needs embeddings).
|
||
// Sampling overrides only apply in local mode — in hybrid the
|
||
// user's params belong to the OpenRouter chat client.
|
||
let apply_sampling_to_ollama = !is_hybrid;
|
||
let mut ollama_client = if let Some(ref model) = custom_model
|
||
&& !is_hybrid
|
||
{
|
||
log::info!("Using custom model for agentic: {}", model);
|
||
span.set_attribute(KeyValue::new("custom_model", model.clone()));
|
||
OllamaClient::new(
|
||
self.ollama.primary_url.clone(),
|
||
self.ollama.fallback_url.clone(),
|
||
model.clone(),
|
||
Some(model.clone()),
|
||
)
|
||
} else {
|
||
if !is_hybrid {
|
||
span.set_attribute(KeyValue::new("model", self.ollama.primary_model.clone()));
|
||
}
|
||
self.ollama.clone()
|
||
};
|
||
|
||
if apply_sampling_to_ollama {
|
||
if let Some(ctx) = num_ctx {
|
||
log::info!("Using custom context size: {}", ctx);
|
||
span.set_attribute(KeyValue::new("num_ctx", ctx as i64));
|
||
ollama_client.set_num_ctx(Some(ctx));
|
||
}
|
||
|
||
if temperature.is_some() || top_p.is_some() || top_k.is_some() || min_p.is_some() {
|
||
log::info!(
|
||
"Using sampling params — temperature: {:?}, top_p: {:?}, top_k: {:?}, min_p: {:?}",
|
||
temperature,
|
||
top_p,
|
||
top_k,
|
||
min_p
|
||
);
|
||
if let Some(t) = temperature {
|
||
span.set_attribute(KeyValue::new("temperature", t as f64));
|
||
}
|
||
if let Some(p) = top_p {
|
||
span.set_attribute(KeyValue::new("top_p", p as f64));
|
||
}
|
||
if let Some(k) = top_k {
|
||
span.set_attribute(KeyValue::new("top_k", k as i64));
|
||
}
|
||
if let Some(m) = min_p {
|
||
span.set_attribute(KeyValue::new("min_p", m as f64));
|
||
}
|
||
ollama_client.set_sampling_params(temperature, top_p, top_k, min_p);
|
||
}
|
||
}
|
||
|
||
// 1c. In hybrid mode, clone the configured OpenRouter client and
|
||
// apply per-request overrides.
|
||
let openrouter_client: Option<OpenRouterClient> = if is_hybrid {
|
||
let arc = self.openrouter.as_ref().ok_or_else(|| {
|
||
anyhow::anyhow!("hybrid backend unavailable: OPENROUTER_API_KEY not configured")
|
||
})?;
|
||
let mut c: OpenRouterClient = (**arc).clone();
|
||
if let Some(ref m) = custom_model {
|
||
c.primary_model = m.clone();
|
||
span.set_attribute(KeyValue::new("custom_model", m.clone()));
|
||
}
|
||
span.set_attribute(KeyValue::new("openrouter_model", c.primary_model.clone()));
|
||
if temperature.is_some() || top_p.is_some() || top_k.is_some() || min_p.is_some() {
|
||
if let Some(t) = temperature {
|
||
span.set_attribute(KeyValue::new("temperature", t as f64));
|
||
}
|
||
if let Some(p) = top_p {
|
||
span.set_attribute(KeyValue::new("top_p", p as f64));
|
||
}
|
||
if let Some(k) = top_k {
|
||
span.set_attribute(KeyValue::new("top_k", k as i64));
|
||
}
|
||
if let Some(m) = min_p {
|
||
span.set_attribute(KeyValue::new("min_p", m as f64));
|
||
}
|
||
c.set_sampling_params(temperature, top_p, top_k, min_p);
|
||
}
|
||
if let Some(ctx) = num_ctx {
|
||
span.set_attribute(KeyValue::new("num_ctx", ctx as i64));
|
||
c.set_num_ctx(Some(ctx));
|
||
}
|
||
Some(c)
|
||
} else {
|
||
None
|
||
};
|
||
|
||
let insight_cx = current_cx.with_span(span);
|
||
|
||
// 2. Verify chat model supports tool calling.
|
||
// - local: existing Ollama model availability + capability check.
|
||
// - hybrid: trust the operator's curated allowlist
|
||
// (OPENROUTER_ALLOWED_MODELS) — no live precheck. A bad model id
|
||
// surfaces as a chat-call error on the next step.
|
||
let has_vision = if is_hybrid {
|
||
// In hybrid mode the chat model never sees images directly — we
|
||
// describe-then-inject, so `has_vision` drives only whether we
|
||
// bother loading the image to describe it, which we always do.
|
||
true
|
||
} else {
|
||
if let Some(ref model_name) = custom_model {
|
||
let available_on_primary =
|
||
OllamaClient::is_model_available(&ollama_client.primary_url, model_name)
|
||
.await
|
||
.unwrap_or(false);
|
||
|
||
let available_on_fallback =
|
||
if let Some(ref fallback_url) = ollama_client.fallback_url {
|
||
OllamaClient::is_model_available(fallback_url, model_name)
|
||
.await
|
||
.unwrap_or(false)
|
||
} else {
|
||
false
|
||
};
|
||
|
||
if !available_on_primary && !available_on_fallback {
|
||
anyhow::bail!(
|
||
"model not available: '{}' not found on any configured server",
|
||
model_name
|
||
);
|
||
}
|
||
}
|
||
|
||
let model_name_for_caps = &ollama_client.primary_model;
|
||
let capabilities = match OllamaClient::check_model_capabilities(
|
||
&ollama_client.primary_url,
|
||
model_name_for_caps,
|
||
)
|
||
.await
|
||
{
|
||
Ok(caps) => caps,
|
||
Err(_) => {
|
||
let fallback_url = ollama_client.fallback_url.as_deref().ok_or_else(|| {
|
||
anyhow::anyhow!(
|
||
"Failed to check model capabilities for '{}': model not found on primary server and no fallback configured",
|
||
model_name_for_caps
|
||
)
|
||
})?;
|
||
OllamaClient::check_model_capabilities(fallback_url, model_name_for_caps)
|
||
.await
|
||
.map_err(|e| {
|
||
anyhow::anyhow!(
|
||
"Failed to check model capabilities for '{}': {}",
|
||
model_name_for_caps,
|
||
e
|
||
)
|
||
})?
|
||
}
|
||
};
|
||
|
||
if !capabilities.has_tool_calling {
|
||
return Err(anyhow::anyhow!(
|
||
"tool calling not supported by model '{}'",
|
||
ollama_client.primary_model
|
||
));
|
||
}
|
||
|
||
insight_cx
|
||
.span()
|
||
.set_attribute(KeyValue::new("model_has_vision", capabilities.has_vision));
|
||
insight_cx
|
||
.span()
|
||
.set_attribute(KeyValue::new("model_has_tool_calling", true));
|
||
|
||
capabilities.has_vision
|
||
};
|
||
|
||
// 3. Fetch EXIF
|
||
let exif = {
|
||
let mut exif_dao = self.exif_dao.lock().expect("Unable to lock ExifDao");
|
||
exif_dao
|
||
.get_exif(&insight_cx, &file_path)
|
||
.map_err(|e| anyhow::anyhow!("Failed to get EXIF: {:?}", e))?
|
||
};
|
||
|
||
// 4. Extract timestamp and contact
|
||
let timestamp = if let Some(ts) = exif.as_ref().and_then(|e| e.date_taken) {
|
||
ts
|
||
} else {
|
||
log::warn!("No date_taken in EXIF for {}, trying filename", file_path);
|
||
extract_date_from_filename(&file_path)
|
||
.map(|dt| dt.timestamp())
|
||
.or_else(|| {
|
||
let full_path = self.resolve_full_path(&file_path)?;
|
||
File::open(&full_path)
|
||
.and_then(|f| f.metadata())
|
||
.inspect_err(|e| {
|
||
log::warn!(
|
||
"Failed to get file timestamp for agentic insight {}: {}",
|
||
file_path,
|
||
e
|
||
)
|
||
})
|
||
.ok()
|
||
.and_then(|m| earliest_fs_time(&m))
|
||
.map(|t| DateTime::<Utc>::from(t).timestamp())
|
||
})
|
||
.unwrap_or_else(|| Utc::now().timestamp())
|
||
};
|
||
|
||
let date_taken = DateTime::from_timestamp(timestamp, 0)
|
||
.map(|dt| dt.date_naive())
|
||
.unwrap_or_else(|| Utc::now().date_naive());
|
||
|
||
let contact = Self::extract_contact_from_path(&file_path);
|
||
log::info!("Agentic: date_taken={}, contact={:?}", date_taken, contact);
|
||
|
||
// 5. Fetch tags
|
||
let tag_names: Vec<String> = {
|
||
let mut dao = self.tag_dao.lock().expect("Unable to lock TagDao");
|
||
dao.get_tags_for_path(&insight_cx, &file_path)
|
||
.unwrap_or_else(|e| {
|
||
log::warn!("Failed to fetch tags for agentic {}: {}", file_path, e);
|
||
Vec::new()
|
||
})
|
||
.into_iter()
|
||
.map(|t| t.name)
|
||
.collect()
|
||
};
|
||
|
||
// 6. Ensure the owner entity exists so the agent can reference it.
|
||
// Prior entity_photo_links for this file are intentionally preserved
|
||
// across regenerations — clearing them made `recall_facts_for_photo`
|
||
// always return empty and discarded hard-won knowledge. Re-linking
|
||
// the same entity is a no-op (INSERT OR IGNORE).
|
||
let owner_name = user_display_name();
|
||
let owner_entity_id: Option<i32> = {
|
||
let mut kdao = self
|
||
.knowledge_dao
|
||
.lock()
|
||
.expect("Unable to lock KnowledgeDao");
|
||
|
||
// Upsert the owner entity so the agent always has a stable entity ID to reference.
|
||
let owner = crate::database::models::InsertEntity {
|
||
name: owner_name.clone(),
|
||
entity_type: "person".to_string(),
|
||
description: format!(
|
||
"The owner of this photo collection. All memories are written from {}'s perspective.",
|
||
owner_name
|
||
),
|
||
embedding: None,
|
||
confidence: 1.0,
|
||
status: "active".to_string(),
|
||
created_at: Utc::now().timestamp(),
|
||
updated_at: Utc::now().timestamp(),
|
||
};
|
||
match kdao.upsert_entity(&insight_cx, owner) {
|
||
Ok(e) => {
|
||
log::info!("Owner entity '{}' ID: {}", owner_name, e.id);
|
||
Some(e.id)
|
||
}
|
||
Err(e) => {
|
||
log::warn!("Failed to upsert owner entity '{}': {:?}", owner_name, e);
|
||
None
|
||
}
|
||
}
|
||
};
|
||
|
||
// 7. Load image if vision capable.
|
||
// In hybrid mode we ALSO describe it locally now so the
|
||
// description can be inlined as text — the OpenRouter chat model
|
||
// never receives the base64 image directly.
|
||
let image_base64 = if has_vision {
|
||
match self.load_image_as_base64(&file_path) {
|
||
Ok(b64) => {
|
||
log::info!("Loaded image for vision-capable agentic model");
|
||
Some(b64)
|
||
}
|
||
Err(e) => {
|
||
log::warn!("Failed to load image for agentic vision: {}", e);
|
||
None
|
||
}
|
||
}
|
||
} else {
|
||
None
|
||
};
|
||
|
||
let hybrid_visual_description: Option<String> = if is_hybrid {
|
||
match image_base64.as_deref() {
|
||
Some(b64) => match self.ollama.describe_image(b64).await {
|
||
Ok(desc) => {
|
||
log::info!(
|
||
"Hybrid: local vision describe succeeded ({} chars)",
|
||
desc.len()
|
||
);
|
||
Some(desc)
|
||
}
|
||
Err(e) => {
|
||
log::warn!(
|
||
"Hybrid: local vision describe failed, continuing without: {}",
|
||
e
|
||
);
|
||
None
|
||
}
|
||
},
|
||
None => None,
|
||
}
|
||
} else {
|
||
None
|
||
};
|
||
|
||
// 8. Build system message via the two-block helper. Custom prompt
|
||
// (when supplied) is the authoritative identity — the framework
|
||
// never appends a competing "you are a personal photo memory
|
||
// assistant" line. The procedural block stays identity-free.
|
||
let owner_id_note = owner_entity_id.map(|id| {
|
||
format!(
|
||
"\n\nYour identity in the knowledge store: {name} (entity ID: {id}). \
|
||
When storing facts where you ({name}) are the object — for example, someone is your friend, \
|
||
sibling, or colleague — use subject_entity_id for the other person and set object_value to \
|
||
\"{name}\" (or use store_fact with the other person as subject). When storing facts about \
|
||
{name} directly, use {id} as the subject_entity_id.",
|
||
name = owner_name,
|
||
id = id
|
||
)
|
||
});
|
||
let fewshot_block = Self::render_fewshot_examples(&fewshot_examples);
|
||
let system_content = Self::build_system_content(
|
||
custom_system_prompt.as_deref(),
|
||
owner_id_note.as_deref(),
|
||
&fewshot_block,
|
||
max_iterations,
|
||
);
|
||
|
||
// 9. Build user message
|
||
let gps_info = exif
|
||
.as_ref()
|
||
.and_then(|e| {
|
||
if let (Some(lat), Some(lon)) = (e.gps_latitude, e.gps_longitude) {
|
||
Some(format!("GPS: {:.4}, {:.4}", lat, lon))
|
||
} else {
|
||
None
|
||
}
|
||
})
|
||
.unwrap_or_else(|| "GPS: unknown".to_string());
|
||
|
||
let tags_info = if tag_names.is_empty() {
|
||
"Tags: none".to_string()
|
||
} else {
|
||
format!("Tags: {}", tag_names.join(", "))
|
||
};
|
||
|
||
let contact_info = contact
|
||
.as_ref()
|
||
.map(|c| format!("Contact/Person: {}", c))
|
||
.unwrap_or_else(|| "Contact/Person: unknown".to_string());
|
||
|
||
let visual_block = hybrid_visual_description
|
||
.as_deref()
|
||
.map(|d| format!("Visual description (from local vision model):\n{}\n\n", d))
|
||
.unwrap_or_default();
|
||
|
||
// Context-only payload — no output-shape prescription. The persona /
|
||
// custom_system_prompt owns voice, length, and structure. The "title
|
||
// and summary" claim that used to live here was unused (the title is
|
||
// regenerated post-hoc from the summary by generate_photo_title).
|
||
let user_content = format!(
|
||
"{visual_block}Photo file path: {file_path}\n\
|
||
Date taken: {date}\n\
|
||
{contact_info}\n\
|
||
{gps_info}\n\
|
||
{tags_info}\n\n\
|
||
Gather context with the available tools, then respond.",
|
||
date = date_taken.format("%B %d, %Y"),
|
||
);
|
||
|
||
// 10. Define tools. Gate flags computed from current data presence;
|
||
// hybrid mode omits describe_photo since the chat model receives
|
||
// the visual description inline (so we pass `false` for has_vision
|
||
// in hybrid mode regardless of the model's actual capability).
|
||
let gate_opts = self.current_gate_opts(has_vision && !is_hybrid);
|
||
let tools = Self::build_tool_definitions(gate_opts);
|
||
|
||
// 11. Build initial messages. In hybrid mode images are never
|
||
// attached to the wire message — the description is part of
|
||
// `user_content`.
|
||
let system_msg = ChatMessage::system(system_content);
|
||
let mut user_msg = ChatMessage::user(user_content);
|
||
if !is_hybrid && let Some(ref img) = image_base64 {
|
||
user_msg.images = Some(vec![img.clone()]);
|
||
}
|
||
|
||
let mut messages = vec![system_msg, user_msg];
|
||
|
||
// 12. Agentic loop — dispatch through the selected backend.
|
||
let chat_backend: &dyn LlmClient = if let Some(ref or_c) = openrouter_client {
|
||
or_c
|
||
} else {
|
||
&ollama_client
|
||
};
|
||
|
||
let loop_span = tracer.start_with_context("ai.agentic.loop", &insight_cx);
|
||
let loop_cx = insight_cx.with_span(loop_span);
|
||
|
||
let mut final_content = String::new();
|
||
let mut iterations_used = 0usize;
|
||
let mut last_prompt_eval_count: Option<i32> = None;
|
||
let mut last_eval_count: Option<i32> = None;
|
||
|
||
for iteration in 0..max_iterations {
|
||
iterations_used = iteration + 1;
|
||
log::info!("Agentic iteration {}/{}", iteration + 1, max_iterations);
|
||
|
||
let (response, prompt_tokens, eval_tokens) = chat_backend
|
||
.chat_with_tools(messages.clone(), tools.clone())
|
||
.await?;
|
||
|
||
last_prompt_eval_count = prompt_tokens;
|
||
last_eval_count = eval_tokens;
|
||
|
||
// Sanitize tool call arguments before pushing back into history.
|
||
// Some models occasionally return non-object arguments (bool, string, null)
|
||
// which Ollama rejects when they are re-sent in a subsequent request.
|
||
let mut response = response;
|
||
if let Some(ref mut tool_calls) = response.tool_calls {
|
||
for tc in tool_calls.iter_mut() {
|
||
if !tc.function.arguments.is_object() {
|
||
log::warn!(
|
||
"Tool '{}' returned non-object arguments ({:?}), normalising to {{}}",
|
||
tc.function.name,
|
||
tc.function.arguments
|
||
);
|
||
tc.function.arguments = serde_json::Value::Object(Default::default());
|
||
}
|
||
}
|
||
}
|
||
|
||
messages.push(response.clone());
|
||
|
||
if let Some(ref tool_calls) = response.tool_calls
|
||
&& !tool_calls.is_empty()
|
||
{
|
||
for tool_call in tool_calls {
|
||
log::info!(
|
||
"Agentic tool call [{}]: {} {}",
|
||
iteration,
|
||
tool_call.function.name,
|
||
tool_call.function.arguments
|
||
);
|
||
let result = self
|
||
.execute_tool(
|
||
&tool_call.function.name,
|
||
&tool_call.function.arguments,
|
||
&ollama_client,
|
||
&image_base64,
|
||
&file_path,
|
||
&loop_cx,
|
||
)
|
||
.await;
|
||
messages.push(ChatMessage::tool_result(result));
|
||
}
|
||
continue;
|
||
}
|
||
|
||
// No tool calls — this is the final answer
|
||
final_content = response.content;
|
||
break;
|
||
}
|
||
|
||
// If loop exhausted without final answer, ask for one
|
||
if final_content.is_empty() {
|
||
log::info!(
|
||
"Agentic loop exhausted after {} iterations, requesting final answer",
|
||
iterations_used
|
||
);
|
||
messages.push(ChatMessage::user(format!(
|
||
"Based on the context gathered, please write the final photo insight: a title and a detailed personal summary. Write in first person as {}.",
|
||
user_display_name()
|
||
)));
|
||
let (final_response, prompt_tokens, eval_tokens) = chat_backend
|
||
.chat_with_tools(messages.clone(), vec![])
|
||
.await?;
|
||
last_prompt_eval_count = prompt_tokens;
|
||
last_eval_count = eval_tokens;
|
||
final_content = final_response.content.clone();
|
||
messages.push(final_response);
|
||
}
|
||
|
||
loop_cx
|
||
.span()
|
||
.set_attribute(KeyValue::new("iterations_used", iterations_used as i64));
|
||
loop_cx.span().set_status(Status::Ok);
|
||
|
||
// 13. Generate title via the same backend so voice stays consistent.
|
||
let title_prompt = format!(
|
||
"Create a short title (maximum 8 words) for the following journal entry:\n\n{}\n\nCapture the key moment or theme. Return ONLY the title, nothing else.",
|
||
final_content
|
||
);
|
||
let title_system = custom_system_prompt.as_deref().unwrap_or(
|
||
"You are my long term memory assistant. Use only the information provided. Do not invent details.",
|
||
);
|
||
let title_raw = chat_backend
|
||
.generate(&title_prompt, Some(title_system), None)
|
||
.await?;
|
||
let title = title_raw.trim().trim_matches('"').to_string();
|
||
|
||
log::info!("Agentic generated title: {}", title);
|
||
log::info!(
|
||
"Agentic generated summary ({} chars): {}",
|
||
final_content.len(),
|
||
&final_content[..final_content.len().min(200)]
|
||
);
|
||
|
||
// 14. Serialize the full message history for training data
|
||
let training_messages = match serde_json::to_string(&messages) {
|
||
Ok(json) => Some(json),
|
||
Err(e) => {
|
||
log::warn!("Failed to serialize training messages: {}", e);
|
||
None
|
||
}
|
||
};
|
||
|
||
// 15. Store insight (returns the persisted row including its new id)
|
||
let model_version = chat_backend.primary_model().to_string();
|
||
let fewshot_source_ids_json = if fewshot_source_ids.is_empty() {
|
||
None
|
||
} else {
|
||
Some(serde_json::to_string(&fewshot_source_ids).unwrap_or_else(|_| "[]".to_string()))
|
||
};
|
||
let insight = InsertPhotoInsight {
|
||
library_id: crate::libraries::PRIMARY_LIBRARY_ID,
|
||
file_path: file_path.to_string(),
|
||
title,
|
||
summary: final_content,
|
||
generated_at: Utc::now().timestamp(),
|
||
model_version,
|
||
is_current: true,
|
||
training_messages,
|
||
backend: backend_label.clone(),
|
||
fewshot_source_ids: fewshot_source_ids_json,
|
||
content_hash: None,
|
||
};
|
||
|
||
let stored = {
|
||
let mut dao = self.insight_dao.lock().expect("Unable to lock InsightDao");
|
||
dao.store_insight(&insight_cx, insight)
|
||
.map_err(|e| anyhow::anyhow!("Failed to store agentic insight: {:?}", e))
|
||
};
|
||
|
||
match &stored {
|
||
Ok(_) => {
|
||
log::info!("Successfully stored agentic insight for {}", file_path);
|
||
insight_cx.span().set_status(Status::Ok);
|
||
}
|
||
Err(e) => {
|
||
log::error!("Failed to store agentic insight: {:?}", e);
|
||
insight_cx.span().set_status(Status::error(e.to_string()));
|
||
}
|
||
}
|
||
|
||
let stored_insight = stored?;
|
||
|
||
// 16. Backfill source_insight_id on all facts recorded for this photo during the loop
|
||
{
|
||
let mut kdao = self
|
||
.knowledge_dao
|
||
.lock()
|
||
.expect("Unable to lock KnowledgeDao");
|
||
if let Err(e) = kdao.update_facts_insight_id(&insight_cx, &file_path, stored_insight.id)
|
||
{
|
||
log::warn!(
|
||
"Failed to backfill source_insight_id for {}: {:?}",
|
||
file_path,
|
||
e
|
||
);
|
||
}
|
||
}
|
||
|
||
Ok((last_prompt_eval_count, last_eval_count))
|
||
}
|
||
|
||
/// Reverse geocode GPS coordinates to human-readable place names
|
||
async fn reverse_geocode(&self, lat: f64, lon: f64) -> Option<String> {
|
||
let url = format!(
|
||
"https://nominatim.openstreetmap.org/reverse?format=json&lat={}&lon={}",
|
||
lat, lon
|
||
);
|
||
|
||
log::debug!("Reverse geocoding {}, {} via Nominatim", lat, lon);
|
||
|
||
let client = reqwest::Client::new();
|
||
let response = match client
|
||
.get(&url)
|
||
.header("User-Agent", "ImageAPI/1.0") // Nominatim requires User-Agent
|
||
.send()
|
||
.await
|
||
{
|
||
Ok(resp) => resp,
|
||
Err(e) => {
|
||
log::warn!("Geocoding network error for {}, {}: {}", lat, lon, e);
|
||
return None;
|
||
}
|
||
};
|
||
|
||
if !response.status().is_success() {
|
||
log::warn!(
|
||
"Geocoding HTTP error for {}, {}: {}",
|
||
lat,
|
||
lon,
|
||
response.status()
|
||
);
|
||
return None;
|
||
}
|
||
|
||
let data: NominatimResponse = match response.json().await {
|
||
Ok(d) => d,
|
||
Err(e) => {
|
||
log::warn!("Geocoding JSON parse error for {}, {}: {}", lat, lon, e);
|
||
return None;
|
||
}
|
||
};
|
||
|
||
// Try to build a concise location name
|
||
if let Some(addr) = data.address {
|
||
let mut parts = Vec::new();
|
||
|
||
// Prefer city/town/village
|
||
if let Some(city) = addr.city.or(addr.town).or(addr.village) {
|
||
parts.push(city);
|
||
}
|
||
|
||
// Add state if available
|
||
if let Some(state) = addr.state {
|
||
parts.push(state);
|
||
}
|
||
|
||
if !parts.is_empty() {
|
||
log::info!("Reverse geocoded {}, {} -> {}", lat, lon, parts.join(", "));
|
||
return Some(parts.join(", "));
|
||
}
|
||
}
|
||
|
||
// Fallback to display_name if structured address not available
|
||
if let Some(ref display_name) = data.display_name {
|
||
log::info!(
|
||
"Reverse geocoded {}, {} -> {} (display_name)",
|
||
lat,
|
||
lon,
|
||
display_name
|
||
);
|
||
} else {
|
||
log::warn!("Geocoding returned no address data for {}, {}", lat, lon);
|
||
}
|
||
data.display_name
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
use crate::ai::ollama::{ToolCall, ToolCallFunction};
|
||
|
||
#[test]
|
||
fn build_tool_definitions_drops_gated_tools() {
|
||
let opts = ToolGateOpts {
|
||
has_vision: false,
|
||
apollo_enabled: false,
|
||
daily_summaries_present: false,
|
||
calendar_present: false,
|
||
location_history_present: false,
|
||
faces_present: false,
|
||
};
|
||
let tools = InsightGenerator::build_tool_definitions(opts);
|
||
let names: Vec<&str> = tools.iter().map(|t| t.function.name.as_str()).collect();
|
||
|
||
// Always-on tools survive.
|
||
assert!(names.contains(&"search_messages"));
|
||
assert!(names.contains(&"get_sms_messages"));
|
||
assert!(names.contains(&"get_file_tags"));
|
||
assert!(names.contains(&"reverse_geocode"));
|
||
assert!(names.contains(&"get_current_datetime"));
|
||
assert!(names.contains(&"recall_entities"));
|
||
assert!(names.contains(&"recall_facts_for_photo"));
|
||
assert!(names.contains(&"store_entity"));
|
||
assert!(names.contains(&"store_fact"));
|
||
|
||
// Gated tools are absent.
|
||
assert!(!names.contains(&"describe_photo"));
|
||
assert!(!names.contains(&"get_personal_place_at"));
|
||
assert!(!names.contains(&"search_rag"));
|
||
assert!(!names.contains(&"get_calendar_events"));
|
||
assert!(!names.contains(&"get_location_history"));
|
||
assert!(!names.contains(&"get_faces_in_photo"));
|
||
}
|
||
|
||
#[test]
|
||
fn build_tool_definitions_includes_gated_tools_when_present() {
|
||
let opts = ToolGateOpts {
|
||
has_vision: true,
|
||
apollo_enabled: true,
|
||
daily_summaries_present: true,
|
||
calendar_present: true,
|
||
location_history_present: true,
|
||
faces_present: true,
|
||
};
|
||
let tools = InsightGenerator::build_tool_definitions(opts);
|
||
let names: Vec<&str> = tools.iter().map(|t| t.function.name.as_str()).collect();
|
||
assert!(names.contains(&"describe_photo"));
|
||
assert!(names.contains(&"get_personal_place_at"));
|
||
assert!(names.contains(&"search_rag"));
|
||
assert!(names.contains(&"get_calendar_events"));
|
||
assert!(names.contains(&"get_location_history"));
|
||
assert!(names.contains(&"get_faces_in_photo"));
|
||
}
|
||
|
||
fn place(name: &str, description: &str) -> ApolloPlace {
|
||
ApolloPlace {
|
||
id: 1,
|
||
name: name.to_string(),
|
||
description: description.to_string(),
|
||
lat: 0.0,
|
||
lon: 0.0,
|
||
radius_m: 200,
|
||
category: None,
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn compose_location_with_apollo_and_nominatim_and_description() {
|
||
let s = compose_location_string(
|
||
Some(place("Home", "House in Cambridge")),
|
||
Some("Cambridge, MA".to_string()),
|
||
42.36,
|
||
-71.06,
|
||
);
|
||
assert_eq!(s, "Home (House in Cambridge) — near Cambridge, MA");
|
||
}
|
||
|
||
#[test]
|
||
fn compose_location_apollo_without_description_drops_parenthetical() {
|
||
let s = compose_location_string(
|
||
Some(place("Home", "")),
|
||
Some("Cambridge, MA".to_string()),
|
||
42.36,
|
||
-71.06,
|
||
);
|
||
assert_eq!(s, "Home — near Cambridge, MA");
|
||
}
|
||
|
||
#[test]
|
||
fn compose_location_nominatim_only() {
|
||
let s = compose_location_string(None, Some("Cambridge, MA".to_string()), 42.36, -71.06);
|
||
assert_eq!(s, "Cambridge, MA");
|
||
}
|
||
|
||
#[test]
|
||
fn compose_location_apollo_only_no_description() {
|
||
let s = compose_location_string(Some(place("Home", "")), None, 42.36, -71.06);
|
||
assert_eq!(s, "Home");
|
||
}
|
||
|
||
#[test]
|
||
fn compose_location_falls_back_to_coordinates() {
|
||
let s = compose_location_string(None, None, 42.3601, -71.0589);
|
||
assert_eq!(s, "42.3601, -71.0589");
|
||
}
|
||
|
||
#[test]
|
||
fn combine_contexts_includes_tags_section_when_tags_present() {
|
||
let result = InsightGenerator::combine_contexts(
|
||
None,
|
||
None,
|
||
None,
|
||
None,
|
||
Some("vacation, hiking, mountains".to_string()),
|
||
);
|
||
assert!(result.contains("## Tags"), "Should include Tags section");
|
||
assert!(
|
||
result.contains("vacation, hiking, mountains"),
|
||
"Should include tag names"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn combine_contexts_omits_tags_section_when_no_tags() {
|
||
let result = InsightGenerator::combine_contexts(
|
||
Some("some messages".to_string()),
|
||
None,
|
||
None,
|
||
None,
|
||
None, // no tags
|
||
);
|
||
assert!(
|
||
!result.contains("## Tags"),
|
||
"Should not include Tags section when None"
|
||
);
|
||
assert!(
|
||
result.contains("## Messages"),
|
||
"Should still include Messages"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn combine_contexts_returns_no_context_message_when_all_none() {
|
||
let result = InsightGenerator::combine_contexts(None, None, None, None, None);
|
||
assert_eq!(result, "No additional context available");
|
||
}
|
||
|
||
// These tests assert the shape of the strings returned by the tool
|
||
// implementations above. If a tool's output format changes, update the
|
||
// tool AND the corresponding arm of `summarize_tool_result` — these
|
||
// tests exist to make that coupling loud.
|
||
|
||
#[test]
|
||
fn summarize_errors_uniformly() {
|
||
assert_eq!(
|
||
InsightGenerator::summarize_tool_result("search_rag", "Error searching RAG: boom"),
|
||
"error"
|
||
);
|
||
assert_eq!(
|
||
InsightGenerator::summarize_tool_result(
|
||
"get_sms_messages",
|
||
"Error fetching SMS messages: timeout"
|
||
),
|
||
"error"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn summarize_empty_results_uniformly() {
|
||
assert_eq!(
|
||
InsightGenerator::summarize_tool_result("search_rag", "No relevant messages found."),
|
||
"empty (pivoted)"
|
||
);
|
||
assert_eq!(
|
||
InsightGenerator::summarize_tool_result("get_sms_messages", "No messages found."),
|
||
"empty (pivoted)"
|
||
);
|
||
assert_eq!(
|
||
InsightGenerator::summarize_tool_result(
|
||
"reverse_geocode",
|
||
"Could not resolve coordinates to a place name."
|
||
),
|
||
"empty (pivoted)"
|
||
);
|
||
assert_eq!(
|
||
InsightGenerator::summarize_tool_result(
|
||
"recall_facts_for_photo",
|
||
"No knowledge facts found for this photo."
|
||
),
|
||
"empty (pivoted)"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn summarize_found_count_per_tool() {
|
||
assert_eq!(
|
||
InsightGenerator::summarize_tool_result(
|
||
"get_sms_messages",
|
||
"Found 7 messages:\n[2023-08-15 10:00] Sarah: hi"
|
||
),
|
||
"7 messages"
|
||
);
|
||
assert_eq!(
|
||
InsightGenerator::summarize_tool_result(
|
||
"search_messages",
|
||
"Found 3 messages (mode: hybrid):\n\n[2023-08-15] Sarah — hi"
|
||
),
|
||
"3 messages"
|
||
);
|
||
assert_eq!(
|
||
InsightGenerator::summarize_tool_result(
|
||
"get_calendar_events",
|
||
"Found 2 calendar events:\n[2023-08-15 10:00] Wedding"
|
||
),
|
||
"2 events"
|
||
);
|
||
assert_eq!(
|
||
InsightGenerator::summarize_tool_result(
|
||
"get_location_history",
|
||
"Found 5 location records:\n[2023-08-15 10:00] 39.0, -120.0"
|
||
),
|
||
"5 location records"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn summarize_search_rag_counts_hits() {
|
||
let raw = "[2023-08-15] Sarah: venue confirmed\n\n[2023-08-14] Mom: travel plans\n\n[2023-08-13] Dad: weather";
|
||
assert_eq!(
|
||
InsightGenerator::summarize_tool_result("search_rag", raw),
|
||
"3 rag hits"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn summarize_get_file_tags() {
|
||
assert_eq!(
|
||
InsightGenerator::summarize_tool_result("get_file_tags", "wedding, tahoe, 2023"),
|
||
"3 tags"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn summarize_describe_photo_truncates() {
|
||
let raw = "A wedding ceremony at Lake Tahoe with about 40 guests seated in rows facing a lakeside arch decorated with white flowers.";
|
||
let out = InsightGenerator::summarize_tool_result("describe_photo", raw);
|
||
assert!(out.starts_with("described: \""));
|
||
assert!(out.contains("A wedding ceremony at Lake Tahoe"));
|
||
assert!(out.ends_with("...\""));
|
||
}
|
||
|
||
#[test]
|
||
fn summarize_reverse_geocode_returns_place() {
|
||
let out =
|
||
InsightGenerator::summarize_tool_result("reverse_geocode", "South Lake Tahoe, CA, USA");
|
||
assert_eq!(out, "place: South Lake Tahoe, CA, USA");
|
||
}
|
||
|
||
#[test]
|
||
fn summarize_recall_entities_counts_lines() {
|
||
let raw = "Known entities:\n- Sarah (person)\n- Tahoe (place)\n- Wedding 2023 (event)";
|
||
assert_eq!(
|
||
InsightGenerator::summarize_tool_result("recall_entities", raw),
|
||
"3 entities"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn summarize_recall_facts_counts_lines() {
|
||
let raw = "Knowledge for this photo:\n- Sarah: college friend\n- Tahoe: vacation spot";
|
||
assert_eq!(
|
||
InsightGenerator::summarize_tool_result("recall_facts_for_photo", raw),
|
||
"2 facts"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn summarize_store_entity_extracts_id() {
|
||
assert_eq!(
|
||
InsightGenerator::summarize_tool_result(
|
||
"store_entity",
|
||
"Entity stored: ID:42 | person | Sarah | confidence:0.80"
|
||
),
|
||
"stored id=42"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn summarize_store_fact_extracts_id() {
|
||
assert_eq!(
|
||
InsightGenerator::summarize_tool_result(
|
||
"store_fact",
|
||
"Stored new fact: ID:17 | confidence:0.60"
|
||
),
|
||
"stored id=17"
|
||
);
|
||
assert_eq!(
|
||
InsightGenerator::summarize_tool_result(
|
||
"store_fact",
|
||
"Corroborated existing fact: ID:17 | confidence:0.85"
|
||
),
|
||
"stored id=17"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn summarize_current_datetime() {
|
||
assert_eq!(
|
||
InsightGenerator::summarize_tool_result(
|
||
"get_current_datetime",
|
||
"Current date/time: 2024-01-15 12:00:00 PST (Monday)"
|
||
),
|
||
"time noted"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn summarize_unknown_tool_falls_back_to_char_count() {
|
||
let out = InsightGenerator::summarize_tool_result("never_heard_of_it", "some output");
|
||
assert_eq!(out, "11 chars");
|
||
}
|
||
|
||
#[test]
|
||
fn build_system_content_uses_custom_prompt_verbatim_for_identity() {
|
||
let out = InsightGenerator::build_system_content(
|
||
Some("You are a journal writer in first person, warm and reflective."),
|
||
None,
|
||
"",
|
||
6,
|
||
);
|
||
assert!(
|
||
out.starts_with("You are a journal writer in first person, warm and reflective."),
|
||
"custom prompt must lead the system content; got: {}",
|
||
&out[..out.len().min(200)],
|
||
);
|
||
assert!(
|
||
!out.contains("personal photo memory assistant"),
|
||
"framework identity must not leak when custom prompt is supplied"
|
||
);
|
||
assert!(out.contains("Tool-use guidance"));
|
||
assert!(out.contains("budget of 6"));
|
||
}
|
||
|
||
#[test]
|
||
fn build_system_content_uses_neutral_default_when_no_custom() {
|
||
let out = InsightGenerator::build_system_content(None, None, "", 6);
|
||
assert!(out.contains("reconstructing a memory from a photo"));
|
||
assert!(!out.contains("personal photo memory assistant"));
|
||
assert!(out.contains("Tool-use guidance"));
|
||
}
|
||
|
||
#[test]
|
||
fn build_system_content_includes_fewshot_and_owner_id() {
|
||
let owner = "\n\nYour identity in the knowledge store: Alice (entity ID: 7).";
|
||
let fewshot = "## Examples\n\n### Example 1\n...\n\n---\n\n";
|
||
let out = InsightGenerator::build_system_content(None, Some(owner), fewshot, 6);
|
||
assert!(out.contains("Alice (entity ID: 7)"));
|
||
assert!(out.contains("## Examples"));
|
||
}
|
||
|
||
#[test]
|
||
fn render_fewshot_empty_returns_empty_string() {
|
||
assert!(InsightGenerator::render_fewshot_examples(&[]).is_empty());
|
||
}
|
||
|
||
#[test]
|
||
fn render_single_trajectory_walks_tool_calls_in_order() {
|
||
let arguments = serde_json::json!({ "query": "wedding", "date": "2023-08-15" });
|
||
let msgs = vec![
|
||
ChatMessage::system("ignored"),
|
||
ChatMessage::user("Photo file path: /photos/img.jpg\nDate taken: August 15, 2023"),
|
||
ChatMessage {
|
||
role: "assistant".to_string(),
|
||
content: String::new(),
|
||
tool_calls: Some(vec![ToolCall {
|
||
function: ToolCallFunction {
|
||
name: "search_rag".to_string(),
|
||
arguments,
|
||
},
|
||
id: None,
|
||
}]),
|
||
images: None,
|
||
},
|
||
ChatMessage::tool_result("No relevant messages found."),
|
||
ChatMessage {
|
||
role: "assistant".to_string(),
|
||
content: "Final title\n\nFinal body.".to_string(),
|
||
tool_calls: None,
|
||
images: None,
|
||
},
|
||
];
|
||
let out = InsightGenerator::render_single_trajectory(&msgs);
|
||
assert!(out.contains("Input:"));
|
||
assert!(out.contains("/photos/img.jpg"));
|
||
assert!(out.contains("1. search_rag("));
|
||
assert!(out.contains("query=\"wedding\""));
|
||
assert!(out.contains("-> empty (pivoted)"));
|
||
assert!(out.contains("Final insight: Final title"));
|
||
}
|
||
}
|