ffcddbb843
Adds a `libraries` registry table and threads library_id through per-instance metadata tables (image_exif, photo_insights, entity_photo_links, video_preview_clips). File-path columns renamed to rel_path to make the relative-to-root semantics explicit. Adds content_hash + size_bytes on image_exif to support future hash-keyed thumbnail/HLS dedup. Tags and favorites stay library-agnostic so they share across libraries by rel_path. Behavior is unchanged: a single primary library (id=1) is seeded from BASE_PATH on first boot; all handlers and DAOs route through it as a transitional shim until the API gains a library query param. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2916 lines
109 KiB
Rust
2916 lines
109 KiB
Rust
use anyhow::Result;
|
|
use base64::Engine as _;
|
|
use chrono::{DateTime, Local, NaiveDate, Utc};
|
|
use image::ImageFormat;
|
|
use opentelemetry::KeyValue;
|
|
use opentelemetry::trace::{Span, Status, TraceContextExt, Tracer};
|
|
use serde::Deserialize;
|
|
use std::fs::File;
|
|
use std::io::Cursor;
|
|
use std::sync::{Arc, Mutex};
|
|
|
|
use crate::ai::ollama::{ChatMessage, OllamaClient, Tool};
|
|
use crate::ai::sms_client::SmsApiClient;
|
|
use crate::database::models::InsertPhotoInsight;
|
|
use crate::database::{
|
|
CalendarEventDao, DailySummaryDao, ExifDao, InsightDao, KnowledgeDao, LocationHistoryDao,
|
|
SearchHistoryDao,
|
|
};
|
|
use crate::memories::extract_date_from_filename;
|
|
use crate::otel::global_tracer;
|
|
use crate::tags::TagDao;
|
|
use crate::utils::normalize_path;
|
|
|
|
#[derive(Deserialize)]
|
|
struct NominatimResponse {
|
|
display_name: Option<String>,
|
|
address: Option<NominatimAddress>,
|
|
}
|
|
|
|
#[derive(Deserialize)]
|
|
struct NominatimAddress {
|
|
city: Option<String>,
|
|
town: Option<String>,
|
|
village: Option<String>,
|
|
state: Option<String>,
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
pub struct InsightGenerator {
|
|
ollama: OllamaClient,
|
|
sms_client: SmsApiClient,
|
|
insight_dao: Arc<Mutex<Box<dyn InsightDao>>>,
|
|
exif_dao: Arc<Mutex<Box<dyn ExifDao>>>,
|
|
daily_summary_dao: Arc<Mutex<Box<dyn DailySummaryDao>>>,
|
|
|
|
// Google Takeout data sources
|
|
calendar_dao: Arc<Mutex<Box<dyn CalendarEventDao>>>,
|
|
location_dao: Arc<Mutex<Box<dyn LocationHistoryDao>>>,
|
|
search_dao: Arc<Mutex<Box<dyn SearchHistoryDao>>>,
|
|
tag_dao: Arc<Mutex<Box<dyn TagDao>>>,
|
|
|
|
// Knowledge memory
|
|
knowledge_dao: Arc<Mutex<Box<dyn KnowledgeDao>>>,
|
|
|
|
base_path: String,
|
|
}
|
|
|
|
impl InsightGenerator {
|
|
pub fn new(
|
|
ollama: OllamaClient,
|
|
sms_client: SmsApiClient,
|
|
insight_dao: Arc<Mutex<Box<dyn InsightDao>>>,
|
|
exif_dao: Arc<Mutex<Box<dyn ExifDao>>>,
|
|
daily_summary_dao: Arc<Mutex<Box<dyn DailySummaryDao>>>,
|
|
calendar_dao: Arc<Mutex<Box<dyn CalendarEventDao>>>,
|
|
location_dao: Arc<Mutex<Box<dyn LocationHistoryDao>>>,
|
|
search_dao: Arc<Mutex<Box<dyn SearchHistoryDao>>>,
|
|
tag_dao: Arc<Mutex<Box<dyn TagDao>>>,
|
|
knowledge_dao: Arc<Mutex<Box<dyn KnowledgeDao>>>,
|
|
base_path: String,
|
|
) -> Self {
|
|
Self {
|
|
ollama,
|
|
sms_client,
|
|
insight_dao,
|
|
exif_dao,
|
|
daily_summary_dao,
|
|
calendar_dao,
|
|
location_dao,
|
|
search_dao,
|
|
tag_dao,
|
|
knowledge_dao,
|
|
base_path,
|
|
}
|
|
}
|
|
|
|
/// Extract contact name from file path
|
|
/// e.g., "Sarah/img.jpeg" -> Some("Sarah")
|
|
/// e.g., "img.jpeg" -> None
|
|
fn extract_contact_from_path(file_path: &str) -> Option<String> {
|
|
use std::path::Path;
|
|
|
|
let path = Path::new(file_path);
|
|
let components: Vec<_> = path.components().collect();
|
|
|
|
// If path has at least 2 components (directory + file), extract first directory
|
|
if components.len() >= 2
|
|
&& let Some(component) = components.first()
|
|
&& let Some(os_str) = component.as_os_str().to_str()
|
|
{
|
|
return Some(os_str.to_string());
|
|
}
|
|
|
|
None
|
|
}
|
|
|
|
/// Load image file, resize it, and encode as base64 for vision models
|
|
/// Resizes to max 1024px on longest edge to reduce context usage
|
|
fn load_image_as_base64(&self, file_path: &str) -> Result<String> {
|
|
use image::imageops::FilterType;
|
|
use std::path::Path;
|
|
|
|
let full_path = Path::new(&self.base_path).join(file_path);
|
|
|
|
log::debug!("Loading image for vision model: {:?}", full_path);
|
|
|
|
// Open and decode the image
|
|
let img = image::open(&full_path)
|
|
.map_err(|e| anyhow::anyhow!("Failed to open image file: {}", e))?;
|
|
|
|
let (original_width, original_height) = (img.width(), img.height());
|
|
|
|
// Resize to max 1024px on longest edge
|
|
let resized = img.resize(1024, 1024, FilterType::Lanczos3);
|
|
|
|
log::debug!(
|
|
"Resized image from {}x{} to {}x{}",
|
|
original_width,
|
|
original_height,
|
|
resized.width(),
|
|
resized.height()
|
|
);
|
|
|
|
// Encode as JPEG at 85% quality
|
|
let mut buffer = Vec::new();
|
|
let mut cursor = Cursor::new(&mut buffer);
|
|
resized
|
|
.write_to(&mut cursor, ImageFormat::Jpeg)
|
|
.map_err(|e| anyhow::anyhow!("Failed to encode image as JPEG: {}", e))?;
|
|
|
|
let base64_string = base64::engine::general_purpose::STANDARD.encode(&buffer);
|
|
|
|
log::debug!(
|
|
"Encoded image as base64 ({} bytes -> {} chars)",
|
|
buffer.len(),
|
|
base64_string.len()
|
|
);
|
|
|
|
Ok(base64_string)
|
|
}
|
|
|
|
/// Find relevant messages using RAG, excluding recent messages (>30 days ago)
|
|
/// This prevents RAG from returning messages already in the immediate time window
|
|
async fn find_relevant_messages_rag_historical(
|
|
&self,
|
|
parent_cx: &opentelemetry::Context,
|
|
date: chrono::NaiveDate,
|
|
location: Option<&str>,
|
|
contact: Option<&str>,
|
|
topics: Option<&[String]>,
|
|
limit: usize,
|
|
extra_context: Option<&str>,
|
|
) -> Result<Vec<String>> {
|
|
let tracer = global_tracer();
|
|
let span = tracer.start_with_context("ai.rag.filter_historical", parent_cx);
|
|
let filter_cx = parent_cx.with_span(span);
|
|
|
|
filter_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("date", date.to_string()));
|
|
filter_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("limit", limit as i64));
|
|
filter_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("exclusion_window_days", 30));
|
|
if let Some(t) = topics {
|
|
filter_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("topics", t.join(", ")));
|
|
}
|
|
|
|
let query_results = self
|
|
.find_relevant_messages_rag(date, location, contact, topics, limit * 2, extra_context)
|
|
.await?;
|
|
|
|
filter_cx.span().set_attribute(KeyValue::new(
|
|
"rag_results_count",
|
|
query_results.len() as i64,
|
|
));
|
|
|
|
// Filter out messages from within 30 days of the photo date
|
|
let photo_timestamp = date
|
|
.and_hms_opt(12, 0, 0)
|
|
.ok_or_else(|| anyhow::anyhow!("Invalid date"))?
|
|
.and_utc()
|
|
.timestamp();
|
|
let exclusion_window = 30 * 86400; // 30 days in seconds
|
|
|
|
let historical_only: Vec<String> = query_results
|
|
.into_iter()
|
|
.filter(|msg| {
|
|
// Extract date from formatted daily summary "[2024-08-15] Contact ..."
|
|
if let Some(bracket_end) = msg.find(']')
|
|
&& let Some(date_str) = msg.get(1..bracket_end)
|
|
{
|
|
// Parse just the date (daily summaries don't have time)
|
|
if let Ok(msg_date) = chrono::NaiveDate::parse_from_str(date_str, "%Y-%m-%d") {
|
|
let msg_timestamp = msg_date
|
|
.and_hms_opt(12, 0, 0)
|
|
.unwrap()
|
|
.and_utc()
|
|
.timestamp();
|
|
let time_diff = (photo_timestamp - msg_timestamp).abs();
|
|
return time_diff > exclusion_window;
|
|
}
|
|
}
|
|
false
|
|
})
|
|
.take(limit)
|
|
.collect();
|
|
|
|
log::info!(
|
|
"Found {} historical messages (>30 days from photo date)",
|
|
historical_only.len()
|
|
);
|
|
|
|
filter_cx.span().set_attribute(KeyValue::new(
|
|
"historical_results_count",
|
|
historical_only.len() as i64,
|
|
));
|
|
filter_cx.span().set_status(Status::Ok);
|
|
|
|
Ok(historical_only)
|
|
}
|
|
|
|
/// Find relevant daily summaries using RAG (semantic search)
|
|
/// Returns formatted daily summary strings for LLM context
|
|
async fn find_relevant_messages_rag(
|
|
&self,
|
|
date: chrono::NaiveDate,
|
|
location: Option<&str>,
|
|
contact: Option<&str>,
|
|
topics: Option<&[String]>,
|
|
limit: usize,
|
|
extra_context: Option<&str>,
|
|
) -> Result<Vec<String>> {
|
|
let tracer = global_tracer();
|
|
let current_cx = opentelemetry::Context::current();
|
|
let mut span = tracer.start_with_context("ai.rag.search_daily_summaries", ¤t_cx);
|
|
span.set_attribute(KeyValue::new("date", date.to_string()));
|
|
span.set_attribute(KeyValue::new("limit", limit as i64));
|
|
if let Some(loc) = location {
|
|
span.set_attribute(KeyValue::new("location", loc.to_string()));
|
|
}
|
|
if let Some(c) = contact {
|
|
span.set_attribute(KeyValue::new("contact", c.to_string()));
|
|
}
|
|
|
|
// Build query string - prioritize topics if available (semantically meaningful)
|
|
let base_query = if let Some(topics) = topics {
|
|
if !topics.is_empty() {
|
|
// Use topics for semantic search - these are actual content keywords
|
|
let topic_str = topics.join(", ");
|
|
if let Some(c) = contact {
|
|
format!("Conversations about {} with {}", topic_str, c)
|
|
} else {
|
|
format!("Conversations about {}", topic_str)
|
|
}
|
|
} else {
|
|
// Fallback to metadata-based query
|
|
Self::build_metadata_query(date, location, contact)
|
|
}
|
|
} else {
|
|
// Fallback to metadata-based query
|
|
Self::build_metadata_query(date, location, contact)
|
|
};
|
|
|
|
let query = if let Some(extra) = extra_context {
|
|
format!("{}. {}", base_query, extra)
|
|
} else {
|
|
base_query
|
|
};
|
|
|
|
span.set_attribute(KeyValue::new("query", query.clone()));
|
|
|
|
// Create context with this span for child operations
|
|
let search_cx = current_cx.with_span(span);
|
|
|
|
log::info!("========================================");
|
|
log::info!("RAG QUERY: {}", query);
|
|
log::info!("========================================");
|
|
|
|
// Generate embedding for the query
|
|
let query_embedding = self.ollama.generate_embedding(&query).await?;
|
|
|
|
// Search for similar daily summaries with time-based weighting
|
|
// This prioritizes summaries temporally close to the query date
|
|
let mut summary_dao = self
|
|
.daily_summary_dao
|
|
.lock()
|
|
.expect("Unable to lock DailySummaryDao");
|
|
|
|
let date_str = date.format("%Y-%m-%d").to_string();
|
|
let similar_summaries = summary_dao
|
|
.find_similar_summaries_with_time_weight(&search_cx, &query_embedding, &date_str, limit)
|
|
.map_err(|e| anyhow::anyhow!("Failed to find similar summaries: {:?}", e))?;
|
|
|
|
log::info!(
|
|
"Found {} relevant daily summaries via RAG",
|
|
similar_summaries.len()
|
|
);
|
|
|
|
search_cx.span().set_attribute(KeyValue::new(
|
|
"results_count",
|
|
similar_summaries.len() as i64,
|
|
));
|
|
|
|
// Format daily summaries for LLM context
|
|
let formatted = similar_summaries
|
|
.into_iter()
|
|
.map(|s| {
|
|
format!(
|
|
"[{}] {} ({} messages):\n{}",
|
|
s.date, s.contact, s.message_count, s.summary
|
|
)
|
|
})
|
|
.collect();
|
|
|
|
search_cx.span().set_status(Status::Ok);
|
|
|
|
Ok(formatted)
|
|
}
|
|
|
|
/// Build a metadata-based query (fallback when no topics available)
|
|
fn build_metadata_query(
|
|
date: chrono::NaiveDate,
|
|
location: Option<&str>,
|
|
contact: Option<&str>,
|
|
) -> String {
|
|
let mut query_parts = Vec::new();
|
|
|
|
// Add temporal context
|
|
query_parts.push(format!("On {}", date.format("%B %d, %Y")));
|
|
|
|
// Add location if available
|
|
if let Some(loc) = location {
|
|
query_parts.push(format!("at {}", loc));
|
|
}
|
|
|
|
// Add contact context if available
|
|
if let Some(c) = contact {
|
|
query_parts.push(format!("conversation with {}", c));
|
|
}
|
|
|
|
// Add day of week for temporal context
|
|
let weekday = date.format("%A");
|
|
query_parts.push(format!("it was a {}", weekday));
|
|
|
|
query_parts.join(", ")
|
|
}
|
|
|
|
/// Haversine distance calculation for GPS proximity (in kilometers)
|
|
fn haversine_distance(lat1: f64, lon1: f64, lat2: f64, lon2: f64) -> f64 {
|
|
const R: f64 = 6371.0; // Earth radius in km
|
|
let d_lat = (lat2 - lat1).to_radians();
|
|
let d_lon = (lon2 - lon1).to_radians();
|
|
let a = (d_lat / 2.0).sin().powi(2)
|
|
+ lat1.to_radians().cos() * lat2.to_radians().cos() * (d_lon / 2.0).sin().powi(2);
|
|
R * 2.0 * a.sqrt().atan2((1.0 - a).sqrt())
|
|
}
|
|
|
|
/// Gather calendar context for photo timestamp
|
|
/// Uses hybrid time + semantic search (±7 days, ranked by relevance)
|
|
async fn gather_calendar_context(
|
|
&self,
|
|
parent_cx: &opentelemetry::Context,
|
|
timestamp: i64,
|
|
location: Option<&str>,
|
|
) -> Result<Option<String>> {
|
|
let tracer = global_tracer();
|
|
let span = tracer.start_with_context("ai.context.calendar", parent_cx);
|
|
let calendar_cx = parent_cx.with_span(span);
|
|
|
|
let query_embedding = if let Some(loc) = location {
|
|
match self.ollama.generate_embedding(loc).await {
|
|
Ok(emb) => Some(emb),
|
|
Err(e) => {
|
|
log::warn!("Failed to generate embedding for location '{}': {}", loc, e);
|
|
None
|
|
}
|
|
}
|
|
} else {
|
|
None
|
|
};
|
|
|
|
let events = {
|
|
let mut dao = self
|
|
.calendar_dao
|
|
.lock()
|
|
.expect("Unable to lock CalendarEventDao");
|
|
dao.find_relevant_events_hybrid(
|
|
&calendar_cx,
|
|
timestamp,
|
|
7, // ±7 days window
|
|
query_embedding.as_deref(),
|
|
5, // Top 5 events
|
|
)
|
|
.ok()
|
|
};
|
|
|
|
calendar_cx.span().set_status(Status::Ok);
|
|
|
|
if let Some(events) = events {
|
|
if events.is_empty() {
|
|
return Ok(None);
|
|
}
|
|
|
|
let formatted = events
|
|
.iter()
|
|
.map(|e| {
|
|
let date = DateTime::from_timestamp(e.start_time, 0)
|
|
.map(|dt| dt.format("%Y-%m-%d %H:%M").to_string())
|
|
.unwrap_or_else(|| "unknown".to_string());
|
|
|
|
let attendees = e
|
|
.attendees
|
|
.as_ref()
|
|
.and_then(|a| serde_json::from_str::<Vec<String>>(a).ok())
|
|
.map(|list| format!(" (with {})", list.join(", ")))
|
|
.unwrap_or_default();
|
|
|
|
format!("[{}] {}{}", date, e.summary, attendees)
|
|
})
|
|
.collect::<Vec<_>>()
|
|
.join("\n");
|
|
|
|
Ok(Some(format!("Calendar events:\n{}", formatted)))
|
|
} else {
|
|
Ok(None)
|
|
}
|
|
}
|
|
|
|
/// Gather location context for photo timestamp
|
|
/// Finds nearest location record (±30 minutes)
|
|
async fn gather_location_context(
|
|
&self,
|
|
parent_cx: &opentelemetry::Context,
|
|
timestamp: i64,
|
|
exif_gps: Option<(f64, f64)>,
|
|
) -> Result<Option<String>> {
|
|
let tracer = global_tracer();
|
|
let span = tracer.start_with_context("ai.context.location", parent_cx);
|
|
let location_cx = parent_cx.with_span(span);
|
|
|
|
let nearest = {
|
|
let mut dao = self
|
|
.location_dao
|
|
.lock()
|
|
.expect("Unable to lock LocationHistoryDao");
|
|
dao.find_nearest_location(
|
|
&location_cx,
|
|
timestamp,
|
|
10800, // ±3 hours (more realistic for photo timing)
|
|
)
|
|
.ok()
|
|
.flatten()
|
|
};
|
|
|
|
location_cx.span().set_status(Status::Ok);
|
|
|
|
if let Some(loc) = nearest {
|
|
// Check if this adds NEW information compared to EXIF
|
|
if let Some((exif_lat, exif_lon)) = exif_gps {
|
|
let distance =
|
|
Self::haversine_distance(exif_lat, exif_lon, loc.latitude, loc.longitude);
|
|
|
|
// Skip only if very close AND no useful activity/place info
|
|
// Allow activity context even if coordinates match
|
|
if distance < 0.5 && loc.activity.is_none() && loc.place_name.is_none() {
|
|
log::debug!(
|
|
"Location history matches EXIF GPS ({}m) with no extra context, skipping",
|
|
(distance * 1000.0) as i32
|
|
);
|
|
return Ok(None);
|
|
} else if distance < 0.5 {
|
|
log::debug!(
|
|
"Location history close to EXIF ({}m) but has activity/place info",
|
|
(distance * 1000.0) as i32
|
|
);
|
|
}
|
|
}
|
|
|
|
let activity = loc
|
|
.activity
|
|
.as_ref()
|
|
.map(|a| format!(" ({})", a))
|
|
.unwrap_or_default();
|
|
|
|
let place = loc
|
|
.place_name
|
|
.as_ref()
|
|
.map(|p| format!(" at {}", p))
|
|
.unwrap_or_default();
|
|
|
|
Ok(Some(format!(
|
|
"Location history: You were{}{}{}",
|
|
activity,
|
|
place,
|
|
if activity.is_empty() && place.is_empty() {
|
|
format!(" near {:.4}, {:.4}", loc.latitude, loc.longitude)
|
|
} else {
|
|
String::new()
|
|
}
|
|
)))
|
|
} else {
|
|
Ok(None)
|
|
}
|
|
}
|
|
|
|
/// Gather search context for photo date
|
|
/// Uses semantic search on queries (±30 days, top 5 relevant)
|
|
async fn gather_search_context(
|
|
&self,
|
|
parent_cx: &opentelemetry::Context,
|
|
timestamp: i64,
|
|
location: Option<&str>,
|
|
contact: Option<&str>,
|
|
enrichment: Option<&str>,
|
|
) -> Result<Option<String>> {
|
|
let tracer = global_tracer();
|
|
let span = tracer.start_with_context("ai.context.search", parent_cx);
|
|
let search_cx = parent_cx.with_span(span);
|
|
|
|
// Use enrichment (topics + photo description + tags) if available;
|
|
// fall back to generic temporal query.
|
|
let query_text = if let Some(enriched) = enrichment {
|
|
enriched.to_string()
|
|
} else {
|
|
// Fallback: generic temporal query
|
|
format!(
|
|
"searches about {} {} {}",
|
|
DateTime::from_timestamp(timestamp, 0)
|
|
.map(|dt| dt.format("%B %Y").to_string())
|
|
.unwrap_or_default(),
|
|
location.unwrap_or(""),
|
|
contact
|
|
.map(|c| format!("involving {}", c))
|
|
.unwrap_or_default()
|
|
)
|
|
};
|
|
|
|
let query_embedding = match self.ollama.generate_embedding(&query_text).await {
|
|
Ok(emb) => emb,
|
|
Err(e) => {
|
|
log::warn!("Failed to generate search embedding: {}", e);
|
|
search_cx.span().set_status(Status::Error {
|
|
description: e.to_string().into(),
|
|
});
|
|
return Ok(None);
|
|
}
|
|
};
|
|
|
|
let searches = {
|
|
let mut dao = self
|
|
.search_dao
|
|
.lock()
|
|
.expect("Unable to lock SearchHistoryDao");
|
|
dao.find_relevant_searches_hybrid(
|
|
&search_cx,
|
|
timestamp,
|
|
30, // ±30 days (wider window than calendar)
|
|
Some(&query_embedding),
|
|
5, // Top 5 searches
|
|
)
|
|
.ok()
|
|
};
|
|
|
|
search_cx.span().set_status(Status::Ok);
|
|
|
|
if let Some(searches) = searches {
|
|
if searches.is_empty() {
|
|
log::warn!(
|
|
"No relevant searches found for photo timestamp {}",
|
|
timestamp
|
|
);
|
|
return Ok(None);
|
|
}
|
|
|
|
let formatted = searches
|
|
.iter()
|
|
.map(|s| {
|
|
let date = DateTime::from_timestamp(s.timestamp, 0)
|
|
.map(|dt| dt.format("%Y-%m-%d").to_string())
|
|
.unwrap_or_else(|| "unknown".to_string());
|
|
format!("[{}] \"{}\"", date, s.query)
|
|
})
|
|
.collect::<Vec<_>>()
|
|
.join("\n");
|
|
|
|
Ok(Some(format!("Search history:\n{}", formatted)))
|
|
} else {
|
|
Ok(None)
|
|
}
|
|
}
|
|
|
|
/// Combine all context sources with equal weight
|
|
fn combine_contexts(
|
|
sms: Option<String>,
|
|
calendar: Option<String>,
|
|
location: Option<String>,
|
|
search: Option<String>,
|
|
tags: Option<String>,
|
|
) -> String {
|
|
let mut parts = Vec::new();
|
|
|
|
if let Some(s) = sms {
|
|
parts.push(format!("## Messages\n{}", s));
|
|
}
|
|
if let Some(c) = calendar {
|
|
parts.push(format!("## Calendar\n{}", c));
|
|
}
|
|
if let Some(l) = location {
|
|
parts.push(format!("## Location\n{}", l));
|
|
}
|
|
if let Some(s) = search {
|
|
parts.push(format!("## Searches\n{}", s));
|
|
}
|
|
if let Some(t) = tags {
|
|
parts.push(format!("## Tags\n{}", t));
|
|
}
|
|
|
|
if parts.is_empty() {
|
|
"No additional context available".to_string()
|
|
} else {
|
|
parts.join("\n\n")
|
|
}
|
|
}
|
|
|
|
/// Generate AI insight for a single photo with custom configuration
|
|
pub async fn generate_insight_for_photo_with_config(
|
|
&self,
|
|
file_path: &str,
|
|
custom_model: Option<String>,
|
|
custom_system_prompt: Option<String>,
|
|
num_ctx: Option<i32>,
|
|
temperature: Option<f32>,
|
|
top_p: Option<f32>,
|
|
top_k: Option<i32>,
|
|
min_p: Option<f32>,
|
|
) -> Result<()> {
|
|
let tracer = global_tracer();
|
|
let current_cx = opentelemetry::Context::current();
|
|
let mut span = tracer.start_with_context("ai.insight.generate", ¤t_cx);
|
|
|
|
// Normalize path to ensure consistent forward slashes in database
|
|
let file_path = normalize_path(file_path);
|
|
log::info!("Generating insight for photo: {}", file_path);
|
|
|
|
span.set_attribute(KeyValue::new("file_path", file_path.clone()));
|
|
|
|
// Create custom Ollama client if model is specified
|
|
let mut ollama_client = if let Some(model) = custom_model {
|
|
log::info!("Using custom model: {}", model);
|
|
span.set_attribute(KeyValue::new("custom_model", model.clone()));
|
|
OllamaClient::new(
|
|
self.ollama.primary_url.clone(),
|
|
self.ollama.fallback_url.clone(),
|
|
model.clone(),
|
|
Some(model), // Use the same custom model for fallback server
|
|
)
|
|
} else {
|
|
span.set_attribute(KeyValue::new("model", self.ollama.primary_model.clone()));
|
|
self.ollama.clone()
|
|
};
|
|
|
|
// Set context size if specified
|
|
if let Some(ctx) = num_ctx {
|
|
log::info!("Using custom context size: {}", ctx);
|
|
span.set_attribute(KeyValue::new("num_ctx", ctx as i64));
|
|
ollama_client.set_num_ctx(Some(ctx));
|
|
}
|
|
|
|
// Apply sampling parameters if any were provided
|
|
if temperature.is_some() || top_p.is_some() || top_k.is_some() || min_p.is_some() {
|
|
log::info!(
|
|
"Using sampling params — temperature: {:?}, top_p: {:?}, top_k: {:?}, min_p: {:?}",
|
|
temperature,
|
|
top_p,
|
|
top_k,
|
|
min_p
|
|
);
|
|
if let Some(t) = temperature {
|
|
span.set_attribute(KeyValue::new("temperature", t as f64));
|
|
}
|
|
if let Some(p) = top_p {
|
|
span.set_attribute(KeyValue::new("top_p", p as f64));
|
|
}
|
|
if let Some(k) = top_k {
|
|
span.set_attribute(KeyValue::new("top_k", k as i64));
|
|
}
|
|
if let Some(m) = min_p {
|
|
span.set_attribute(KeyValue::new("min_p", m as f64));
|
|
}
|
|
ollama_client.set_sampling_params(temperature, top_p, top_k, min_p);
|
|
}
|
|
|
|
// Create context with this span for child operations
|
|
let insight_cx = current_cx.with_span(span);
|
|
|
|
// 1. Get EXIF data for the photo
|
|
let exif = {
|
|
let mut exif_dao = self.exif_dao.lock().expect("Unable to lock ExifDao");
|
|
exif_dao
|
|
.get_exif(&insight_cx, &file_path)
|
|
.map_err(|e| anyhow::anyhow!("Failed to get EXIF: {:?}", e))?
|
|
};
|
|
|
|
// Get full timestamp for proximity-based message filtering
|
|
let timestamp = if let Some(ts) = exif.as_ref().and_then(|e| e.date_taken) {
|
|
ts
|
|
} else {
|
|
log::warn!("No date_taken in EXIF for {}, trying filename", file_path);
|
|
|
|
extract_date_from_filename(&file_path)
|
|
.map(|dt| dt.timestamp())
|
|
.or_else(|| {
|
|
// Combine base_path with file_path to get full path
|
|
let full_path = std::path::Path::new(&self.base_path).join(&file_path);
|
|
File::open(&full_path)
|
|
.and_then(|f| f.metadata())
|
|
.and_then(|m| m.created().or(m.modified()))
|
|
.map(|t| DateTime::<Utc>::from(t).timestamp())
|
|
.inspect_err(|e| {
|
|
log::warn!(
|
|
"Failed to get file timestamp for insight {}: {}",
|
|
file_path,
|
|
e
|
|
)
|
|
})
|
|
.ok()
|
|
})
|
|
.unwrap_or_else(|| Utc::now().timestamp())
|
|
};
|
|
|
|
let date_taken = DateTime::from_timestamp(timestamp, 0)
|
|
.map(|dt| dt.date_naive())
|
|
.unwrap_or_else(|| Utc::now().date_naive());
|
|
|
|
// 3. Extract contact name from file path
|
|
let contact = Self::extract_contact_from_path(&file_path);
|
|
log::info!("Extracted contact from path: {:?}", contact);
|
|
|
|
insight_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("date_taken", date_taken.to_string()));
|
|
if let Some(ref c) = contact {
|
|
insight_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("contact", c.clone()));
|
|
}
|
|
|
|
// Fetch file tags (used to enrich RAG and final context)
|
|
let tag_names: Vec<String> = {
|
|
let mut dao = self.tag_dao.lock().expect("Unable to lock TagDao");
|
|
dao.get_tags_for_path(&insight_cx, &file_path)
|
|
.unwrap_or_else(|e| {
|
|
log::warn!("Failed to fetch tags for insight {}: {}", file_path, e);
|
|
Vec::new()
|
|
})
|
|
.into_iter()
|
|
.map(|t| t.name)
|
|
.collect()
|
|
};
|
|
log::info!(
|
|
"Fetched {} tags for photo: {:?}",
|
|
tag_names.len(),
|
|
tag_names
|
|
);
|
|
|
|
// 4. Get location name from GPS coordinates (needed for RAG query)
|
|
let location = match exif {
|
|
Some(ref exif) => {
|
|
if let (Some(lat), Some(lon)) = (exif.gps_latitude, exif.gps_longitude) {
|
|
let loc = self.reverse_geocode(lat as f64, lon as f64).await;
|
|
if let Some(ref l) = loc {
|
|
insight_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("location", l.clone()));
|
|
Some(l.clone())
|
|
} else {
|
|
// Fallback: If reverse geocoding fails, use coordinates
|
|
log::warn!(
|
|
"Reverse geocoding failed for {}, {}, using coordinates as fallback",
|
|
lat,
|
|
lon
|
|
);
|
|
Some(format!("{:.4}, {:.4}", lat, lon))
|
|
}
|
|
} else {
|
|
None
|
|
}
|
|
}
|
|
None => None,
|
|
};
|
|
|
|
// Check if the model has vision capabilities
|
|
let model_to_check = ollama_client.primary_model.clone();
|
|
let has_vision = match OllamaClient::check_model_capabilities(
|
|
&ollama_client.primary_url,
|
|
&model_to_check,
|
|
)
|
|
.await
|
|
{
|
|
Ok(capabilities) => {
|
|
log::info!(
|
|
"Model '{}' vision capability: {}",
|
|
model_to_check,
|
|
capabilities.has_vision
|
|
);
|
|
capabilities.has_vision
|
|
}
|
|
Err(e) => {
|
|
log::warn!(
|
|
"Failed to check vision capabilities for model '{}', assuming no vision support: {}",
|
|
model_to_check,
|
|
e
|
|
);
|
|
false
|
|
}
|
|
};
|
|
|
|
insight_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("model_has_vision", has_vision));
|
|
|
|
// Load image and encode as base64 only if model supports vision
|
|
let image_base64 = if has_vision {
|
|
match self.load_image_as_base64(&file_path) {
|
|
Ok(b64) => {
|
|
log::info!(
|
|
"Successfully loaded image for vision-capable model '{}'",
|
|
model_to_check
|
|
);
|
|
Some(b64)
|
|
}
|
|
Err(e) => {
|
|
log::warn!("Failed to load image for vision model: {}", e);
|
|
None
|
|
}
|
|
}
|
|
} else {
|
|
log::info!(
|
|
"Model '{}' does not support vision, skipping image processing",
|
|
model_to_check
|
|
);
|
|
None
|
|
};
|
|
|
|
// Generate brief photo description for RAG enrichment (vision models only)
|
|
let photo_description: Option<String> = if let Some(ref img_b64) = image_base64 {
|
|
match ollama_client.generate_photo_description(img_b64).await {
|
|
Ok(desc) => {
|
|
log::info!("Photo description for RAG enrichment: {}", desc);
|
|
Some(desc)
|
|
}
|
|
Err(e) => {
|
|
log::warn!(
|
|
"Failed to generate photo description for RAG enrichment: {}",
|
|
e
|
|
);
|
|
None
|
|
}
|
|
}
|
|
} else {
|
|
None
|
|
};
|
|
|
|
// Build enriched context string for RAG: photo description + tags
|
|
// (SMS topics are passed separately to RAG functions)
|
|
let enriched_query: Option<String> = {
|
|
let mut parts: Vec<String> = Vec::new();
|
|
if let Some(ref desc) = photo_description {
|
|
parts.push(desc.clone());
|
|
}
|
|
if !tag_names.is_empty() {
|
|
parts.push(format!("tags: {}", tag_names.join(", ")));
|
|
}
|
|
if parts.is_empty() {
|
|
None
|
|
} else {
|
|
Some(parts.join(". "))
|
|
}
|
|
};
|
|
|
|
let mut search_enrichment: Option<String> = enriched_query.clone();
|
|
|
|
// 5. Intelligent retrieval: Hybrid approach for better context
|
|
let mut sms_summary = None;
|
|
let mut used_rag = false;
|
|
|
|
// TEMPORARY: Set to true to disable RAG and use only time-based retrieval for testing
|
|
let disable_rag_for_testing = false;
|
|
|
|
if disable_rag_for_testing {
|
|
log::warn!("RAG DISABLED FOR TESTING - Using only time-based retrieval (±4 days)");
|
|
// Skip directly to fallback
|
|
} else {
|
|
// ALWAYS use Strategy B: Expanded immediate context + historical RAG
|
|
// This is more reliable than pure semantic search which can match irrelevant messages
|
|
log::info!("Using expanded immediate context + historical RAG approach");
|
|
|
|
// Step 1: Get FULL immediate temporal context (±4 days, ALL messages)
|
|
let immediate_messages = self
|
|
.sms_client
|
|
.fetch_messages_for_contact(contact.as_deref(), timestamp)
|
|
.await
|
|
.unwrap_or_else(|e| {
|
|
log::error!("Failed to fetch immediate messages: {}", e);
|
|
Vec::new()
|
|
});
|
|
|
|
log::info!(
|
|
"Fetched {} messages from ±4 days window (using ALL for immediate context)",
|
|
immediate_messages.len()
|
|
);
|
|
|
|
if !immediate_messages.is_empty() {
|
|
// Step 2: Extract topics from immediate messages to enrich RAG query
|
|
let topics = self
|
|
.extract_topics_from_messages(&immediate_messages, &ollama_client)
|
|
.await;
|
|
|
|
log::info!("Extracted topics for query enrichment: {:?}", topics);
|
|
|
|
// Build full search enrichment: SMS topics + photo description + tag names
|
|
search_enrichment = {
|
|
let mut parts: Vec<String> = Vec::new();
|
|
if !topics.is_empty() {
|
|
parts.push(topics.join(", "));
|
|
}
|
|
if let Some(ref desc) = photo_description {
|
|
parts.push(desc.clone());
|
|
}
|
|
if !tag_names.is_empty() {
|
|
parts.push(format!("tags: {}", tag_names.join(", ")));
|
|
}
|
|
if parts.is_empty() {
|
|
None
|
|
} else {
|
|
Some(parts.join(". "))
|
|
}
|
|
};
|
|
|
|
// Step 3: Try historical RAG (>30 days ago) using extracted topics
|
|
let topics_slice = if topics.is_empty() {
|
|
None
|
|
} else {
|
|
Some(topics.as_slice())
|
|
};
|
|
match self
|
|
.find_relevant_messages_rag_historical(
|
|
&insight_cx,
|
|
date_taken,
|
|
None,
|
|
contact.as_deref(),
|
|
topics_slice,
|
|
10, // Top 10 historical matches
|
|
enriched_query.as_deref(),
|
|
)
|
|
.await
|
|
{
|
|
Ok(historical_messages) if !historical_messages.is_empty() => {
|
|
log::info!(
|
|
"Two-context approach: {} immediate (full conversation) + {} historical (similar past moments)",
|
|
immediate_messages.len(),
|
|
historical_messages.len()
|
|
);
|
|
used_rag = true;
|
|
|
|
// Step 4: Summarize contexts separately, then combine
|
|
let immediate_summary = self
|
|
.summarize_context_from_messages(
|
|
&immediate_messages,
|
|
&ollama_client,
|
|
custom_system_prompt.as_deref(),
|
|
)
|
|
.await
|
|
.unwrap_or_else(|| String::from("No immediate context"));
|
|
|
|
let historical_summary = self
|
|
.summarize_messages(
|
|
&historical_messages,
|
|
&ollama_client,
|
|
custom_system_prompt.as_deref(),
|
|
)
|
|
.await
|
|
.unwrap_or_else(|| String::from("No historical context"));
|
|
|
|
// Combine summaries
|
|
sms_summary = Some(format!(
|
|
"Immediate context (±4 days): {}\n\nSimilar moments from the past: {}",
|
|
immediate_summary, historical_summary
|
|
));
|
|
}
|
|
Ok(_) => {
|
|
// RAG found no historical matches, just use immediate context
|
|
log::info!("No historical RAG matches, using immediate context only");
|
|
sms_summary = self
|
|
.summarize_context_from_messages(
|
|
&immediate_messages,
|
|
&ollama_client,
|
|
custom_system_prompt.as_deref(),
|
|
)
|
|
.await;
|
|
}
|
|
Err(e) => {
|
|
log::warn!("Historical RAG failed, using immediate context only: {}", e);
|
|
sms_summary = self
|
|
.summarize_context_from_messages(
|
|
&immediate_messages,
|
|
&ollama_client,
|
|
custom_system_prompt.as_deref(),
|
|
)
|
|
.await;
|
|
}
|
|
}
|
|
} else {
|
|
log::info!("No immediate messages found, trying basic RAG as fallback");
|
|
// Fallback to basic RAG even without strong query
|
|
match self
|
|
.find_relevant_messages_rag(
|
|
date_taken,
|
|
None,
|
|
contact.as_deref(),
|
|
None,
|
|
20,
|
|
enriched_query.as_deref(),
|
|
)
|
|
.await
|
|
{
|
|
Ok(rag_messages) if !rag_messages.is_empty() => {
|
|
used_rag = true;
|
|
sms_summary = self
|
|
.summarize_messages(
|
|
&rag_messages,
|
|
&ollama_client,
|
|
custom_system_prompt.as_deref(),
|
|
)
|
|
.await;
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
}
|
|
|
|
// 6. Fallback to traditional time-based message retrieval if RAG didn't work
|
|
if !used_rag {
|
|
log::info!("Using traditional time-based message retrieval (±4 days)");
|
|
let sms_messages = self
|
|
.sms_client
|
|
.fetch_messages_for_contact(contact.as_deref(), timestamp)
|
|
.await
|
|
.unwrap_or_else(|e| {
|
|
log::error!("Failed to fetch SMS messages: {}", e);
|
|
Vec::new()
|
|
});
|
|
|
|
log::info!(
|
|
"Fetched {} SMS messages closest to {}",
|
|
sms_messages.len(),
|
|
DateTime::from_timestamp(timestamp, 0)
|
|
.map(|dt| dt.format("%Y-%m-%d %H:%M:%S").to_string())
|
|
.unwrap_or_else(|| "unknown time".to_string())
|
|
);
|
|
|
|
// Summarize time-based messages
|
|
if !sms_messages.is_empty() {
|
|
match self
|
|
.sms_client
|
|
.summarize_context(&sms_messages, &ollama_client)
|
|
.await
|
|
{
|
|
Ok(summary) => {
|
|
sms_summary = Some(summary);
|
|
}
|
|
Err(e) => {
|
|
log::warn!("Failed to summarize SMS context: {}", e);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
let retrieval_method = if used_rag { "RAG" } else { "time-based" };
|
|
insight_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("retrieval_method", retrieval_method));
|
|
insight_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("has_sms_context", sms_summary.is_some()));
|
|
|
|
log::info!(
|
|
"Photo context: date={}, location={:?}, retrieval_method={}",
|
|
date_taken,
|
|
location,
|
|
retrieval_method
|
|
);
|
|
|
|
// 6. Gather Google Takeout context from all sources
|
|
let calendar_context = self
|
|
.gather_calendar_context(&insight_cx, timestamp, location.as_deref())
|
|
.await
|
|
.ok()
|
|
.flatten();
|
|
|
|
let exif_gps = exif.as_ref().and_then(|e| {
|
|
if let (Some(lat), Some(lon)) = (e.gps_latitude, e.gps_longitude) {
|
|
Some((lat as f64, lon as f64))
|
|
} else {
|
|
None
|
|
}
|
|
});
|
|
|
|
let location_context = self
|
|
.gather_location_context(&insight_cx, timestamp, exif_gps)
|
|
.await
|
|
.ok()
|
|
.flatten();
|
|
|
|
let search_context = self
|
|
.gather_search_context(
|
|
&insight_cx,
|
|
timestamp,
|
|
location.as_deref(),
|
|
contact.as_deref(),
|
|
search_enrichment.as_deref(),
|
|
)
|
|
.await
|
|
.ok()
|
|
.flatten();
|
|
|
|
// 7. Combine all context sources with equal weight
|
|
let tags_context = if tag_names.is_empty() {
|
|
None
|
|
} else {
|
|
Some(tag_names.join(", "))
|
|
};
|
|
|
|
let combined_context = Self::combine_contexts(
|
|
sms_summary,
|
|
calendar_context,
|
|
location_context,
|
|
search_context,
|
|
tags_context,
|
|
);
|
|
|
|
log::info!(
|
|
"Combined context from all sources ({} chars)",
|
|
combined_context.len()
|
|
);
|
|
|
|
// 10. Generate summary first, then derive title from the summary
|
|
let summary = ollama_client
|
|
.generate_photo_summary(
|
|
date_taken,
|
|
location.as_deref(),
|
|
contact.as_deref(),
|
|
Some(&combined_context),
|
|
custom_system_prompt.as_deref(),
|
|
image_base64.clone(),
|
|
)
|
|
.await?;
|
|
|
|
let title = ollama_client
|
|
.generate_photo_title(&summary, custom_system_prompt.as_deref())
|
|
.await?;
|
|
|
|
log::info!("Generated title: {}", title);
|
|
log::info!("Generated summary: {}", summary);
|
|
|
|
insight_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("title_length", title.len() as i64));
|
|
insight_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("summary_length", summary.len() as i64));
|
|
|
|
// 11. Store in database
|
|
let insight = InsertPhotoInsight {
|
|
library_id: crate::libraries::PRIMARY_LIBRARY_ID,
|
|
file_path: file_path.to_string(),
|
|
title,
|
|
summary,
|
|
generated_at: Utc::now().timestamp(),
|
|
model_version: ollama_client.primary_model.clone(),
|
|
is_current: true,
|
|
training_messages: None,
|
|
};
|
|
|
|
let mut dao = self.insight_dao.lock().expect("Unable to lock InsightDao");
|
|
let result = dao
|
|
.store_insight(&insight_cx, insight)
|
|
.map_err(|e| anyhow::anyhow!("Failed to store insight: {:?}", e));
|
|
|
|
match &result {
|
|
Ok(_) => {
|
|
log::info!("Successfully stored insight for {}", file_path);
|
|
insight_cx.span().set_status(Status::Ok);
|
|
}
|
|
Err(e) => {
|
|
log::error!("Failed to store insight: {:?}", e);
|
|
insight_cx.span().set_status(Status::error(e.to_string()));
|
|
}
|
|
}
|
|
|
|
result?;
|
|
Ok(())
|
|
}
|
|
|
|
/// Extract key topics/entities from messages using LLM for query enrichment
|
|
async fn extract_topics_from_messages(
|
|
&self,
|
|
messages: &[crate::ai::SmsMessage],
|
|
ollama: &OllamaClient,
|
|
) -> Vec<String> {
|
|
if messages.is_empty() {
|
|
return Vec::new();
|
|
}
|
|
|
|
// Format a sample of messages for topic extraction
|
|
let sample_size = messages.len().min(20);
|
|
let sample_text: Vec<String> = messages
|
|
.iter()
|
|
.take(sample_size)
|
|
.map(|m| format!("{}: {}", if m.is_sent { "Me" } else { &m.contact }, m.body))
|
|
.collect();
|
|
|
|
let prompt = format!(
|
|
r#"Extract important entities from these messages that provide context about what was happening. Focus on:
|
|
|
|
1. **People**: Names of specific people mentioned (first names, nicknames)
|
|
2. **Places**: Locations, cities, buildings, workplaces, parks, restaurants, venues
|
|
3. **Activities**: Specific events, hobbies, groups, organizations (e.g., "drum corps", "auditions")
|
|
4. **Unique terms**: Domain-specific words or phrases that might need explanation (e.g., "Hyland", "Vanguard", "DCI")
|
|
|
|
Messages:
|
|
{}
|
|
|
|
Return a comma-separated list of 3-7 specific entities (people, places, activities, unique terms).
|
|
Focus on proper nouns and specific terms that provide context.
|
|
Return ONLY the comma-separated list, nothing else."#,
|
|
sample_text.join("\n")
|
|
);
|
|
|
|
match ollama
|
|
.generate(&prompt, Some("You are an entity extraction assistant. Extract proper nouns, people, places, and domain-specific terms that provide context."))
|
|
.await
|
|
{
|
|
Ok(response) => {
|
|
log::debug!("Topic extraction raw response: {}", response);
|
|
|
|
// Parse comma-separated topics
|
|
let topics: Vec<String> = response
|
|
.split(',')
|
|
.map(|s| s.trim().to_string())
|
|
.filter(|s| !s.is_empty() && s.len() > 1) // Filter out single chars
|
|
.take(7) // Increased from 5 to 7
|
|
.collect();
|
|
|
|
if topics.is_empty() {
|
|
log::warn!("Topic extraction returned empty list from {} messages", messages.len());
|
|
} else {
|
|
log::info!("Extracted {} topics from {} messages: {}", topics.len(), messages.len(), topics.join(", "));
|
|
}
|
|
|
|
topics
|
|
}
|
|
Err(e) => {
|
|
log::warn!("Failed to extract topics from messages: {}", e);
|
|
Vec::new()
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Summarize pre-formatted message strings using LLM (concise version for historical context)
|
|
async fn summarize_messages(
|
|
&self,
|
|
messages: &[String],
|
|
ollama: &OllamaClient,
|
|
custom_system: Option<&str>,
|
|
) -> Option<String> {
|
|
if messages.is_empty() {
|
|
return None;
|
|
}
|
|
|
|
let messages_text = messages.join("\n");
|
|
|
|
let prompt = format!(
|
|
r#"Summarize the context from these messages in 2-3 sentences. Focus on activities, locations, events, and relationships mentioned.
|
|
|
|
Messages:
|
|
{}
|
|
|
|
Return ONLY the summary, nothing else."#,
|
|
messages_text
|
|
);
|
|
|
|
let system = custom_system
|
|
.unwrap_or("You are a context summarization assistant. Be concise and factual.");
|
|
|
|
match ollama.generate(&prompt, Some(system)).await {
|
|
Ok(summary) => Some(summary),
|
|
Err(e) => {
|
|
log::warn!("Failed to summarize messages: {}", e);
|
|
None
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Convert SmsMessage objects to formatted strings and summarize with more detail
|
|
/// This is used for immediate context (±2 days) to preserve conversation details
|
|
async fn summarize_context_from_messages(
|
|
&self,
|
|
messages: &[crate::ai::SmsMessage],
|
|
ollama: &OllamaClient,
|
|
custom_system: Option<&str>,
|
|
) -> Option<String> {
|
|
if messages.is_empty() {
|
|
return None;
|
|
}
|
|
|
|
// Format messages
|
|
let formatted: Vec<String> = messages
|
|
.iter()
|
|
.map(|m| {
|
|
let sender = if m.is_sent { "Me" } else { &m.contact };
|
|
let timestamp = chrono::DateTime::from_timestamp(m.timestamp, 0)
|
|
.map(|dt| dt.format("%Y-%m-%d %H:%M").to_string())
|
|
.unwrap_or_else(|| "unknown time".to_string());
|
|
format!("[{}] {}: {}", timestamp, sender, m.body)
|
|
})
|
|
.collect();
|
|
|
|
let messages_text = formatted.join("\n");
|
|
|
|
// Use a more detailed prompt for immediate context
|
|
let prompt = format!(
|
|
r#"Provide a detailed summary of the conversation context from these messages. Include:
|
|
- Key activities, events, and plans discussed
|
|
- Important locations or places mentioned
|
|
- Emotional tone and relationship dynamics
|
|
- Any significant details that provide context about what was happening
|
|
|
|
Be thorough but organized.
|
|
|
|
Messages:
|
|
{}
|
|
|
|
Return ONLY the summary, nothing else."#,
|
|
messages_text
|
|
);
|
|
|
|
let system = custom_system.unwrap_or(
|
|
"You are a context summarization assistant. Be detailed and factual, preserving important context.",
|
|
);
|
|
|
|
match ollama.generate(&prompt, Some(system)).await {
|
|
Ok(summary) => Some(summary),
|
|
Err(e) => {
|
|
log::warn!("Failed to summarize immediate context: {}", e);
|
|
None
|
|
}
|
|
}
|
|
}
|
|
|
|
// ── Tool executors for agentic loop ────────────────────────────────
|
|
|
|
/// Dispatch a tool call to the appropriate executor
|
|
async fn execute_tool(
|
|
&self,
|
|
tool_name: &str,
|
|
arguments: &serde_json::Value,
|
|
ollama: &OllamaClient,
|
|
image_base64: &Option<String>,
|
|
file_path: &str,
|
|
cx: &opentelemetry::Context,
|
|
) -> String {
|
|
let result = match tool_name {
|
|
"search_rag" => self.tool_search_rag(arguments, cx).await,
|
|
"get_sms_messages" => self.tool_get_sms_messages(arguments, cx).await,
|
|
"get_calendar_events" => self.tool_get_calendar_events(arguments, cx).await,
|
|
"get_location_history" => self.tool_get_location_history(arguments, cx).await,
|
|
"get_file_tags" => self.tool_get_file_tags(arguments, cx).await,
|
|
"describe_photo" => self.tool_describe_photo(ollama, image_base64).await,
|
|
"reverse_geocode" => self.tool_reverse_geocode(arguments).await,
|
|
"recall_entities" => self.tool_recall_entities(arguments, cx).await,
|
|
"recall_facts_for_photo" => self.tool_recall_facts_for_photo(arguments, cx).await,
|
|
"store_entity" => self.tool_store_entity(arguments, ollama, cx).await,
|
|
"store_fact" => self.tool_store_fact(arguments, file_path, cx).await,
|
|
"get_current_datetime" => Self::tool_get_current_datetime(),
|
|
unknown => format!("Unknown tool: {}", unknown),
|
|
};
|
|
if result.starts_with("Error") || result.starts_with("No ") {
|
|
log::warn!("Tool '{}' result: {}", tool_name, result);
|
|
} else {
|
|
log::info!("Tool '{}' result: {} chars", tool_name, result.len());
|
|
}
|
|
result
|
|
}
|
|
|
|
/// Tool: search_rag — semantic search over daily summaries
|
|
async fn tool_search_rag(
|
|
&self,
|
|
args: &serde_json::Value,
|
|
_cx: &opentelemetry::Context,
|
|
) -> String {
|
|
let query = match args.get("query").and_then(|v| v.as_str()) {
|
|
Some(q) => q.to_string(),
|
|
None => return "Error: missing required parameter 'query'".to_string(),
|
|
};
|
|
let date_str = match args.get("date").and_then(|v| v.as_str()) {
|
|
Some(d) => d,
|
|
None => return "Error: missing required parameter 'date'".to_string(),
|
|
};
|
|
let date = match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") {
|
|
Ok(d) => d,
|
|
Err(e) => return format!("Error: failed to parse date '{}': {}", date_str, e),
|
|
};
|
|
let contact = args
|
|
.get("contact")
|
|
.and_then(|v| v.as_str())
|
|
.map(|s| s.to_string());
|
|
|
|
log::info!(
|
|
"tool_search_rag: query='{}', date={}, contact={:?}",
|
|
query,
|
|
date,
|
|
contact
|
|
);
|
|
|
|
match self
|
|
.find_relevant_messages_rag(date, None, contact.as_deref(), None, 5, Some(&query))
|
|
.await
|
|
{
|
|
Ok(results) if !results.is_empty() => results.join("\n\n"),
|
|
Ok(_) => "No relevant messages found.".to_string(),
|
|
Err(e) => format!("Error searching RAG: {}", e),
|
|
}
|
|
}
|
|
|
|
/// Tool: get_sms_messages — fetch SMS messages near a date for a contact
|
|
async fn tool_get_sms_messages(
|
|
&self,
|
|
args: &serde_json::Value,
|
|
_cx: &opentelemetry::Context,
|
|
) -> String {
|
|
let date_str = match args.get("date").and_then(|v| v.as_str()) {
|
|
Some(d) => d,
|
|
None => return "Error: missing required parameter 'date'".to_string(),
|
|
};
|
|
let contact = args
|
|
.get("contact")
|
|
.and_then(|v| v.as_str())
|
|
.map(|s| s.to_string());
|
|
let days_radius = args
|
|
.get("days_radius")
|
|
.and_then(|v| v.as_i64())
|
|
.unwrap_or(4);
|
|
|
|
let date = match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") {
|
|
Ok(d) => d,
|
|
Err(e) => return format!("Error: failed to parse date '{}': {}", date_str, e),
|
|
};
|
|
let timestamp = date.and_hms_opt(12, 0, 0).unwrap().and_utc().timestamp();
|
|
|
|
log::info!(
|
|
"tool_get_sms_messages: date={}, contact={:?}, days_radius={}",
|
|
date,
|
|
contact,
|
|
days_radius
|
|
);
|
|
|
|
match self
|
|
.sms_client
|
|
.fetch_messages_for_contact(contact.as_deref(), timestamp)
|
|
.await
|
|
{
|
|
Ok(messages) if !messages.is_empty() => {
|
|
let formatted: Vec<String> = messages
|
|
.iter()
|
|
.take(30)
|
|
.map(|m| {
|
|
let sender = if m.is_sent { "Me" } else { &m.contact };
|
|
let ts = DateTime::from_timestamp(m.timestamp, 0)
|
|
.map(|dt| dt.format("%Y-%m-%d %H:%M").to_string())
|
|
.unwrap_or_else(|| "unknown".to_string());
|
|
format!("[{}] {}: {}", ts, sender, m.body)
|
|
})
|
|
.collect();
|
|
format!(
|
|
"Found {} messages:\n{}",
|
|
messages.len(),
|
|
formatted.join("\n")
|
|
)
|
|
}
|
|
Ok(_) => "No messages found.".to_string(),
|
|
Err(e) => {
|
|
log::warn!("tool_get_sms_messages failed: {}", e);
|
|
format!("Error fetching SMS messages: {}", e)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Tool: get_calendar_events — fetch calendar events near a date
|
|
async fn tool_get_calendar_events(
|
|
&self,
|
|
args: &serde_json::Value,
|
|
cx: &opentelemetry::Context,
|
|
) -> String {
|
|
let date_str = match args.get("date").and_then(|v| v.as_str()) {
|
|
Some(d) => d,
|
|
None => return "Error: missing required parameter 'date'".to_string(),
|
|
};
|
|
let days_radius = args
|
|
.get("days_radius")
|
|
.and_then(|v| v.as_i64())
|
|
.unwrap_or(7);
|
|
|
|
let date = match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") {
|
|
Ok(d) => d,
|
|
Err(e) => return format!("Error: failed to parse date '{}': {}", date_str, e),
|
|
};
|
|
let timestamp = date.and_hms_opt(12, 0, 0).unwrap().and_utc().timestamp();
|
|
|
|
log::info!(
|
|
"tool_get_calendar_events: date={}, days_radius={}",
|
|
date,
|
|
days_radius
|
|
);
|
|
|
|
let events = {
|
|
let mut dao = self
|
|
.calendar_dao
|
|
.lock()
|
|
.expect("Unable to lock CalendarEventDao");
|
|
dao.find_relevant_events_hybrid(cx, timestamp, days_radius, None, 10)
|
|
.ok()
|
|
};
|
|
|
|
match events {
|
|
Some(evts) if !evts.is_empty() => {
|
|
let formatted: Vec<String> = evts
|
|
.iter()
|
|
.map(|e| {
|
|
let dt = DateTime::from_timestamp(e.start_time, 0)
|
|
.map(|dt| dt.format("%Y-%m-%d %H:%M").to_string())
|
|
.unwrap_or_else(|| "unknown".to_string());
|
|
let loc = e
|
|
.location
|
|
.as_ref()
|
|
.map(|l| format!(" at {}", l))
|
|
.unwrap_or_default();
|
|
let attendees = e
|
|
.attendees
|
|
.as_ref()
|
|
.and_then(|a| serde_json::from_str::<Vec<String>>(a).ok())
|
|
.map(|list| format!(" (with {})", list.join(", ")))
|
|
.unwrap_or_default();
|
|
format!("[{}] {}{}{}", dt, e.summary, loc, attendees)
|
|
})
|
|
.collect();
|
|
format!(
|
|
"Found {} calendar events:\n{}",
|
|
evts.len(),
|
|
formatted.join("\n")
|
|
)
|
|
}
|
|
Some(_) => "No calendar events found.".to_string(),
|
|
None => "No calendar events found.".to_string(),
|
|
}
|
|
}
|
|
|
|
/// Tool: get_location_history — fetch location records near a date
|
|
async fn tool_get_location_history(
|
|
&self,
|
|
args: &serde_json::Value,
|
|
cx: &opentelemetry::Context,
|
|
) -> String {
|
|
let date_str = match args.get("date").and_then(|v| v.as_str()) {
|
|
Some(d) => d,
|
|
None => return "Error: missing required parameter 'date'".to_string(),
|
|
};
|
|
let days_radius = args
|
|
.get("days_radius")
|
|
.and_then(|v| v.as_i64())
|
|
.unwrap_or(14);
|
|
|
|
let date = match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") {
|
|
Ok(d) => d,
|
|
Err(e) => return format!("Error: failed to parse date '{}': {}", date_str, e),
|
|
};
|
|
let timestamp = date.and_hms_opt(12, 0, 0).unwrap().and_utc().timestamp();
|
|
|
|
log::info!(
|
|
"tool_get_location_history: date={}, days_radius={}",
|
|
date,
|
|
days_radius
|
|
);
|
|
|
|
let start_ts = timestamp - (days_radius * 86400);
|
|
let end_ts = timestamp + (days_radius * 86400);
|
|
|
|
let locations = {
|
|
let mut dao = self
|
|
.location_dao
|
|
.lock()
|
|
.expect("Unable to lock LocationHistoryDao");
|
|
dao.find_locations_in_range(cx, start_ts, end_ts).ok()
|
|
};
|
|
|
|
match locations {
|
|
Some(locs) if !locs.is_empty() => {
|
|
let formatted: Vec<String> = locs
|
|
.iter()
|
|
.take(20)
|
|
.map(|loc| {
|
|
let dt = DateTime::from_timestamp(loc.timestamp, 0)
|
|
.map(|dt| dt.format("%Y-%m-%d %H:%M").to_string())
|
|
.unwrap_or_else(|| "unknown".to_string());
|
|
let activity = loc
|
|
.activity
|
|
.as_ref()
|
|
.map(|a| format!(" ({})", a))
|
|
.unwrap_or_default();
|
|
let place = loc
|
|
.place_name
|
|
.as_ref()
|
|
.map(|p| format!(" at {}", p))
|
|
.unwrap_or_default();
|
|
format!(
|
|
"[{}] {:.4}, {:.4}{}{}",
|
|
dt, loc.latitude, loc.longitude, place, activity
|
|
)
|
|
})
|
|
.collect();
|
|
format!(
|
|
"Found {} location records:\n{}",
|
|
locs.len(),
|
|
formatted.join("\n")
|
|
)
|
|
}
|
|
Some(_) => "No location history found.".to_string(),
|
|
None => "No location history found.".to_string(),
|
|
}
|
|
}
|
|
|
|
/// Tool: get_file_tags — fetch tags for a file path
|
|
async fn tool_get_file_tags(
|
|
&self,
|
|
args: &serde_json::Value,
|
|
cx: &opentelemetry::Context,
|
|
) -> String {
|
|
let file_path = match args.get("file_path").and_then(|v| v.as_str()) {
|
|
Some(p) => p.to_string(),
|
|
None => return "Error: missing required parameter 'file_path'".to_string(),
|
|
};
|
|
|
|
log::info!("tool_get_file_tags: file_path='{}'", file_path);
|
|
|
|
let tags = {
|
|
let mut dao = self.tag_dao.lock().expect("Unable to lock TagDao");
|
|
dao.get_tags_for_path(cx, &file_path).ok()
|
|
};
|
|
|
|
match tags {
|
|
Some(t) if !t.is_empty() => {
|
|
let names: Vec<String> = t.into_iter().map(|tag| tag.name).collect();
|
|
names.join(", ")
|
|
}
|
|
Some(_) => "No tags found.".to_string(),
|
|
None => "No tags found.".to_string(),
|
|
}
|
|
}
|
|
|
|
/// Tool: describe_photo — generate a visual description of the photo
|
|
async fn tool_describe_photo(
|
|
&self,
|
|
ollama: &OllamaClient,
|
|
image_base64: &Option<String>,
|
|
) -> String {
|
|
log::info!("tool_describe_photo: generating visual description");
|
|
|
|
match image_base64 {
|
|
Some(img) => match ollama.generate_photo_description(img).await {
|
|
Ok(desc) => desc,
|
|
Err(e) => format!("Error describing photo: {}", e),
|
|
},
|
|
None => "No image available for description.".to_string(),
|
|
}
|
|
}
|
|
|
|
/// Tool: reverse_geocode — convert GPS coordinates to a human-readable place name
|
|
async fn tool_reverse_geocode(&self, args: &serde_json::Value) -> String {
|
|
let lat = match args.get("latitude").and_then(|v| v.as_f64()) {
|
|
Some(v) => v,
|
|
None => return "Error: missing required parameter 'latitude'".to_string(),
|
|
};
|
|
let lon = match args.get("longitude").and_then(|v| v.as_f64()) {
|
|
Some(v) => v,
|
|
None => return "Error: missing required parameter 'longitude'".to_string(),
|
|
};
|
|
|
|
log::info!("tool_reverse_geocode: lat={}, lon={}", lat, lon);
|
|
|
|
match self.reverse_geocode(lat, lon).await {
|
|
Some(place) => place,
|
|
None => "Could not resolve coordinates to a place name.".to_string(),
|
|
}
|
|
}
|
|
|
|
/// Tool: recall_entities — search the knowledge memory for known entities
|
|
async fn tool_recall_entities(
|
|
&self,
|
|
args: &serde_json::Value,
|
|
cx: &opentelemetry::Context,
|
|
) -> String {
|
|
use crate::database::EntityFilter;
|
|
|
|
let name_search = args
|
|
.get("name")
|
|
.and_then(|v| v.as_str())
|
|
.map(|s| s.to_string());
|
|
let entity_type = args
|
|
.get("entity_type")
|
|
.and_then(|v| v.as_str())
|
|
.map(|s| s.to_string());
|
|
let limit = args.get("limit").and_then(|v| v.as_i64()).unwrap_or(10);
|
|
|
|
log::info!(
|
|
"tool_recall_entities: name={:?}, type={:?}, limit={}",
|
|
name_search,
|
|
entity_type,
|
|
limit
|
|
);
|
|
|
|
let filter = EntityFilter {
|
|
entity_type,
|
|
status: Some("active".to_string()),
|
|
search: name_search,
|
|
limit,
|
|
offset: 0,
|
|
};
|
|
|
|
let mut kdao = self
|
|
.knowledge_dao
|
|
.lock()
|
|
.expect("Unable to lock KnowledgeDao");
|
|
match kdao.list_entities(cx, filter) {
|
|
Ok((entities, _total)) if entities.is_empty() => {
|
|
"No known entities found matching the query.".to_string()
|
|
}
|
|
Ok((entities, _total)) => {
|
|
let lines: Vec<String> = entities
|
|
.iter()
|
|
.map(|e| {
|
|
format!(
|
|
"ID:{} | {} | {} | {} | confidence:{:.2}",
|
|
e.id, e.entity_type, e.name, e.description, e.confidence
|
|
)
|
|
})
|
|
.collect();
|
|
format!("Known entities:\n{}", lines.join("\n"))
|
|
}
|
|
Err(e) => format!("Error recalling entities: {:?}", e),
|
|
}
|
|
}
|
|
|
|
/// Tool: recall_facts_for_photo — retrieve facts linked to a specific photo
|
|
async fn tool_recall_facts_for_photo(
|
|
&self,
|
|
args: &serde_json::Value,
|
|
cx: &opentelemetry::Context,
|
|
) -> String {
|
|
let file_path = match args.get("file_path").and_then(|v| v.as_str()) {
|
|
Some(p) => p.to_string(),
|
|
None => return "Error: missing required parameter 'file_path'".to_string(),
|
|
};
|
|
|
|
log::info!("tool_recall_facts_for_photo: file_path={}", file_path);
|
|
|
|
let mut kdao = self
|
|
.knowledge_dao
|
|
.lock()
|
|
.expect("Unable to lock KnowledgeDao");
|
|
|
|
// Fetch photo links to find which entities appear in this photo
|
|
let links = match kdao.get_links_for_photo(cx, &file_path) {
|
|
Ok(l) => l,
|
|
Err(e) => return format!("Error fetching photo links: {:?}", e),
|
|
};
|
|
|
|
if links.is_empty() {
|
|
return "No knowledge facts found for this photo.".to_string();
|
|
}
|
|
|
|
let mut output_lines = Vec::new();
|
|
let entity_ids: Vec<i32> = links.iter().map(|l| l.entity_id).collect();
|
|
|
|
// For each linked entity, fetch its facts
|
|
for entity_id in entity_ids {
|
|
if let Ok(entity) = kdao.get_entity_by_id(cx, entity_id) {
|
|
if let Some(e) = entity {
|
|
let role = links
|
|
.iter()
|
|
.find(|l| l.entity_id == entity_id)
|
|
.map(|l| l.role.as_str())
|
|
.unwrap_or("subject");
|
|
output_lines.push(format!(
|
|
"Entity: {} ({}, role: {})",
|
|
e.name, e.entity_type, role
|
|
));
|
|
if let Ok(facts) = kdao.get_facts_for_entity(cx, entity_id) {
|
|
for f in facts.iter().filter(|f| f.status == "active") {
|
|
let obj = if let Some(ref v) = f.object_value {
|
|
v.clone()
|
|
} else if let Some(oid) = f.object_entity_id {
|
|
kdao.get_entity_by_id(cx, oid)
|
|
.ok()
|
|
.flatten()
|
|
.map(|e| format!("{} (entity ID: {})", e.name, e.id))
|
|
.unwrap_or_else(|| format!("entity:{}", oid))
|
|
} else {
|
|
"(unknown)".to_string()
|
|
};
|
|
output_lines.push(format!(" - {} {}", f.predicate, obj));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if output_lines.is_empty() {
|
|
"No active knowledge facts found for this photo.".to_string()
|
|
} else {
|
|
format!("Knowledge for this photo:\n{}", output_lines.join("\n"))
|
|
}
|
|
}
|
|
|
|
/// Tool: store_entity — upsert an entity into the knowledge memory
|
|
async fn tool_store_entity(
|
|
&self,
|
|
args: &serde_json::Value,
|
|
ollama: &OllamaClient,
|
|
cx: &opentelemetry::Context,
|
|
) -> String {
|
|
use crate::database::models::InsertEntity;
|
|
|
|
let name = match args.get("name").and_then(|v| v.as_str()) {
|
|
Some(n) => n.to_string(),
|
|
None => return "Error: missing required parameter 'name'".to_string(),
|
|
};
|
|
let entity_type = match args.get("entity_type").and_then(|v| v.as_str()) {
|
|
Some(t) => t.to_string(),
|
|
None => return "Error: missing required parameter 'entity_type'".to_string(),
|
|
};
|
|
let description = args
|
|
.get("description")
|
|
.and_then(|v| v.as_str())
|
|
.unwrap_or("")
|
|
.to_string();
|
|
|
|
log::info!(
|
|
"tool_store_entity: name='{}', type='{}', description='{}'",
|
|
name,
|
|
entity_type,
|
|
description
|
|
);
|
|
|
|
// Pre-flight similarity check — surface near-duplicates to the model
|
|
// before it commits to a new entity. Uses the first name token as the
|
|
// search term so "Sarah" matches when storing "Sarah Johnson" and vice
|
|
// versa. Exact-name matches are excluded (upsert_entity deduplicates
|
|
// those already). Results are appended to the tool response so the
|
|
// model can choose to use an existing entity's ID instead.
|
|
let similar_entities: Vec<String> = {
|
|
use crate::database::{EntityFilter, KnowledgeDao};
|
|
use crate::database::knowledge_dao::normalize_entity_type;
|
|
let normalised_type = normalize_entity_type(&entity_type);
|
|
let first_token = name
|
|
.split_whitespace()
|
|
.next()
|
|
.unwrap_or(&name)
|
|
.to_string();
|
|
let filter = EntityFilter {
|
|
entity_type: None, // search all types, filter client-side to avoid case issues
|
|
status: Some("active".to_string()),
|
|
search: Some(first_token),
|
|
limit: 10,
|
|
offset: 0,
|
|
};
|
|
let mut kdao = self.knowledge_dao.lock().expect("Unable to lock KnowledgeDao");
|
|
kdao.list_entities(cx, filter)
|
|
.unwrap_or_default()
|
|
.0
|
|
.into_iter()
|
|
.filter(|e| {
|
|
normalize_entity_type(&e.entity_type) == normalised_type
|
|
&& e.name.to_lowercase() != name.to_lowercase()
|
|
})
|
|
.map(|e| format!(" ID:{} | {} | {}", e.id, e.name, e.description))
|
|
.collect()
|
|
};
|
|
|
|
// Generate embedding for name + description (best-effort)
|
|
let embed_text = format!("{} {}", name, description);
|
|
let embedding: Option<Vec<u8>> = match ollama.generate_embedding(&embed_text).await {
|
|
Ok(vec) => {
|
|
let bytes: Vec<u8> = vec.iter().flat_map(|f| f.to_le_bytes()).collect();
|
|
Some(bytes)
|
|
}
|
|
Err(e) => {
|
|
log::warn!("Embedding generation failed for entity '{}': {}", name, e);
|
|
None
|
|
}
|
|
};
|
|
|
|
let now = chrono::Utc::now().timestamp();
|
|
let insert = InsertEntity {
|
|
name,
|
|
entity_type,
|
|
description,
|
|
embedding,
|
|
confidence: 0.6,
|
|
status: "active".to_string(),
|
|
created_at: now,
|
|
updated_at: now,
|
|
};
|
|
|
|
let mut kdao = self
|
|
.knowledge_dao
|
|
.lock()
|
|
.expect("Unable to lock KnowledgeDao");
|
|
match kdao.upsert_entity(cx, insert) {
|
|
Ok(entity) => {
|
|
let mut response = format!(
|
|
"Entity stored: ID:{} | {} | {} | confidence:{:.2}",
|
|
entity.id, entity.entity_type, entity.name, entity.confidence
|
|
);
|
|
if !similar_entities.is_empty() {
|
|
response.push_str(
|
|
"\nSimilar existing entities found — verify this is not a duplicate:\n",
|
|
);
|
|
response.push_str(&similar_entities.join("\n"));
|
|
response.push_str(
|
|
"\nIf one of these is the same entity, use their existing ID in store_fact instead of the newly created one.",
|
|
);
|
|
}
|
|
response
|
|
}
|
|
Err(e) => format!("Error storing entity: {:?}", e),
|
|
}
|
|
}
|
|
|
|
/// Tool: store_fact — record a fact about an entity, linked to the current photo
|
|
async fn tool_store_fact(
|
|
&self,
|
|
args: &serde_json::Value,
|
|
file_path: &str,
|
|
cx: &opentelemetry::Context,
|
|
) -> String {
|
|
use crate::database::models::{InsertEntityFact, InsertEntityPhotoLink};
|
|
|
|
let subject_entity_id = match args.get("subject_entity_id").and_then(|v| v.as_i64()) {
|
|
Some(id) => id as i32,
|
|
None => return "Error: missing required parameter 'subject_entity_id'".to_string(),
|
|
};
|
|
let predicate = match args.get("predicate").and_then(|v| v.as_str()) {
|
|
Some(p) => p.to_string(),
|
|
None => return "Error: missing required parameter 'predicate'".to_string(),
|
|
};
|
|
let object_entity_id = args
|
|
.get("object_entity_id")
|
|
.and_then(|v| v.as_i64())
|
|
.map(|id| id as i32);
|
|
let object_value = args
|
|
.get("object_value")
|
|
.and_then(|v| v.as_str())
|
|
.map(|s| s.to_string());
|
|
|
|
if object_entity_id.is_none() && object_value.is_none() {
|
|
return "Error: provide either object_entity_id or object_value".to_string();
|
|
}
|
|
|
|
let photo_role = args
|
|
.get("photo_role")
|
|
.and_then(|v| v.as_str())
|
|
.unwrap_or("subject")
|
|
.to_string();
|
|
|
|
log::info!(
|
|
"tool_store_fact: entity_id={}, predicate='{}', object_entity_id={:?}, object_value={:?}, photo='{}'",
|
|
subject_entity_id,
|
|
predicate,
|
|
object_entity_id,
|
|
object_value,
|
|
file_path
|
|
);
|
|
|
|
let fact = InsertEntityFact {
|
|
subject_entity_id,
|
|
predicate,
|
|
object_entity_id,
|
|
object_value,
|
|
source_photo: Some(file_path.to_string()),
|
|
source_insight_id: None, // will be backfilled after store_insight
|
|
confidence: 0.6,
|
|
status: "active".to_string(),
|
|
created_at: chrono::Utc::now().timestamp(),
|
|
};
|
|
|
|
let mut kdao = self
|
|
.knowledge_dao
|
|
.lock()
|
|
.expect("Unable to lock KnowledgeDao");
|
|
|
|
// Upsert the fact (corroboration bumps confidence if duplicate)
|
|
let (stored_fact, is_new) = match kdao.upsert_fact(cx, fact) {
|
|
Ok(r) => r,
|
|
Err(e) => return format!("Error storing fact: {:?}", e),
|
|
};
|
|
|
|
// Upsert a photo link so this entity is associated with this photo
|
|
let link = InsertEntityPhotoLink {
|
|
entity_id: subject_entity_id,
|
|
library_id: crate::libraries::PRIMARY_LIBRARY_ID,
|
|
file_path: file_path.to_string(),
|
|
role: photo_role,
|
|
};
|
|
if let Err(e) = kdao.upsert_photo_link(cx, link) {
|
|
log::warn!(
|
|
"Failed to upsert photo link for entity {}: {:?}",
|
|
subject_entity_id,
|
|
e
|
|
);
|
|
}
|
|
|
|
let action = if is_new {
|
|
"Stored new fact"
|
|
} else {
|
|
"Corroborated existing fact"
|
|
};
|
|
format!(
|
|
"{}: ID:{} | confidence:{:.2}",
|
|
action, stored_fact.id, stored_fact.confidence
|
|
)
|
|
}
|
|
|
|
/// Tool: get_current_datetime — returns the current local date and time
|
|
fn tool_get_current_datetime() -> String {
|
|
let now = Local::now();
|
|
format!(
|
|
"Current date/time: {} ({})",
|
|
now.format("%Y-%m-%d %H:%M:%S %Z"),
|
|
now.format("%A")
|
|
)
|
|
}
|
|
|
|
// ── Agentic insight generation ──────────────────────────────────────
|
|
|
|
/// Build the list of tool definitions for the agentic loop
|
|
fn build_tool_definitions(has_vision: bool) -> Vec<Tool> {
|
|
let mut tools = vec![
|
|
Tool::function(
|
|
"search_rag",
|
|
"Search conversation history using semantic search. Use this to find relevant past conversations about specific topics, people, or events.",
|
|
serde_json::json!({
|
|
"type": "object",
|
|
"required": ["query", "date"],
|
|
"properties": {
|
|
"query": {
|
|
"type": "string",
|
|
"description": "The search query to find relevant conversations"
|
|
},
|
|
"date": {
|
|
"type": "string",
|
|
"description": "The reference date in YYYY-MM-DD format"
|
|
},
|
|
"contact": {
|
|
"type": "string",
|
|
"description": "Optional contact name to filter results"
|
|
}
|
|
}
|
|
}),
|
|
),
|
|
Tool::function(
|
|
"get_sms_messages",
|
|
"Fetch SMS/text messages near a specific date. Returns the actual message conversation. Omit contact to search across all conversations.",
|
|
serde_json::json!({
|
|
"type": "object",
|
|
"required": ["date"],
|
|
"properties": {
|
|
"date": {
|
|
"type": "string",
|
|
"description": "The center date in YYYY-MM-DD format"
|
|
},
|
|
"contact": {
|
|
"type": "string",
|
|
"description": "Optional contact name to filter messages. If omitted, searches all conversations."
|
|
},
|
|
"days_radius": {
|
|
"type": "integer",
|
|
"description": "Number of days before and after the date to search (default: 4)"
|
|
}
|
|
}
|
|
}),
|
|
),
|
|
Tool::function(
|
|
"get_calendar_events",
|
|
"Fetch calendar events near a specific date. Shows scheduled events, meetings, and activities.",
|
|
serde_json::json!({
|
|
"type": "object",
|
|
"required": ["date"],
|
|
"properties": {
|
|
"date": {
|
|
"type": "string",
|
|
"description": "The center date in YYYY-MM-DD format"
|
|
},
|
|
"days_radius": {
|
|
"type": "integer",
|
|
"description": "Number of days before and after the date to search (default: 7)"
|
|
}
|
|
}
|
|
}),
|
|
),
|
|
Tool::function(
|
|
"get_location_history",
|
|
"Fetch location history records near a specific date. Shows places visited and activities.",
|
|
serde_json::json!({
|
|
"type": "object",
|
|
"required": ["date"],
|
|
"properties": {
|
|
"date": {
|
|
"type": "string",
|
|
"description": "The center date in YYYY-MM-DD format"
|
|
},
|
|
"days_radius": {
|
|
"type": "integer",
|
|
"description": "Number of days before and after the date to search (default: 14)"
|
|
}
|
|
}
|
|
}),
|
|
),
|
|
Tool::function(
|
|
"get_file_tags",
|
|
"Get tags/labels that have been applied to a specific photo file.",
|
|
serde_json::json!({
|
|
"type": "object",
|
|
"required": ["file_path"],
|
|
"properties": {
|
|
"file_path": {
|
|
"type": "string",
|
|
"description": "The file path of the photo to get tags for"
|
|
}
|
|
}
|
|
}),
|
|
),
|
|
];
|
|
|
|
tools.push(Tool::function(
|
|
"reverse_geocode",
|
|
"Convert GPS latitude/longitude coordinates to a human-readable place name (city, state). Use this when GPS coordinates are available in the photo metadata, or to resolve coordinates returned by get_location_history.",
|
|
serde_json::json!({
|
|
"type": "object",
|
|
"required": ["latitude", "longitude"],
|
|
"properties": {
|
|
"latitude": {
|
|
"type": "number",
|
|
"description": "GPS latitude in decimal degrees"
|
|
},
|
|
"longitude": {
|
|
"type": "number",
|
|
"description": "GPS longitude in decimal degrees"
|
|
}
|
|
}
|
|
}),
|
|
));
|
|
|
|
// Knowledge memory tools
|
|
tools.push(Tool::function(
|
|
"recall_entities",
|
|
"Search the knowledge memory for people, places, events, or things previously learned from other photos. Use this to retrieve context about subjects appearing in this photo.",
|
|
serde_json::json!({
|
|
"type": "object",
|
|
"properties": {
|
|
"name": {
|
|
"type": "string",
|
|
"description": "Name or partial name to search for (case-insensitive substring match)"
|
|
},
|
|
"entity_type": {
|
|
"type": "string",
|
|
"enum": ["person", "place", "event", "thing"],
|
|
"description": "Filter by entity type (optional)"
|
|
},
|
|
"limit": {
|
|
"type": "integer",
|
|
"description": "Maximum number of results to return (default: 10)"
|
|
}
|
|
}
|
|
}),
|
|
));
|
|
|
|
tools.push(Tool::function(
|
|
"recall_facts_for_photo",
|
|
"Retrieve all known facts linked to a specific photo from the knowledge memory. Use this at the start of insight generation to load any previously stored knowledge about subjects in this photo.",
|
|
serde_json::json!({
|
|
"type": "object",
|
|
"required": ["file_path"],
|
|
"properties": {
|
|
"file_path": {
|
|
"type": "string",
|
|
"description": "The file path of the photo to retrieve facts for"
|
|
}
|
|
}
|
|
}),
|
|
));
|
|
|
|
tools.push(Tool::function(
|
|
"store_entity",
|
|
"Store or update a person, place, event, or thing in the knowledge memory. Call this when you identify a subject in this photo that should be remembered for future insights.",
|
|
serde_json::json!({
|
|
"type": "object",
|
|
"required": ["name", "entity_type"],
|
|
"properties": {
|
|
"name": {
|
|
"type": "string",
|
|
"description": "The canonical name of the entity (e.g. 'John Smith', 'Banff National Park')"
|
|
},
|
|
"entity_type": {
|
|
"type": "string",
|
|
"enum": ["person", "place", "event", "thing"],
|
|
"description": "The type of entity"
|
|
},
|
|
"description": {
|
|
"type": "string",
|
|
"description": "A brief description of the entity"
|
|
}
|
|
}
|
|
}),
|
|
));
|
|
|
|
tools.push(Tool::function(
|
|
"store_fact",
|
|
"Record a fact about an entity in the knowledge memory. Provide EITHER object_entity_id (when the object is a known entity whose ID you have) OR object_value (for free-text attributes). The fact will be linked to the current photo automatically.",
|
|
serde_json::json!({
|
|
"type": "object",
|
|
"required": ["subject_entity_id", "predicate"],
|
|
"properties": {
|
|
"subject_entity_id": {
|
|
"type": "integer",
|
|
"description": "The ID of the entity this fact is about (returned by store_entity or recall_entities)"
|
|
},
|
|
"predicate": {
|
|
"type": "string",
|
|
"description": "The relationship or attribute (e.g. 'is_friend_of', 'located_in', 'attended_event', 'is_sibling_of')"
|
|
},
|
|
"object_entity_id": {
|
|
"type": "integer",
|
|
"description": "Use when the object is a known entity (e.g. Cameron's entity ID for 'is_friend_of Cameron'). Takes precedence over object_value."
|
|
},
|
|
"object_value": {
|
|
"type": "string",
|
|
"description": "Use for free-text attributes where the object is not a stored entity (e.g. 'Portland, Oregon', 'software engineer')"
|
|
},
|
|
"photo_role": {
|
|
"type": "string",
|
|
"description": "How this entity appears in the photo (e.g. 'subject', 'background', 'location'). Defaults to 'subject'."
|
|
}
|
|
}
|
|
}),
|
|
));
|
|
|
|
tools.push(Tool::function(
|
|
"get_current_datetime",
|
|
"Get the current date and time. Useful for understanding how long ago the photo was taken.",
|
|
serde_json::json!({
|
|
"type": "object",
|
|
"properties": {}
|
|
}),
|
|
));
|
|
|
|
if has_vision {
|
|
tools.push(Tool::function(
|
|
"describe_photo",
|
|
"Generate a visual description of the photo. Describes people, location, and activity visible in the image.",
|
|
serde_json::json!({
|
|
"type": "object",
|
|
"properties": {}
|
|
}),
|
|
));
|
|
}
|
|
|
|
tools
|
|
}
|
|
|
|
/// Generate an AI insight for a photo using an agentic tool-calling loop.
|
|
/// The model decides which tools to call to gather context before writing the final insight.
|
|
pub async fn generate_agentic_insight_for_photo(
|
|
&self,
|
|
file_path: &str,
|
|
custom_model: Option<String>,
|
|
custom_system_prompt: Option<String>,
|
|
num_ctx: Option<i32>,
|
|
temperature: Option<f32>,
|
|
top_p: Option<f32>,
|
|
top_k: Option<i32>,
|
|
min_p: Option<f32>,
|
|
max_iterations: usize,
|
|
) -> Result<(Option<i32>, Option<i32>)> {
|
|
let tracer = global_tracer();
|
|
let current_cx = opentelemetry::Context::current();
|
|
let mut span = tracer.start_with_context("ai.insight.generate_agentic", ¤t_cx);
|
|
|
|
let file_path = normalize_path(file_path);
|
|
log::info!("Generating agentic insight for photo: {}", file_path);
|
|
|
|
span.set_attribute(KeyValue::new("file_path", file_path.clone()));
|
|
span.set_attribute(KeyValue::new("max_iterations", max_iterations as i64));
|
|
|
|
// 1. Create OllamaClient
|
|
let mut ollama_client = if let Some(ref model) = custom_model {
|
|
log::info!("Using custom model for agentic: {}", model);
|
|
span.set_attribute(KeyValue::new("custom_model", model.clone()));
|
|
OllamaClient::new(
|
|
self.ollama.primary_url.clone(),
|
|
self.ollama.fallback_url.clone(),
|
|
model.clone(),
|
|
Some(model.clone()),
|
|
)
|
|
} else {
|
|
span.set_attribute(KeyValue::new("model", self.ollama.primary_model.clone()));
|
|
self.ollama.clone()
|
|
};
|
|
|
|
if let Some(ctx) = num_ctx {
|
|
log::info!("Using custom context size: {}", ctx);
|
|
span.set_attribute(KeyValue::new("num_ctx", ctx as i64));
|
|
ollama_client.set_num_ctx(Some(ctx));
|
|
}
|
|
|
|
if temperature.is_some() || top_p.is_some() || top_k.is_some() || min_p.is_some() {
|
|
log::info!(
|
|
"Using sampling params — temperature: {:?}, top_p: {:?}, top_k: {:?}, min_p: {:?}",
|
|
temperature,
|
|
top_p,
|
|
top_k,
|
|
min_p
|
|
);
|
|
if let Some(t) = temperature {
|
|
span.set_attribute(KeyValue::new("temperature", t as f64));
|
|
}
|
|
if let Some(p) = top_p {
|
|
span.set_attribute(KeyValue::new("top_p", p as f64));
|
|
}
|
|
if let Some(k) = top_k {
|
|
span.set_attribute(KeyValue::new("top_k", k as i64));
|
|
}
|
|
if let Some(m) = min_p {
|
|
span.set_attribute(KeyValue::new("min_p", m as f64));
|
|
}
|
|
ollama_client.set_sampling_params(temperature, top_p, top_k, min_p);
|
|
}
|
|
|
|
let insight_cx = current_cx.with_span(span);
|
|
|
|
// 2a. Verify the model exists on at least one server before checking capabilities
|
|
if let Some(ref model_name) = custom_model {
|
|
let available_on_primary =
|
|
OllamaClient::is_model_available(&ollama_client.primary_url, model_name)
|
|
.await
|
|
.unwrap_or(false);
|
|
|
|
let available_on_fallback = if let Some(ref fallback_url) = ollama_client.fallback_url {
|
|
OllamaClient::is_model_available(fallback_url, model_name)
|
|
.await
|
|
.unwrap_or(false)
|
|
} else {
|
|
false
|
|
};
|
|
|
|
if !available_on_primary && !available_on_fallback {
|
|
anyhow::bail!(
|
|
"model not available: '{}' not found on any configured server",
|
|
model_name
|
|
);
|
|
}
|
|
}
|
|
|
|
// 2b. Check tool calling capability — try primary, fall back to fallback URL
|
|
let model_name_for_caps = &ollama_client.primary_model;
|
|
let capabilities = match OllamaClient::check_model_capabilities(
|
|
&ollama_client.primary_url,
|
|
model_name_for_caps,
|
|
)
|
|
.await
|
|
{
|
|
Ok(caps) => caps,
|
|
Err(_) => {
|
|
// Model may only be on the fallback server
|
|
let fallback_url = ollama_client.fallback_url.as_deref().ok_or_else(|| {
|
|
anyhow::anyhow!(
|
|
"Failed to check model capabilities for '{}': model not found on primary server and no fallback configured",
|
|
model_name_for_caps
|
|
)
|
|
})?;
|
|
OllamaClient::check_model_capabilities(fallback_url, model_name_for_caps)
|
|
.await
|
|
.map_err(|e| {
|
|
anyhow::anyhow!(
|
|
"Failed to check model capabilities for '{}': {}",
|
|
model_name_for_caps,
|
|
e
|
|
)
|
|
})?
|
|
}
|
|
};
|
|
|
|
if !capabilities.has_tool_calling {
|
|
return Err(anyhow::anyhow!(
|
|
"tool calling not supported by model '{}'",
|
|
ollama_client.primary_model
|
|
));
|
|
}
|
|
|
|
let has_vision = capabilities.has_vision;
|
|
insight_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("model_has_vision", has_vision));
|
|
insight_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("model_has_tool_calling", true));
|
|
|
|
// 3. Fetch EXIF
|
|
let exif = {
|
|
let mut exif_dao = self.exif_dao.lock().expect("Unable to lock ExifDao");
|
|
exif_dao
|
|
.get_exif(&insight_cx, &file_path)
|
|
.map_err(|e| anyhow::anyhow!("Failed to get EXIF: {:?}", e))?
|
|
};
|
|
|
|
// 4. Extract timestamp and contact
|
|
let timestamp = if let Some(ts) = exif.as_ref().and_then(|e| e.date_taken) {
|
|
ts
|
|
} else {
|
|
log::warn!("No date_taken in EXIF for {}, trying filename", file_path);
|
|
extract_date_from_filename(&file_path)
|
|
.map(|dt| dt.timestamp())
|
|
.or_else(|| {
|
|
let full_path = std::path::Path::new(&self.base_path).join(&file_path);
|
|
File::open(&full_path)
|
|
.and_then(|f| f.metadata())
|
|
.and_then(|m| m.created().or(m.modified()))
|
|
.map(|t| DateTime::<Utc>::from(t).timestamp())
|
|
.inspect_err(|e| {
|
|
log::warn!(
|
|
"Failed to get file timestamp for agentic insight {}: {}",
|
|
file_path,
|
|
e
|
|
)
|
|
})
|
|
.ok()
|
|
})
|
|
.unwrap_or_else(|| Utc::now().timestamp())
|
|
};
|
|
|
|
let date_taken = DateTime::from_timestamp(timestamp, 0)
|
|
.map(|dt| dt.date_naive())
|
|
.unwrap_or_else(|| Utc::now().date_naive());
|
|
|
|
let contact = Self::extract_contact_from_path(&file_path);
|
|
log::info!("Agentic: date_taken={}, contact={:?}", date_taken, contact);
|
|
|
|
// 5. Fetch tags
|
|
let tag_names: Vec<String> = {
|
|
let mut dao = self.tag_dao.lock().expect("Unable to lock TagDao");
|
|
dao.get_tags_for_path(&insight_cx, &file_path)
|
|
.unwrap_or_else(|e| {
|
|
log::warn!("Failed to fetch tags for agentic {}: {}", file_path, e);
|
|
Vec::new()
|
|
})
|
|
.into_iter()
|
|
.map(|t| t.name)
|
|
.collect()
|
|
};
|
|
|
|
// 6. Clear existing entity-photo links for this file so the run starts fresh,
|
|
// and ensure the owner entity (Cameron) exists so the agent can reference it.
|
|
let cameron_entity_id: Option<i32> = {
|
|
let mut kdao = self
|
|
.knowledge_dao
|
|
.lock()
|
|
.expect("Unable to lock KnowledgeDao");
|
|
|
|
if let Err(e) = kdao.delete_photo_links_for_file(&insight_cx, &file_path) {
|
|
log::warn!(
|
|
"Failed to clear entity_photo_links for {}: {:?}",
|
|
file_path,
|
|
e
|
|
);
|
|
}
|
|
|
|
// Upsert the owner entity so the agent always has a stable entity ID to reference.
|
|
let owner = crate::database::models::InsertEntity {
|
|
name: "Cameron".to_string(),
|
|
entity_type: "person".to_string(),
|
|
description: "The owner of this photo collection. All memories are written from Cameron's perspective.".to_string(),
|
|
embedding: None,
|
|
confidence: 1.0,
|
|
status: "active".to_string(),
|
|
created_at: Utc::now().timestamp(),
|
|
updated_at: Utc::now().timestamp(),
|
|
};
|
|
match kdao.upsert_entity(&insight_cx, owner) {
|
|
Ok(e) => {
|
|
log::info!("Cameron entity ID: {}", e.id);
|
|
Some(e.id)
|
|
}
|
|
Err(e) => {
|
|
log::warn!("Failed to upsert Cameron entity: {:?}", e);
|
|
None
|
|
}
|
|
}
|
|
};
|
|
|
|
// 7. Load image if vision capable
|
|
let image_base64 = if has_vision {
|
|
match self.load_image_as_base64(&file_path) {
|
|
Ok(b64) => {
|
|
log::info!("Loaded image for vision-capable agentic model");
|
|
Some(b64)
|
|
}
|
|
Err(e) => {
|
|
log::warn!("Failed to load image for agentic vision: {}", e);
|
|
None
|
|
}
|
|
}
|
|
} else {
|
|
None
|
|
};
|
|
|
|
// 8. Build system message
|
|
let cameron_id_note = match cameron_entity_id {
|
|
Some(id) => format!(
|
|
"\n\nYour identity in the knowledge store: Cameron (entity ID: {}). \
|
|
When storing facts where you (Cameron) are the object — for example, someone is your friend, \
|
|
sibling, or colleague — use subject_entity_id for the other person and set object_value to \
|
|
\"Cameron\" (or use store_fact with the other person as subject). When storing facts about \
|
|
Cameron directly, use {} as the subject_entity_id.",
|
|
id, id
|
|
),
|
|
None => String::new(),
|
|
};
|
|
let base_system = format!(
|
|
"You are a personal photo memory assistant helping to reconstruct a memory from a photo.{cameron_id_note}\n\n\
|
|
IMPORTANT INSTRUCTIONS:\n\
|
|
1. You MUST call multiple tools to gather context BEFORE writing any final insight. Do not produce a final answer after only one or two tool calls.\n\
|
|
2. When calling get_sms_messages and search_rag, always make at least one call WITHOUT a contact filter to capture what else was happening in Cameron's life around this date — other conversations, events, and activities provide important wider context even when a specific contact is known.\n\
|
|
3. Use recall_facts_for_photo to load any previously stored knowledge about subjects in this photo.\n\
|
|
4. Use recall_entities to look up known people, places, or things that appear in this photo.\n\
|
|
5. When you identify people, places, events, or notable things in this photo: use store_entity to record them and store_fact to record key facts (relationships, roles, attributes). This builds a persistent memory for future insights.\n\
|
|
6. Only produce your final insight AFTER you have gathered context from at least 5-12 tool calls.\n\
|
|
7. If a tool returns no results, that is useful information — continue calling the remaining tools anyway.",
|
|
cameron_id_note = cameron_id_note
|
|
);
|
|
let system_content = if let Some(ref custom) = custom_system_prompt {
|
|
format!("{}\n\n{}", custom, base_system)
|
|
} else {
|
|
base_system.to_string()
|
|
};
|
|
|
|
// 9. Build user message
|
|
let gps_info = exif
|
|
.as_ref()
|
|
.and_then(|e| {
|
|
if let (Some(lat), Some(lon)) = (e.gps_latitude, e.gps_longitude) {
|
|
Some(format!("GPS: {:.4}, {:.4}", lat, lon))
|
|
} else {
|
|
None
|
|
}
|
|
})
|
|
.unwrap_or_else(|| "GPS: unknown".to_string());
|
|
|
|
let tags_info = if tag_names.is_empty() {
|
|
"Tags: none".to_string()
|
|
} else {
|
|
format!("Tags: {}", tag_names.join(", "))
|
|
};
|
|
|
|
let contact_info = contact
|
|
.as_ref()
|
|
.map(|c| format!("Contact/Person: {}", c))
|
|
.unwrap_or_else(|| "Contact/Person: unknown".to_string());
|
|
|
|
let user_content = format!(
|
|
"Please analyze this photo and gather any relevant context from the surrounding weeks.\n\n\
|
|
Photo file path: {}\n\
|
|
Date taken: {}\n\
|
|
{}\n\
|
|
{}\n\
|
|
{}\n\n\
|
|
Use the available tools to gather more context about this moment (messages, calendar events, location history, etc.), \
|
|
then write a detailed insight with a title and summary.",
|
|
file_path,
|
|
date_taken.format("%B %d, %Y"),
|
|
contact_info,
|
|
gps_info,
|
|
tags_info,
|
|
);
|
|
|
|
// 10. Define tools
|
|
let tools = Self::build_tool_definitions(has_vision);
|
|
|
|
// 11. Build initial messages
|
|
let system_msg = ChatMessage::system(system_content);
|
|
let mut user_msg = ChatMessage::user(user_content);
|
|
if let Some(ref img) = image_base64 {
|
|
user_msg.images = Some(vec![img.clone()]);
|
|
}
|
|
|
|
let mut messages = vec![system_msg, user_msg];
|
|
|
|
// 12. Agentic loop
|
|
let loop_span = tracer.start_with_context("ai.agentic.loop", &insight_cx);
|
|
let loop_cx = insight_cx.with_span(loop_span);
|
|
|
|
let mut final_content = String::new();
|
|
let mut iterations_used = 0usize;
|
|
let mut last_prompt_eval_count: Option<i32> = None;
|
|
let mut last_eval_count: Option<i32> = None;
|
|
|
|
for iteration in 0..max_iterations {
|
|
iterations_used = iteration + 1;
|
|
log::info!("Agentic iteration {}/{}", iteration + 1, max_iterations);
|
|
|
|
let (response, prompt_tokens, eval_tokens) = ollama_client
|
|
.chat_with_tools(messages.clone(), tools.clone())
|
|
.await?;
|
|
|
|
last_prompt_eval_count = prompt_tokens;
|
|
last_eval_count = eval_tokens;
|
|
|
|
// Sanitize tool call arguments before pushing back into history.
|
|
// Some models occasionally return non-object arguments (bool, string, null)
|
|
// which Ollama rejects when they are re-sent in a subsequent request.
|
|
let mut response = response;
|
|
if let Some(ref mut tool_calls) = response.tool_calls {
|
|
for tc in tool_calls.iter_mut() {
|
|
if !tc.function.arguments.is_object() {
|
|
log::warn!(
|
|
"Tool '{}' returned non-object arguments ({:?}), normalising to {{}}",
|
|
tc.function.name,
|
|
tc.function.arguments
|
|
);
|
|
tc.function.arguments = serde_json::Value::Object(Default::default());
|
|
}
|
|
}
|
|
}
|
|
|
|
messages.push(response.clone());
|
|
|
|
if let Some(ref tool_calls) = response.tool_calls
|
|
&& !tool_calls.is_empty()
|
|
{
|
|
for tool_call in tool_calls {
|
|
log::info!(
|
|
"Agentic tool call [{}]: {} {:?}",
|
|
iteration,
|
|
tool_call.function.name,
|
|
tool_call.function.arguments
|
|
);
|
|
let result = self
|
|
.execute_tool(
|
|
&tool_call.function.name,
|
|
&tool_call.function.arguments,
|
|
&ollama_client,
|
|
&image_base64,
|
|
&file_path,
|
|
&loop_cx,
|
|
)
|
|
.await;
|
|
messages.push(ChatMessage::tool_result(result));
|
|
}
|
|
continue;
|
|
}
|
|
|
|
// No tool calls — this is the final answer
|
|
final_content = response.content;
|
|
break;
|
|
}
|
|
|
|
// If loop exhausted without final answer, ask for one
|
|
if final_content.is_empty() {
|
|
log::info!(
|
|
"Agentic loop exhausted after {} iterations, requesting final answer",
|
|
iterations_used
|
|
);
|
|
messages.push(ChatMessage::user(
|
|
"Based on the context gathered, please write the final photo insight: a title and a detailed personal summary. Write in first person as Cameron.",
|
|
));
|
|
let (final_response, prompt_tokens, eval_tokens) =
|
|
ollama_client
|
|
.chat_with_tools(messages.clone(), vec![])
|
|
.await?;
|
|
last_prompt_eval_count = prompt_tokens;
|
|
last_eval_count = eval_tokens;
|
|
final_content = final_response.content.clone();
|
|
messages.push(final_response);
|
|
}
|
|
|
|
loop_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("iterations_used", iterations_used as i64));
|
|
loop_cx.span().set_status(Status::Ok);
|
|
|
|
// 13. Generate title
|
|
let title = ollama_client
|
|
.generate_photo_title(&final_content, custom_system_prompt.as_deref())
|
|
.await?;
|
|
|
|
log::info!("Agentic generated title: {}", title);
|
|
log::info!(
|
|
"Agentic generated summary ({} chars): {}",
|
|
final_content.len(),
|
|
&final_content[..final_content.len().min(200)]
|
|
);
|
|
|
|
// 14. Serialize the full message history for training data
|
|
let training_messages = match serde_json::to_string(&messages) {
|
|
Ok(json) => Some(json),
|
|
Err(e) => {
|
|
log::warn!("Failed to serialize training messages: {}", e);
|
|
None
|
|
}
|
|
};
|
|
|
|
// 15. Store insight (returns the persisted row including its new id)
|
|
let insight = InsertPhotoInsight {
|
|
library_id: crate::libraries::PRIMARY_LIBRARY_ID,
|
|
file_path: file_path.to_string(),
|
|
title,
|
|
summary: final_content,
|
|
generated_at: Utc::now().timestamp(),
|
|
model_version: ollama_client.primary_model.clone(),
|
|
is_current: true,
|
|
training_messages,
|
|
};
|
|
|
|
let stored = {
|
|
let mut dao = self.insight_dao.lock().expect("Unable to lock InsightDao");
|
|
dao.store_insight(&insight_cx, insight)
|
|
.map_err(|e| anyhow::anyhow!("Failed to store agentic insight: {:?}", e))
|
|
};
|
|
|
|
match &stored {
|
|
Ok(_) => {
|
|
log::info!("Successfully stored agentic insight for {}", file_path);
|
|
insight_cx.span().set_status(Status::Ok);
|
|
}
|
|
Err(e) => {
|
|
log::error!("Failed to store agentic insight: {:?}", e);
|
|
insight_cx.span().set_status(Status::error(e.to_string()));
|
|
}
|
|
}
|
|
|
|
let stored_insight = stored?;
|
|
|
|
// 16. Backfill source_insight_id on all facts recorded for this photo during the loop
|
|
{
|
|
let mut kdao = self
|
|
.knowledge_dao
|
|
.lock()
|
|
.expect("Unable to lock KnowledgeDao");
|
|
if let Err(e) = kdao.update_facts_insight_id(&insight_cx, &file_path, stored_insight.id)
|
|
{
|
|
log::warn!(
|
|
"Failed to backfill source_insight_id for {}: {:?}",
|
|
file_path,
|
|
e
|
|
);
|
|
}
|
|
}
|
|
|
|
Ok((last_prompt_eval_count, last_eval_count))
|
|
}
|
|
|
|
/// Reverse geocode GPS coordinates to human-readable place names
|
|
async fn reverse_geocode(&self, lat: f64, lon: f64) -> Option<String> {
|
|
let url = format!(
|
|
"https://nominatim.openstreetmap.org/reverse?format=json&lat={}&lon={}",
|
|
lat, lon
|
|
);
|
|
|
|
log::debug!("Reverse geocoding {}, {} via Nominatim", lat, lon);
|
|
|
|
let client = reqwest::Client::new();
|
|
let response = match client
|
|
.get(&url)
|
|
.header("User-Agent", "ImageAPI/1.0") // Nominatim requires User-Agent
|
|
.send()
|
|
.await
|
|
{
|
|
Ok(resp) => resp,
|
|
Err(e) => {
|
|
log::warn!("Geocoding network error for {}, {}: {}", lat, lon, e);
|
|
return None;
|
|
}
|
|
};
|
|
|
|
if !response.status().is_success() {
|
|
log::warn!(
|
|
"Geocoding HTTP error for {}, {}: {}",
|
|
lat,
|
|
lon,
|
|
response.status()
|
|
);
|
|
return None;
|
|
}
|
|
|
|
let data: NominatimResponse = match response.json().await {
|
|
Ok(d) => d,
|
|
Err(e) => {
|
|
log::warn!("Geocoding JSON parse error for {}, {}: {}", lat, lon, e);
|
|
return None;
|
|
}
|
|
};
|
|
|
|
// Try to build a concise location name
|
|
if let Some(addr) = data.address {
|
|
let mut parts = Vec::new();
|
|
|
|
// Prefer city/town/village
|
|
if let Some(city) = addr.city.or(addr.town).or(addr.village) {
|
|
parts.push(city);
|
|
}
|
|
|
|
// Add state if available
|
|
if let Some(state) = addr.state {
|
|
parts.push(state);
|
|
}
|
|
|
|
if !parts.is_empty() {
|
|
log::info!("Reverse geocoded {}, {} -> {}", lat, lon, parts.join(", "));
|
|
return Some(parts.join(", "));
|
|
}
|
|
}
|
|
|
|
// Fallback to display_name if structured address not available
|
|
if let Some(ref display_name) = data.display_name {
|
|
log::info!(
|
|
"Reverse geocoded {}, {} -> {} (display_name)",
|
|
lat,
|
|
lon,
|
|
display_name
|
|
);
|
|
} else {
|
|
log::warn!("Geocoding returned no address data for {}, {}", lat, lon);
|
|
}
|
|
data.display_name
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn combine_contexts_includes_tags_section_when_tags_present() {
|
|
let result = InsightGenerator::combine_contexts(
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
Some("vacation, hiking, mountains".to_string()),
|
|
);
|
|
assert!(result.contains("## Tags"), "Should include Tags section");
|
|
assert!(
|
|
result.contains("vacation, hiking, mountains"),
|
|
"Should include tag names"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn combine_contexts_omits_tags_section_when_no_tags() {
|
|
let result = InsightGenerator::combine_contexts(
|
|
Some("some messages".to_string()),
|
|
None,
|
|
None,
|
|
None,
|
|
None, // no tags
|
|
);
|
|
assert!(
|
|
!result.contains("## Tags"),
|
|
"Should not include Tags section when None"
|
|
);
|
|
assert!(
|
|
result.contains("## Messages"),
|
|
"Should still include Messages"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn combine_contexts_returns_no_context_message_when_all_none() {
|
|
let result = InsightGenerator::combine_contexts(None, None, None, None, None);
|
|
assert_eq!(result, "No additional context available");
|
|
}
|
|
}
|