diff --git a/src/ai/insight_generator.rs b/src/ai/insight_generator.rs index b47b8cf..e33d262 100644 --- a/src/ai/insight_generator.rs +++ b/src/ai/insight_generator.rs @@ -9,7 +9,7 @@ use std::sync::{Arc, Mutex}; use crate::ai::ollama::OllamaClient; use crate::ai::sms_client::SmsApiClient; use crate::database::models::InsertPhotoInsight; -use crate::database::{DailySummaryDao, ExifDao, InsightDao}; +use crate::database::{CalendarEventDao, DailySummaryDao, ExifDao, InsightDao, LocationHistoryDao, SearchHistoryDao}; use crate::memories::extract_date_from_filename; use crate::otel::global_tracer; use crate::utils::normalize_path; @@ -35,6 +35,12 @@ pub struct InsightGenerator { insight_dao: Arc>>, exif_dao: Arc>>, daily_summary_dao: Arc>>, + + // Google Takeout data sources + calendar_dao: Arc>>, + location_dao: Arc>>, + search_dao: Arc>>, + base_path: String, } @@ -45,6 +51,9 @@ impl InsightGenerator { insight_dao: Arc>>, exif_dao: Arc>>, daily_summary_dao: Arc>>, + calendar_dao: Arc>>, + location_dao: Arc>>, + search_dao: Arc>>, base_path: String, ) -> Self { Self { @@ -53,6 +62,9 @@ impl InsightGenerator { insight_dao, exif_dao, daily_summary_dao, + calendar_dao, + location_dao, + search_dao, base_path, } } @@ -249,6 +261,249 @@ impl InsightGenerator { Ok(formatted) } + /// Haversine distance calculation for GPS proximity (in kilometers) + fn haversine_distance(lat1: f64, lon1: f64, lat2: f64, lon2: f64) -> f64 { + const R: f64 = 6371.0; // Earth radius in km + let d_lat = (lat2 - lat1).to_radians(); + let d_lon = (lon2 - lon1).to_radians(); + let a = (d_lat / 2.0).sin().powi(2) + + lat1.to_radians().cos() * lat2.to_radians().cos() * (d_lon / 2.0).sin().powi(2); + R * 2.0 * a.sqrt().atan2((1.0 - a).sqrt()) + } + + /// Gather calendar context for photo timestamp + /// Uses hybrid time + semantic search (±7 days, ranked by relevance) + async fn gather_calendar_context( + &self, + parent_cx: &opentelemetry::Context, + timestamp: i64, + location: Option<&str>, + ) -> Result> { + let tracer = global_tracer(); + let span = tracer.start_with_context("ai.context.calendar", parent_cx); + let calendar_cx = parent_cx.with_span(span); + + let query_embedding = if let Some(loc) = location { + match self.ollama.generate_embedding(loc).await { + Ok(emb) => Some(emb), + Err(e) => { + log::warn!("Failed to generate embedding for location '{}': {}", loc, e); + None + } + } + } else { + None + }; + + let events = { + let mut dao = self.calendar_dao.lock().expect("Unable to lock CalendarEventDao"); + dao.find_relevant_events_hybrid( + &calendar_cx, + timestamp, + 7, // ±7 days window + query_embedding.as_deref(), + 5, // Top 5 events + ) + .ok() + }; + + calendar_cx.span().set_status(Status::Ok); + + if let Some(events) = events { + if events.is_empty() { + return Ok(None); + } + + let formatted = events + .iter() + .map(|e| { + let date = DateTime::from_timestamp(e.start_time, 0) + .map(|dt| dt.format("%Y-%m-%d %H:%M").to_string()) + .unwrap_or_else(|| "unknown".to_string()); + + let attendees = e.attendees + .as_ref() + .and_then(|a| serde_json::from_str::>(a).ok()) + .map(|list| format!(" (with {})", list.join(", "))) + .unwrap_or_default(); + + format!("[{}] {}{}", date, e.summary, attendees) + }) + .collect::>() + .join("\n"); + + Ok(Some(format!("Calendar events:\n{}", formatted))) + } else { + Ok(None) + } + } + + /// Gather location context for photo timestamp + /// Finds nearest location record (±30 minutes) + async fn gather_location_context( + &self, + parent_cx: &opentelemetry::Context, + timestamp: i64, + exif_gps: Option<(f64, f64)>, + ) -> Result> { + let tracer = global_tracer(); + let span = tracer.start_with_context("ai.context.location", parent_cx); + let location_cx = parent_cx.with_span(span); + + let nearest = { + let mut dao = self.location_dao.lock().expect("Unable to lock LocationHistoryDao"); + dao.find_nearest_location( + &location_cx, + timestamp, + 1800, // ±30 minutes + ) + .ok() + .flatten() + }; + + location_cx.span().set_status(Status::Ok); + + if let Some(loc) = nearest { + // Check if this adds NEW information compared to EXIF + if let Some((exif_lat, exif_lon)) = exif_gps { + let distance = Self::haversine_distance( + exif_lat, + exif_lon, + loc.latitude, + loc.longitude, + ); + + // Only use if it's significantly different (>100m) or EXIF lacks GPS + if distance < 0.1 { + log::info!("Location history matches EXIF GPS ({}m), skipping", (distance * 1000.0) as i32); + return Ok(None); + } + } + + let activity = loc.activity + .as_ref() + .map(|a| format!(" ({})", a)) + .unwrap_or_default(); + + let place = loc.place_name + .as_ref() + .map(|p| format!(" at {}", p)) + .unwrap_or_default(); + + Ok(Some(format!( + "Location history: You were{}{}{}", + activity, + place, + if activity.is_empty() && place.is_empty() { + format!(" near {:.4}, {:.4}", loc.latitude, loc.longitude) + } else { + String::new() + } + ))) + } else { + Ok(None) + } + } + + /// Gather search context for photo date + /// Uses semantic search on queries (±30 days, top 5 relevant) + async fn gather_search_context( + &self, + parent_cx: &opentelemetry::Context, + timestamp: i64, + location: Option<&str>, + contact: Option<&str>, + ) -> Result> { + let tracer = global_tracer(); + let span = tracer.start_with_context("ai.context.search", parent_cx); + let search_cx = parent_cx.with_span(span); + + // Build semantic query from metadata + let query_text = format!( + "searches about {} {} {}", + DateTime::from_timestamp(timestamp, 0) + .map(|dt| dt.format("%B %Y").to_string()) + .unwrap_or_else(|| "".to_string()), + location.unwrap_or(""), + contact.map(|c| format!("involving {}", c)).unwrap_or_default() + ); + + let query_embedding = match self.ollama.generate_embedding(&query_text).await { + Ok(emb) => emb, + Err(e) => { + log::warn!("Failed to generate search embedding: {}", e); + search_cx.span().set_status(Status::Error { + description: e.to_string().into(), + }); + return Ok(None); + } + }; + + let searches = { + let mut dao = self.search_dao.lock().expect("Unable to lock SearchHistoryDao"); + dao.find_relevant_searches_hybrid( + &search_cx, + timestamp, + 30, // ±30 days (wider window than calendar) + Some(&query_embedding), + 5, // Top 5 searches + ) + .ok() + }; + + search_cx.span().set_status(Status::Ok); + + if let Some(searches) = searches { + if searches.is_empty() { + return Ok(None); + } + + let formatted = searches + .iter() + .map(|s| { + let date = DateTime::from_timestamp(s.timestamp, 0) + .map(|dt| dt.format("%Y-%m-%d").to_string()) + .unwrap_or_else(|| "unknown".to_string()); + format!("[{}] \"{}\"", date, s.query) + }) + .collect::>() + .join("\n"); + + Ok(Some(format!("Search history:\n{}", formatted))) + } else { + Ok(None) + } + } + + /// Combine all context sources with equal weight + fn combine_contexts( + sms: Option, + calendar: Option, + location: Option, + search: Option, + ) -> String { + let mut parts = Vec::new(); + + if let Some(s) = sms { + parts.push(format!("## Messages\n{}", s)); + } + if let Some(c) = calendar { + parts.push(format!("## Calendar\n{}", c)); + } + if let Some(l) = location { + parts.push(format!("## Location\n{}", l)); + } + if let Some(s) = search { + parts.push(format!("## Searches\n{}", s)); + } + + if parts.is_empty() { + "No additional context available".to_string() + } else { + parts.join("\n\n") + } + } + /// Generate AI insight for a single photo with optional custom model pub async fn generate_insight_for_photo_with_model( &self, @@ -525,13 +780,50 @@ impl InsightGenerator { retrieval_method ); - // 7. Generate title and summary with Ollama + // 6. Gather Google Takeout context from all sources + let calendar_context = self + .gather_calendar_context(&insight_cx, timestamp, location.as_deref()) + .await + .ok() + .flatten(); + + let exif_gps = exif.as_ref().and_then(|e| { + if let (Some(lat), Some(lon)) = (e.gps_latitude, e.gps_longitude) { + Some((lat as f64, lon as f64)) + } else { + None + } + }); + + let location_context = self + .gather_location_context(&insight_cx, timestamp, exif_gps) + .await + .ok() + .flatten(); + + let search_context = self + .gather_search_context(&insight_cx, timestamp, location.as_deref(), contact.as_deref()) + .await + .ok() + .flatten(); + + // 7. Combine all context sources with equal weight + let combined_context = Self::combine_contexts( + sms_summary, + calendar_context, + location_context, + search_context, + ); + + log::info!("Combined context from all sources ({} chars)", combined_context.len()); + + // 8. Generate title and summary with Ollama (using multi-source context) let title = ollama_client - .generate_photo_title(date_taken, location.as_deref(), sms_summary.as_deref()) + .generate_photo_title(date_taken, location.as_deref(), Some(&combined_context)) .await?; let summary = ollama_client - .generate_photo_summary(date_taken, location.as_deref(), sms_summary.as_deref()) + .generate_photo_summary(date_taken, location.as_deref(), Some(&combined_context)) .await?; log::info!("Generated title: {}", title); @@ -544,7 +836,7 @@ impl InsightGenerator { .span() .set_attribute(KeyValue::new("summary_length", summary.len() as i64)); - // 8. Store in database + // 9. Store in database let insight = InsertPhotoInsight { file_path: file_path.to_string(), title, diff --git a/src/database/mod.rs b/src/database/mod.rs index 43d078c..8e4f52d 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -19,15 +19,15 @@ pub mod schema; pub mod search_dao; pub use calendar_dao::{ - CalendarEvent, CalendarEventDao, InsertCalendarEvent, SqliteCalendarEventDao, + CalendarEventDao, SqliteCalendarEventDao, }; pub use daily_summary_dao::{DailySummaryDao, InsertDailySummary, SqliteDailySummaryDao}; pub use embeddings_dao::{EmbeddingDao, InsertMessageEmbedding}; pub use insights_dao::{InsightDao, SqliteInsightDao}; pub use location_dao::{ - InsertLocationRecord, LocationHistoryDao, LocationRecord, SqliteLocationHistoryDao, + LocationHistoryDao, SqliteLocationHistoryDao, }; -pub use search_dao::{InsertSearchRecord, SearchHistoryDao, SearchRecord, SqliteSearchHistoryDao}; +pub use search_dao::{SearchHistoryDao, SqliteSearchHistoryDao}; pub trait UserDao { fn create_user(&mut self, user: &str, password: &str) -> Option; diff --git a/src/state.rs b/src/state.rs index 50922d2..f744715 100644 --- a/src/state.rs +++ b/src/state.rs @@ -1,6 +1,8 @@ use crate::ai::{InsightGenerator, OllamaClient, SmsApiClient}; use crate::database::{ - DailySummaryDao, ExifDao, InsightDao, SqliteDailySummaryDao, SqliteExifDao, SqliteInsightDao, + CalendarEventDao, DailySummaryDao, ExifDao, InsightDao, LocationHistoryDao, SearchHistoryDao, + SqliteCalendarEventDao, SqliteDailySummaryDao, SqliteExifDao, SqliteInsightDao, + SqliteLocationHistoryDao, SqliteSearchHistoryDao, }; use crate::video::actors::{PlaylistGenerator, StreamActor, VideoPlaylistManager}; use actix::{Actor, Addr}; @@ -96,16 +98,27 @@ impl Default for AppState { let daily_summary_dao: Arc>> = Arc::new(Mutex::new(Box::new(SqliteDailySummaryDao::new()))); + // Initialize Google Takeout DAOs + let calendar_dao: Arc>> = + Arc::new(Mutex::new(Box::new(SqliteCalendarEventDao::new()))); + let location_dao: Arc>> = + Arc::new(Mutex::new(Box::new(SqliteLocationHistoryDao::new()))); + let search_dao: Arc>> = + Arc::new(Mutex::new(Box::new(SqliteSearchHistoryDao::new()))); + // Load base path let base_path = env::var("BASE_PATH").expect("BASE_PATH was not set in the env"); - // Initialize InsightGenerator + // Initialize InsightGenerator with all data sources let insight_generator = InsightGenerator::new( ollama.clone(), sms_client.clone(), insight_dao.clone(), exif_dao.clone(), daily_summary_dao.clone(), + calendar_dao.clone(), + location_dao.clone(), + search_dao.clone(), base_path.clone(), ); @@ -155,7 +168,15 @@ impl AppState { let daily_summary_dao: Arc>> = Arc::new(Mutex::new(Box::new(SqliteDailySummaryDao::new()))); - // Initialize test InsightGenerator + // Initialize test Google Takeout DAOs + let calendar_dao: Arc>> = + Arc::new(Mutex::new(Box::new(SqliteCalendarEventDao::new()))); + let location_dao: Arc>> = + Arc::new(Mutex::new(Box::new(SqliteLocationHistoryDao::new()))); + let search_dao: Arc>> = + Arc::new(Mutex::new(Box::new(SqliteSearchHistoryDao::new()))); + + // Initialize test InsightGenerator with all data sources let base_path_str = base_path.to_string_lossy().to_string(); let insight_generator = InsightGenerator::new( ollama.clone(), @@ -163,6 +184,9 @@ impl AppState { insight_dao.clone(), exif_dao.clone(), daily_summary_dao.clone(), + calendar_dao.clone(), + location_dao.clone(), + search_dao.clone(), base_path_str.clone(), );