Apps/ImageApi
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Phase 3: Integrate Google Takeout context into InsightGenerator

Browse Source 
- Updated InsightGenerator struct with calendar, location, and search DAOs
- Implemented hybrid context gathering methods:
  * gather_calendar_context(): ±7 days with semantic ranking
  * gather_location_context(): ±30 min with GPS proximity check
  * gather_search_context(): ±30 days semantic search
- Added haversine_distance() utility for GPS calculations
- Updated generate_insight_for_photo_with_model() to use multi-source context
- Combined all context sources (SMS + Calendar + Location + Search) with equal weight
- Initialized new DAOs in AppState (both default and test implementations)
- All contexts are optional (graceful degradation if data missing)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Cameron
2026-01-05 14:57:31 -05:00
parent d86b2c3746
commit cd66521c17
3 changed files with 327 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

302
src/ai/insight_generator.rs
View File

@@ -9,7 +9,7 @@ use std::sync::{Arc, Mutex};
use crate::ai::ollama::OllamaClient;
use crate::ai::sms_client::SmsApiClient;
use crate::database::models::InsertPhotoInsight;
use crate::database::{DailySummaryDao, ExifDao, InsightDao};
use crate::database::{CalendarEventDao, DailySummaryDao, ExifDao, InsightDao, LocationHistoryDao, SearchHistoryDao};
use crate::memories::extract_date_from_filename;
use crate::otel::global_tracer;
use crate::utils::normalize_path;
@@ -35,6 +35,12 @@ pub struct InsightGenerator {
insight_dao: Arc<Mutex<Box<dyn InsightDao>>>,
exif_dao: Arc<Mutex<Box<dyn ExifDao>>>,
daily_summary_dao: Arc<Mutex<Box<dyn DailySummaryDao>>>,
// Google Takeout data sources
calendar_dao: Arc<Mutex<Box<dyn CalendarEventDao>>>,
location_dao: Arc<Mutex<Box<dyn LocationHistoryDao>>>,
search_dao: Arc<Mutex<Box<dyn SearchHistoryDao>>>,
base_path: String,
}
@@ -45,6 +51,9 @@ impl InsightGenerator {
insight_dao: Arc<Mutex<Box<dyn InsightDao>>>,
exif_dao: Arc<Mutex<Box<dyn ExifDao>>>,
daily_summary_dao: Arc<Mutex<Box<dyn DailySummaryDao>>>,
calendar_dao: Arc<Mutex<Box<dyn CalendarEventDao>>>,
location_dao: Arc<Mutex<Box<dyn LocationHistoryDao>>>,
search_dao: Arc<Mutex<Box<dyn SearchHistoryDao>>>,
base_path: String,
) -> Self {
Self {
@@ -53,6 +62,9 @@ impl InsightGenerator {
insight_dao,
exif_dao,
daily_summary_dao,
calendar_dao,
location_dao,
search_dao,
base_path,
}
}
@@ -249,6 +261,249 @@ impl InsightGenerator {
Ok(formatted)
}
/// Haversine distance calculation for GPS proximity (in kilometers)
fn haversine_distance(lat1: f64, lon1: f64, lat2: f64, lon2: f64) -> f64 {
const R: f64 = 6371.0; // Earth radius in km
let d_lat = (lat2 - lat1).to_radians();
let d_lon = (lon2 - lon1).to_radians();
let a = (d_lat / 2.0).sin().powi(2)
+ lat1.to_radians().cos() * lat2.to_radians().cos() * (d_lon / 2.0).sin().powi(2);
R * 2.0 * a.sqrt().atan2((1.0 - a).sqrt())
}
/// Gather calendar context for photo timestamp
/// Uses hybrid time + semantic search (±7 days, ranked by relevance)
async fn gather_calendar_context(
&self,
parent_cx: &opentelemetry::Context,
timestamp: i64,
location: Option<&str>,
) -> Result<Option<String>> {
let tracer = global_tracer();
let span = tracer.start_with_context("ai.context.calendar", parent_cx);
let calendar_cx = parent_cx.with_span(span);
let query_embedding = if let Some(loc) = location {
match self.ollama.generate_embedding(loc).await {
Ok(emb) => Some(emb),
Err(e) => {
log::warn!("Failed to generate embedding for location '{}': {}", loc, e);
None
}
}
} else {
None
};
let events = {
let mut dao = self.calendar_dao.lock().expect("Unable to lock CalendarEventDao");
dao.find_relevant_events_hybrid(
&calendar_cx,
timestamp,
7, // ±7 days window
query_embedding.as_deref(),
5, // Top 5 events
)
.ok()
};
calendar_cx.span().set_status(Status::Ok);
if let Some(events) = events {
if events.is_empty() {
return Ok(None);
}
let formatted = events
.iter()
.map(|e| {
let date = DateTime::from_timestamp(e.start_time, 0)
.map(|dt| dt.format("%Y-%m-%d %H:%M").to_string())
.unwrap_or_else(|| "unknown".to_string());
let attendees = e.attendees
.as_ref()
.and_then(|a| serde_json::from_str::<Vec<String>>(a).ok())
.map(|list| format!(" (with {})", list.join(", ")))
.unwrap_or_default();
format!("[{}] {}{}", date, e.summary, attendees)
})
.collect::<Vec<_>>()
.join("\n");
Ok(Some(format!("Calendar events:\n{}", formatted)))
} else {
Ok(None)
}
}
/// Gather location context for photo timestamp
/// Finds nearest location record (±30 minutes)
async fn gather_location_context(
&self,
parent_cx: &opentelemetry::Context,
timestamp: i64,
exif_gps: Option<(f64, f64)>,
) -> Result<Option<String>> {
let tracer = global_tracer();
let span = tracer.start_with_context("ai.context.location", parent_cx);
let location_cx = parent_cx.with_span(span);
let nearest = {
let mut dao = self.location_dao.lock().expect("Unable to lock LocationHistoryDao");
dao.find_nearest_location(
&location_cx,
timestamp,
1800, // ±30 minutes
)
.ok()
.flatten()
};
location_cx.span().set_status(Status::Ok);
if let Some(loc) = nearest {
// Check if this adds NEW information compared to EXIF
if let Some((exif_lat, exif_lon)) = exif_gps {
let distance = Self::haversine_distance(
exif_lat,
exif_lon,
loc.latitude,
loc.longitude,
);
// Only use if it's significantly different (>100m) or EXIF lacks GPS
if distance < 0.1 {
log::info!("Location history matches EXIF GPS ({}m), skipping", (distance * 1000.0) as i32);
return Ok(None);
}
}
let activity = loc.activity
.as_ref()
.map(|a| format!(" ({})", a))
.unwrap_or_default();
let place = loc.place_name
.as_ref()
.map(|p| format!(" at {}", p))
.unwrap_or_default();
Ok(Some(format!(
"Location history: You were{}{}{}",
activity,
place,
if activity.is_empty() && place.is_empty() {
format!(" near {:.4}, {:.4}", loc.latitude, loc.longitude)
} else {
String::new()
}
)))
} else {
Ok(None)
}
}
/// Gather search context for photo date
/// Uses semantic search on queries (±30 days, top 5 relevant)
async fn gather_search_context(
&self,
parent_cx: &opentelemetry::Context,
timestamp: i64,
location: Option<&str>,
contact: Option<&str>,
) -> Result<Option<String>> {
let tracer = global_tracer();
let span = tracer.start_with_context("ai.context.search", parent_cx);
let search_cx = parent_cx.with_span(span);
// Build semantic query from metadata
let query_text = format!(
"searches about {} {} {}",
DateTime::from_timestamp(timestamp, 0)
.map(|dt| dt.format("%B %Y").to_string())
.unwrap_or_else(|| "".to_string()),
location.unwrap_or(""),
contact.map(|c| format!("involving {}", c)).unwrap_or_default()
);
let query_embedding = match self.ollama.generate_embedding(&query_text).await {
Ok(emb) => emb,
Err(e) => {
log::warn!("Failed to generate search embedding: {}", e);
search_cx.span().set_status(Status::Error {
description: e.to_string().into(),
});
return Ok(None);
}
};
let searches = {
let mut dao = self.search_dao.lock().expect("Unable to lock SearchHistoryDao");
dao.find_relevant_searches_hybrid(
&search_cx,
timestamp,
30, // ±30 days (wider window than calendar)
Some(&query_embedding),
5, // Top 5 searches
)
.ok()
};
search_cx.span().set_status(Status::Ok);
if let Some(searches) = searches {
if searches.is_empty() {
return Ok(None);
}
let formatted = searches
.iter()
.map(|s| {
let date = DateTime::from_timestamp(s.timestamp, 0)
.map(|dt| dt.format("%Y-%m-%d").to_string())
.unwrap_or_else(|| "unknown".to_string());
format!("[{}] \"{}\"", date, s.query)
})
.collect::<Vec<_>>()
.join("\n");
Ok(Some(format!("Search history:\n{}", formatted)))
} else {
Ok(None)
}
}
/// Combine all context sources with equal weight
fn combine_contexts(
sms: Option<String>,
calendar: Option<String>,
location: Option<String>,
search: Option<String>,
) -> String {
let mut parts = Vec::new();
if let Some(s) = sms {
parts.push(format!("## Messages\n{}", s));
}
if let Some(c) = calendar {
parts.push(format!("## Calendar\n{}", c));
}
if let Some(l) = location {
parts.push(format!("## Location\n{}", l));
}
if let Some(s) = search {
parts.push(format!("## Searches\n{}", s));
}
if parts.is_empty() {
"No additional context available".to_string()
} else {
parts.join("\n\n")
}
}
/// Generate AI insight for a single photo with optional custom model
pub async fn generate_insight_for_photo_with_model(
&self,
@@ -525,13 +780,50 @@ impl InsightGenerator {
retrieval_method
);
// 7. Generate title and summary with Ollama
// 6. Gather Google Takeout context from all sources
let calendar_context = self
.gather_calendar_context(&insight_cx, timestamp, location.as_deref())
.await
.ok()
.flatten();
let exif_gps = exif.as_ref().and_then(|e| {
if let (Some(lat), Some(lon)) = (e.gps_latitude, e.gps_longitude) {
Some((lat as f64, lon as f64))
} else {
None
}
});
let location_context = self
.gather_location_context(&insight_cx, timestamp, exif_gps)
.await
.ok()
.flatten();
let search_context = self
.gather_search_context(&insight_cx, timestamp, location.as_deref(), contact.as_deref())
.await
.ok()
.flatten();
// 7. Combine all context sources with equal weight
let combined_context = Self::combine_contexts(
sms_summary,
calendar_context,
location_context,
search_context,
);
log::info!("Combined context from all sources ({} chars)", combined_context.len());
// 8. Generate title and summary with Ollama (using multi-source context)
let title = ollama_client
.generate_photo_title(date_taken, location.as_deref(), sms_summary.as_deref())
.generate_photo_title(date_taken, location.as_deref(), Some(&combined_context))
.await?;
let summary = ollama_client
.generate_photo_summary(date_taken, location.as_deref(), sms_summary.as_deref())
.generate_photo_summary(date_taken, location.as_deref(), Some(&combined_context))
.await?;
log::info!("Generated title: {}", title);
@@ -544,7 +836,7 @@ impl InsightGenerator {
.span()
.set_attribute(KeyValue::new("summary_length", summary.len() as i64));
// 8. Store in database
// 9. Store in database
let insight = InsertPhotoInsight {
file_path: file_path.to_string(),
title,

6
src/database/mod.rs
View File

@@ -19,15 +19,15 @@ pub mod schema;
pub mod search_dao;
pub use calendar_dao::{
CalendarEvent, CalendarEventDao, InsertCalendarEvent, SqliteCalendarEventDao,
CalendarEventDao, SqliteCalendarEventDao,
};
pub use daily_summary_dao::{DailySummaryDao, InsertDailySummary, SqliteDailySummaryDao};
pub use embeddings_dao::{EmbeddingDao, InsertMessageEmbedding};
pub use insights_dao::{InsightDao, SqliteInsightDao};
pub use location_dao::{
InsertLocationRecord, LocationHistoryDao, LocationRecord, SqliteLocationHistoryDao,
LocationHistoryDao, SqliteLocationHistoryDao,
};
pub use search_dao::{InsertSearchRecord, SearchHistoryDao, SearchRecord, SqliteSearchHistoryDao};
pub use search_dao::{SearchHistoryDao, SqliteSearchHistoryDao};
pub trait UserDao {
fn create_user(&mut self, user: &str, password: &str) -> Option<User>;

30
src/state.rs
View File

@@ -1,6 +1,8 @@
use crate::ai::{InsightGenerator, OllamaClient, SmsApiClient};
use crate::database::{
DailySummaryDao, ExifDao, InsightDao, SqliteDailySummaryDao, SqliteExifDao, SqliteInsightDao,
CalendarEventDao, DailySummaryDao, ExifDao, InsightDao, LocationHistoryDao, SearchHistoryDao,
SqliteCalendarEventDao, SqliteDailySummaryDao, SqliteExifDao, SqliteInsightDao,
SqliteLocationHistoryDao, SqliteSearchHistoryDao,
};
use crate::video::actors::{PlaylistGenerator, StreamActor, VideoPlaylistManager};
use actix::{Actor, Addr};
@@ -96,16 +98,27 @@ impl Default for AppState {
let daily_summary_dao: Arc<Mutex<Box<dyn DailySummaryDao>>> =
Arc::new(Mutex::new(Box::new(SqliteDailySummaryDao::new())));
// Initialize Google Takeout DAOs
let calendar_dao: Arc<Mutex<Box<dyn CalendarEventDao>>> =
Arc::new(Mutex::new(Box::new(SqliteCalendarEventDao::new())));
let location_dao: Arc<Mutex<Box<dyn LocationHistoryDao>>> =
Arc::new(Mutex::new(Box::new(SqliteLocationHistoryDao::new())));
let search_dao: Arc<Mutex<Box<dyn SearchHistoryDao>>> =
Arc::new(Mutex::new(Box::new(SqliteSearchHistoryDao::new())));
// Load base path
let base_path = env::var("BASE_PATH").expect("BASE_PATH was not set in the env");
// Initialize InsightGenerator
// Initialize InsightGenerator with all data sources
let insight_generator = InsightGenerator::new(
ollama.clone(),
sms_client.clone(),
insight_dao.clone(),
exif_dao.clone(),
daily_summary_dao.clone(),
calendar_dao.clone(),
location_dao.clone(),
search_dao.clone(),
base_path.clone(),
);
@@ -155,7 +168,15 @@ impl AppState {
let daily_summary_dao: Arc<Mutex<Box<dyn DailySummaryDao>>> =
Arc::new(Mutex::new(Box::new(SqliteDailySummaryDao::new())));
// Initialize test InsightGenerator
// Initialize test Google Takeout DAOs
let calendar_dao: Arc<Mutex<Box<dyn CalendarEventDao>>> =
Arc::new(Mutex::new(Box::new(SqliteCalendarEventDao::new())));
let location_dao: Arc<Mutex<Box<dyn LocationHistoryDao>>> =
Arc::new(Mutex::new(Box::new(SqliteLocationHistoryDao::new())));
let search_dao: Arc<Mutex<Box<dyn SearchHistoryDao>>> =
Arc::new(Mutex::new(Box::new(SqliteSearchHistoryDao::new())));
// Initialize test InsightGenerator with all data sources
let base_path_str = base_path.to_string_lossy().to_string();
let insight_generator = InsightGenerator::new(
ollama.clone(),
@@ -163,6 +184,9 @@ impl AppState {
insight_dao.clone(),
exif_dao.clone(),
daily_summary_dao.clone(),
calendar_dao.clone(),
location_dao.clone(),
search_dao.clone(),
base_path_str.clone(),
);