1307 lines
46 KiB
Rust
1307 lines
46 KiB
Rust
use anyhow::Result;
|
|
use base64::Engine as _;
|
|
use chrono::{DateTime, Utc};
|
|
use image::ImageFormat;
|
|
use opentelemetry::KeyValue;
|
|
use opentelemetry::trace::{Span, Status, TraceContextExt, Tracer};
|
|
use serde::Deserialize;
|
|
use std::fs::File;
|
|
use std::io::Cursor;
|
|
use std::sync::{Arc, Mutex};
|
|
|
|
use crate::ai::ollama::OllamaClient;
|
|
use crate::ai::sms_client::SmsApiClient;
|
|
use crate::database::models::InsertPhotoInsight;
|
|
use crate::database::{
|
|
CalendarEventDao, DailySummaryDao, ExifDao, InsightDao, LocationHistoryDao, SearchHistoryDao,
|
|
};
|
|
use crate::memories::extract_date_from_filename;
|
|
use crate::otel::global_tracer;
|
|
use crate::utils::normalize_path;
|
|
|
|
#[derive(Deserialize)]
|
|
struct NominatimResponse {
|
|
display_name: Option<String>,
|
|
address: Option<NominatimAddress>,
|
|
}
|
|
|
|
#[derive(Deserialize)]
|
|
struct NominatimAddress {
|
|
city: Option<String>,
|
|
town: Option<String>,
|
|
village: Option<String>,
|
|
state: Option<String>,
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
pub struct InsightGenerator {
|
|
ollama: OllamaClient,
|
|
sms_client: SmsApiClient,
|
|
insight_dao: Arc<Mutex<Box<dyn InsightDao>>>,
|
|
exif_dao: Arc<Mutex<Box<dyn ExifDao>>>,
|
|
daily_summary_dao: Arc<Mutex<Box<dyn DailySummaryDao>>>,
|
|
|
|
// Google Takeout data sources
|
|
calendar_dao: Arc<Mutex<Box<dyn CalendarEventDao>>>,
|
|
location_dao: Arc<Mutex<Box<dyn LocationHistoryDao>>>,
|
|
search_dao: Arc<Mutex<Box<dyn SearchHistoryDao>>>,
|
|
|
|
base_path: String,
|
|
}
|
|
|
|
impl InsightGenerator {
|
|
pub fn new(
|
|
ollama: OllamaClient,
|
|
sms_client: SmsApiClient,
|
|
insight_dao: Arc<Mutex<Box<dyn InsightDao>>>,
|
|
exif_dao: Arc<Mutex<Box<dyn ExifDao>>>,
|
|
daily_summary_dao: Arc<Mutex<Box<dyn DailySummaryDao>>>,
|
|
calendar_dao: Arc<Mutex<Box<dyn CalendarEventDao>>>,
|
|
location_dao: Arc<Mutex<Box<dyn LocationHistoryDao>>>,
|
|
search_dao: Arc<Mutex<Box<dyn SearchHistoryDao>>>,
|
|
base_path: String,
|
|
) -> Self {
|
|
Self {
|
|
ollama,
|
|
sms_client,
|
|
insight_dao,
|
|
exif_dao,
|
|
daily_summary_dao,
|
|
calendar_dao,
|
|
location_dao,
|
|
search_dao,
|
|
base_path,
|
|
}
|
|
}
|
|
|
|
/// Extract contact name from file path
|
|
/// e.g., "Sarah/img.jpeg" -> Some("Sarah")
|
|
/// e.g., "img.jpeg" -> None
|
|
fn extract_contact_from_path(file_path: &str) -> Option<String> {
|
|
use std::path::Path;
|
|
|
|
let path = Path::new(file_path);
|
|
let components: Vec<_> = path.components().collect();
|
|
|
|
// If path has at least 2 components (directory + file), extract first directory
|
|
if components.len() >= 2
|
|
&& let Some(component) = components.first()
|
|
&& let Some(os_str) = component.as_os_str().to_str() {
|
|
return Some(os_str.to_string());
|
|
}
|
|
|
|
None
|
|
}
|
|
|
|
/// Load image file, resize it, and encode as base64 for vision models
|
|
/// Resizes to max 1024px on longest edge to reduce context usage
|
|
fn load_image_as_base64(&self, file_path: &str) -> Result<String> {
|
|
use image::imageops::FilterType;
|
|
use std::path::Path;
|
|
|
|
let full_path = Path::new(&self.base_path).join(file_path);
|
|
|
|
log::debug!("Loading image for vision model: {:?}", full_path);
|
|
|
|
// Open and decode the image
|
|
let img = image::open(&full_path)
|
|
.map_err(|e| anyhow::anyhow!("Failed to open image file: {}", e))?;
|
|
|
|
let (original_width, original_height) = (img.width(), img.height());
|
|
|
|
// Resize to max 1024px on longest edge
|
|
let resized = img.resize(1024, 1024, FilterType::Lanczos3);
|
|
|
|
log::debug!(
|
|
"Resized image from {}x{} to {}x{}",
|
|
original_width,
|
|
original_height,
|
|
resized.width(),
|
|
resized.height()
|
|
);
|
|
|
|
// Encode as JPEG at 85% quality
|
|
let mut buffer = Vec::new();
|
|
let mut cursor = Cursor::new(&mut buffer);
|
|
resized
|
|
.write_to(&mut cursor, ImageFormat::Jpeg)
|
|
.map_err(|e| anyhow::anyhow!("Failed to encode image as JPEG: {}", e))?;
|
|
|
|
let base64_string = base64::engine::general_purpose::STANDARD.encode(&buffer);
|
|
|
|
log::debug!(
|
|
"Encoded image as base64 ({} bytes -> {} chars)",
|
|
buffer.len(),
|
|
base64_string.len()
|
|
);
|
|
|
|
Ok(base64_string)
|
|
}
|
|
|
|
/// Find relevant messages using RAG, excluding recent messages (>30 days ago)
|
|
/// This prevents RAG from returning messages already in the immediate time window
|
|
async fn find_relevant_messages_rag_historical(
|
|
&self,
|
|
parent_cx: &opentelemetry::Context,
|
|
date: chrono::NaiveDate,
|
|
location: Option<&str>,
|
|
contact: Option<&str>,
|
|
topics: Option<&[String]>,
|
|
limit: usize,
|
|
) -> Result<Vec<String>> {
|
|
let tracer = global_tracer();
|
|
let span = tracer.start_with_context("ai.rag.filter_historical", parent_cx);
|
|
let filter_cx = parent_cx.with_span(span);
|
|
|
|
filter_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("date", date.to_string()));
|
|
filter_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("limit", limit as i64));
|
|
filter_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("exclusion_window_days", 30));
|
|
if let Some(t) = topics {
|
|
filter_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("topics", t.join(", ")));
|
|
}
|
|
|
|
let query_results = self
|
|
.find_relevant_messages_rag(date, location, contact, topics, limit * 2)
|
|
.await?;
|
|
|
|
filter_cx.span().set_attribute(KeyValue::new(
|
|
"rag_results_count",
|
|
query_results.len() as i64,
|
|
));
|
|
|
|
// Filter out messages from within 30 days of the photo date
|
|
let photo_timestamp = date
|
|
.and_hms_opt(12, 0, 0)
|
|
.ok_or_else(|| anyhow::anyhow!("Invalid date"))?
|
|
.and_utc()
|
|
.timestamp();
|
|
let exclusion_window = 30 * 86400; // 30 days in seconds
|
|
|
|
let historical_only: Vec<String> = query_results
|
|
.into_iter()
|
|
.filter(|msg| {
|
|
// Extract date from formatted daily summary "[2024-08-15] Contact ..."
|
|
if let Some(bracket_end) = msg.find(']')
|
|
&& let Some(date_str) = msg.get(1..bracket_end) {
|
|
// Parse just the date (daily summaries don't have time)
|
|
if let Ok(msg_date) =
|
|
chrono::NaiveDate::parse_from_str(date_str, "%Y-%m-%d")
|
|
{
|
|
let msg_timestamp = msg_date
|
|
.and_hms_opt(12, 0, 0)
|
|
.unwrap()
|
|
.and_utc()
|
|
.timestamp();
|
|
let time_diff = (photo_timestamp - msg_timestamp).abs();
|
|
return time_diff > exclusion_window;
|
|
}
|
|
}
|
|
false
|
|
})
|
|
.take(limit)
|
|
.collect();
|
|
|
|
log::info!(
|
|
"Found {} historical messages (>30 days from photo date)",
|
|
historical_only.len()
|
|
);
|
|
|
|
filter_cx.span().set_attribute(KeyValue::new(
|
|
"historical_results_count",
|
|
historical_only.len() as i64,
|
|
));
|
|
filter_cx.span().set_status(Status::Ok);
|
|
|
|
Ok(historical_only)
|
|
}
|
|
|
|
/// Find relevant daily summaries using RAG (semantic search)
|
|
/// Returns formatted daily summary strings for LLM context
|
|
async fn find_relevant_messages_rag(
|
|
&self,
|
|
date: chrono::NaiveDate,
|
|
location: Option<&str>,
|
|
contact: Option<&str>,
|
|
topics: Option<&[String]>,
|
|
limit: usize,
|
|
) -> Result<Vec<String>> {
|
|
let tracer = global_tracer();
|
|
let current_cx = opentelemetry::Context::current();
|
|
let mut span = tracer.start_with_context("ai.rag.search_daily_summaries", ¤t_cx);
|
|
span.set_attribute(KeyValue::new("date", date.to_string()));
|
|
span.set_attribute(KeyValue::new("limit", limit as i64));
|
|
if let Some(loc) = location {
|
|
span.set_attribute(KeyValue::new("location", loc.to_string()));
|
|
}
|
|
if let Some(c) = contact {
|
|
span.set_attribute(KeyValue::new("contact", c.to_string()));
|
|
}
|
|
|
|
// Build query string - prioritize topics if available (semantically meaningful)
|
|
let query = if let Some(topics) = topics {
|
|
if !topics.is_empty() {
|
|
// Use topics for semantic search - these are actual content keywords
|
|
let topic_str = topics.join(", ");
|
|
if let Some(c) = contact {
|
|
format!("Conversations about {} with {}", topic_str, c)
|
|
} else {
|
|
format!("Conversations about {}", topic_str)
|
|
}
|
|
} else {
|
|
// Fallback to metadata-based query
|
|
Self::build_metadata_query(date, location, contact)
|
|
}
|
|
} else {
|
|
// Fallback to metadata-based query
|
|
Self::build_metadata_query(date, location, contact)
|
|
};
|
|
|
|
span.set_attribute(KeyValue::new("query", query.clone()));
|
|
|
|
// Create context with this span for child operations
|
|
let search_cx = current_cx.with_span(span);
|
|
|
|
log::info!("========================================");
|
|
log::info!("RAG QUERY: {}", query);
|
|
log::info!("========================================");
|
|
|
|
// Generate embedding for the query
|
|
let query_embedding = self.ollama.generate_embedding(&query).await?;
|
|
|
|
// Search for similar daily summaries with time-based weighting
|
|
// This prioritizes summaries temporally close to the query date
|
|
let mut summary_dao = self
|
|
.daily_summary_dao
|
|
.lock()
|
|
.expect("Unable to lock DailySummaryDao");
|
|
|
|
let date_str = date.format("%Y-%m-%d").to_string();
|
|
let similar_summaries = summary_dao
|
|
.find_similar_summaries_with_time_weight(&search_cx, &query_embedding, &date_str, limit)
|
|
.map_err(|e| anyhow::anyhow!("Failed to find similar summaries: {:?}", e))?;
|
|
|
|
log::info!(
|
|
"Found {} relevant daily summaries via RAG",
|
|
similar_summaries.len()
|
|
);
|
|
|
|
search_cx.span().set_attribute(KeyValue::new(
|
|
"results_count",
|
|
similar_summaries.len() as i64,
|
|
));
|
|
|
|
// Format daily summaries for LLM context
|
|
let formatted = similar_summaries
|
|
.into_iter()
|
|
.map(|s| {
|
|
format!(
|
|
"[{}] {} ({} messages):\n{}",
|
|
s.date, s.contact, s.message_count, s.summary
|
|
)
|
|
})
|
|
.collect();
|
|
|
|
search_cx.span().set_status(Status::Ok);
|
|
|
|
Ok(formatted)
|
|
}
|
|
|
|
/// Build a metadata-based query (fallback when no topics available)
|
|
fn build_metadata_query(
|
|
date: chrono::NaiveDate,
|
|
location: Option<&str>,
|
|
contact: Option<&str>,
|
|
) -> String {
|
|
let mut query_parts = Vec::new();
|
|
|
|
// Add temporal context
|
|
query_parts.push(format!("On {}", date.format("%B %d, %Y")));
|
|
|
|
// Add location if available
|
|
if let Some(loc) = location {
|
|
query_parts.push(format!("at {}", loc));
|
|
}
|
|
|
|
// Add contact context if available
|
|
if let Some(c) = contact {
|
|
query_parts.push(format!("conversation with {}", c));
|
|
}
|
|
|
|
// Add day of week for temporal context
|
|
let weekday = date.format("%A");
|
|
query_parts.push(format!("it was a {}", weekday));
|
|
|
|
query_parts.join(", ")
|
|
}
|
|
|
|
/// Haversine distance calculation for GPS proximity (in kilometers)
|
|
fn haversine_distance(lat1: f64, lon1: f64, lat2: f64, lon2: f64) -> f64 {
|
|
const R: f64 = 6371.0; // Earth radius in km
|
|
let d_lat = (lat2 - lat1).to_radians();
|
|
let d_lon = (lon2 - lon1).to_radians();
|
|
let a = (d_lat / 2.0).sin().powi(2)
|
|
+ lat1.to_radians().cos() * lat2.to_radians().cos() * (d_lon / 2.0).sin().powi(2);
|
|
R * 2.0 * a.sqrt().atan2((1.0 - a).sqrt())
|
|
}
|
|
|
|
/// Gather calendar context for photo timestamp
|
|
/// Uses hybrid time + semantic search (±7 days, ranked by relevance)
|
|
async fn gather_calendar_context(
|
|
&self,
|
|
parent_cx: &opentelemetry::Context,
|
|
timestamp: i64,
|
|
location: Option<&str>,
|
|
) -> Result<Option<String>> {
|
|
let tracer = global_tracer();
|
|
let span = tracer.start_with_context("ai.context.calendar", parent_cx);
|
|
let calendar_cx = parent_cx.with_span(span);
|
|
|
|
let query_embedding = if let Some(loc) = location {
|
|
match self.ollama.generate_embedding(loc).await {
|
|
Ok(emb) => Some(emb),
|
|
Err(e) => {
|
|
log::warn!("Failed to generate embedding for location '{}': {}", loc, e);
|
|
None
|
|
}
|
|
}
|
|
} else {
|
|
None
|
|
};
|
|
|
|
let events = {
|
|
let mut dao = self
|
|
.calendar_dao
|
|
.lock()
|
|
.expect("Unable to lock CalendarEventDao");
|
|
dao.find_relevant_events_hybrid(
|
|
&calendar_cx,
|
|
timestamp,
|
|
7, // ±7 days window
|
|
query_embedding.as_deref(),
|
|
5, // Top 5 events
|
|
)
|
|
.ok()
|
|
};
|
|
|
|
calendar_cx.span().set_status(Status::Ok);
|
|
|
|
if let Some(events) = events {
|
|
if events.is_empty() {
|
|
return Ok(None);
|
|
}
|
|
|
|
let formatted = events
|
|
.iter()
|
|
.map(|e| {
|
|
let date = DateTime::from_timestamp(e.start_time, 0)
|
|
.map(|dt| dt.format("%Y-%m-%d %H:%M").to_string())
|
|
.unwrap_or_else(|| "unknown".to_string());
|
|
|
|
let attendees = e
|
|
.attendees
|
|
.as_ref()
|
|
.and_then(|a| serde_json::from_str::<Vec<String>>(a).ok())
|
|
.map(|list| format!(" (with {})", list.join(", ")))
|
|
.unwrap_or_default();
|
|
|
|
format!("[{}] {}{}", date, e.summary, attendees)
|
|
})
|
|
.collect::<Vec<_>>()
|
|
.join("\n");
|
|
|
|
Ok(Some(format!("Calendar events:\n{}", formatted)))
|
|
} else {
|
|
Ok(None)
|
|
}
|
|
}
|
|
|
|
/// Gather location context for photo timestamp
|
|
/// Finds nearest location record (±30 minutes)
|
|
async fn gather_location_context(
|
|
&self,
|
|
parent_cx: &opentelemetry::Context,
|
|
timestamp: i64,
|
|
exif_gps: Option<(f64, f64)>,
|
|
) -> Result<Option<String>> {
|
|
let tracer = global_tracer();
|
|
let span = tracer.start_with_context("ai.context.location", parent_cx);
|
|
let location_cx = parent_cx.with_span(span);
|
|
|
|
let nearest = {
|
|
let mut dao = self
|
|
.location_dao
|
|
.lock()
|
|
.expect("Unable to lock LocationHistoryDao");
|
|
dao.find_nearest_location(
|
|
&location_cx,
|
|
timestamp,
|
|
10800, // ±3 hours (more realistic for photo timing)
|
|
)
|
|
.ok()
|
|
.flatten()
|
|
};
|
|
|
|
location_cx.span().set_status(Status::Ok);
|
|
|
|
if let Some(loc) = nearest {
|
|
// Check if this adds NEW information compared to EXIF
|
|
if let Some((exif_lat, exif_lon)) = exif_gps {
|
|
let distance =
|
|
Self::haversine_distance(exif_lat, exif_lon, loc.latitude, loc.longitude);
|
|
|
|
// Skip only if very close AND no useful activity/place info
|
|
// Allow activity context even if coordinates match
|
|
if distance < 0.5 && loc.activity.is_none() && loc.place_name.is_none() {
|
|
log::debug!(
|
|
"Location history matches EXIF GPS ({}m) with no extra context, skipping",
|
|
(distance * 1000.0) as i32
|
|
);
|
|
return Ok(None);
|
|
} else if distance < 0.5 {
|
|
log::debug!(
|
|
"Location history close to EXIF ({}m) but has activity/place info",
|
|
(distance * 1000.0) as i32
|
|
);
|
|
}
|
|
}
|
|
|
|
let activity = loc
|
|
.activity
|
|
.as_ref()
|
|
.map(|a| format!(" ({})", a))
|
|
.unwrap_or_default();
|
|
|
|
let place = loc
|
|
.place_name
|
|
.as_ref()
|
|
.map(|p| format!(" at {}", p))
|
|
.unwrap_or_default();
|
|
|
|
Ok(Some(format!(
|
|
"Location history: You were{}{}{}",
|
|
activity,
|
|
place,
|
|
if activity.is_empty() && place.is_empty() {
|
|
format!(" near {:.4}, {:.4}", loc.latitude, loc.longitude)
|
|
} else {
|
|
String::new()
|
|
}
|
|
)))
|
|
} else {
|
|
Ok(None)
|
|
}
|
|
}
|
|
|
|
/// Gather search context for photo date
|
|
/// Uses semantic search on queries (±30 days, top 5 relevant)
|
|
async fn gather_search_context(
|
|
&self,
|
|
parent_cx: &opentelemetry::Context,
|
|
timestamp: i64,
|
|
location: Option<&str>,
|
|
contact: Option<&str>,
|
|
) -> Result<Option<String>> {
|
|
let tracer = global_tracer();
|
|
let span = tracer.start_with_context("ai.context.search", parent_cx);
|
|
let search_cx = parent_cx.with_span(span);
|
|
|
|
// Build semantic query from metadata
|
|
let query_text = format!(
|
|
"searches about {} {} {}",
|
|
DateTime::from_timestamp(timestamp, 0)
|
|
.map(|dt| dt.format("%B %Y").to_string())
|
|
.unwrap_or_default(),
|
|
location.unwrap_or(""),
|
|
contact
|
|
.map(|c| format!("involving {}", c))
|
|
.unwrap_or_default()
|
|
);
|
|
|
|
let query_embedding = match self.ollama.generate_embedding(&query_text).await {
|
|
Ok(emb) => emb,
|
|
Err(e) => {
|
|
log::warn!("Failed to generate search embedding: {}", e);
|
|
search_cx.span().set_status(Status::Error {
|
|
description: e.to_string().into(),
|
|
});
|
|
return Ok(None);
|
|
}
|
|
};
|
|
|
|
let searches = {
|
|
let mut dao = self
|
|
.search_dao
|
|
.lock()
|
|
.expect("Unable to lock SearchHistoryDao");
|
|
dao.find_relevant_searches_hybrid(
|
|
&search_cx,
|
|
timestamp,
|
|
30, // ±30 days (wider window than calendar)
|
|
Some(&query_embedding),
|
|
5, // Top 5 searches
|
|
)
|
|
.ok()
|
|
};
|
|
|
|
search_cx.span().set_status(Status::Ok);
|
|
|
|
if let Some(searches) = searches {
|
|
if searches.is_empty() {
|
|
log::warn!(
|
|
"No relevant searches found for photo timestamp {}",
|
|
timestamp
|
|
);
|
|
return Ok(None);
|
|
}
|
|
|
|
let formatted = searches
|
|
.iter()
|
|
.map(|s| {
|
|
let date = DateTime::from_timestamp(s.timestamp, 0)
|
|
.map(|dt| dt.format("%Y-%m-%d").to_string())
|
|
.unwrap_or_else(|| "unknown".to_string());
|
|
format!("[{}] \"{}\"", date, s.query)
|
|
})
|
|
.collect::<Vec<_>>()
|
|
.join("\n");
|
|
|
|
Ok(Some(format!("Search history:\n{}", formatted)))
|
|
} else {
|
|
Ok(None)
|
|
}
|
|
}
|
|
|
|
/// Combine all context sources with equal weight
|
|
fn combine_contexts(
|
|
sms: Option<String>,
|
|
calendar: Option<String>,
|
|
location: Option<String>,
|
|
search: Option<String>,
|
|
) -> String {
|
|
let mut parts = Vec::new();
|
|
|
|
if let Some(s) = sms {
|
|
parts.push(format!("## Messages\n{}", s));
|
|
}
|
|
if let Some(c) = calendar {
|
|
parts.push(format!("## Calendar\n{}", c));
|
|
}
|
|
if let Some(l) = location {
|
|
parts.push(format!("## Location\n{}", l));
|
|
}
|
|
if let Some(s) = search {
|
|
parts.push(format!("## Searches\n{}", s));
|
|
}
|
|
|
|
if parts.is_empty() {
|
|
"No additional context available".to_string()
|
|
} else {
|
|
parts.join("\n\n")
|
|
}
|
|
}
|
|
|
|
/// Generate AI insight for a single photo with custom configuration
|
|
pub async fn generate_insight_for_photo_with_config(
|
|
&self,
|
|
file_path: &str,
|
|
custom_model: Option<String>,
|
|
custom_system_prompt: Option<String>,
|
|
num_ctx: Option<i32>,
|
|
) -> Result<()> {
|
|
let tracer = global_tracer();
|
|
let current_cx = opentelemetry::Context::current();
|
|
let mut span = tracer.start_with_context("ai.insight.generate", ¤t_cx);
|
|
|
|
// Normalize path to ensure consistent forward slashes in database
|
|
let file_path = normalize_path(file_path);
|
|
log::info!("Generating insight for photo: {}", file_path);
|
|
|
|
span.set_attribute(KeyValue::new("file_path", file_path.clone()));
|
|
|
|
// Create custom Ollama client if model is specified
|
|
let mut ollama_client = if let Some(model) = custom_model {
|
|
log::info!("Using custom model: {}", model);
|
|
span.set_attribute(KeyValue::new("custom_model", model.clone()));
|
|
OllamaClient::new(
|
|
self.ollama.primary_url.clone(),
|
|
self.ollama.fallback_url.clone(),
|
|
model.clone(),
|
|
Some(model), // Use the same custom model for fallback server
|
|
)
|
|
} else {
|
|
span.set_attribute(KeyValue::new("model", self.ollama.primary_model.clone()));
|
|
self.ollama.clone()
|
|
};
|
|
|
|
// Set context size if specified
|
|
if let Some(ctx) = num_ctx {
|
|
log::info!("Using custom context size: {}", ctx);
|
|
span.set_attribute(KeyValue::new("num_ctx", ctx as i64));
|
|
ollama_client.set_num_ctx(Some(ctx));
|
|
}
|
|
|
|
// Create context with this span for child operations
|
|
let insight_cx = current_cx.with_span(span);
|
|
|
|
// 1. Get EXIF data for the photo
|
|
let exif = {
|
|
let mut exif_dao = self.exif_dao.lock().expect("Unable to lock ExifDao");
|
|
exif_dao
|
|
.get_exif(&insight_cx, &file_path)
|
|
.map_err(|e| anyhow::anyhow!("Failed to get EXIF: {:?}", e))?
|
|
};
|
|
|
|
// Get full timestamp for proximity-based message filtering
|
|
let timestamp = if let Some(ts) = exif.as_ref().and_then(|e| e.date_taken) {
|
|
ts
|
|
} else {
|
|
log::warn!("No date_taken in EXIF for {}, trying filename", file_path);
|
|
|
|
extract_date_from_filename(&file_path)
|
|
.map(|dt| dt.timestamp())
|
|
.or_else(|| {
|
|
// Combine base_path with file_path to get full path
|
|
let full_path = std::path::Path::new(&self.base_path).join(&file_path);
|
|
File::open(&full_path)
|
|
.and_then(|f| f.metadata())
|
|
.and_then(|m| m.created().or(m.modified()))
|
|
.map(|t| DateTime::<Utc>::from(t).timestamp())
|
|
.inspect_err(|e| {
|
|
log::warn!(
|
|
"Failed to get file timestamp for insight {}: {}",
|
|
file_path,
|
|
e
|
|
)
|
|
})
|
|
.ok()
|
|
})
|
|
.unwrap_or_else(|| Utc::now().timestamp())
|
|
};
|
|
|
|
let date_taken = DateTime::from_timestamp(timestamp, 0)
|
|
.map(|dt| dt.date_naive())
|
|
.unwrap_or_else(|| Utc::now().date_naive());
|
|
|
|
// 3. Extract contact name from file path
|
|
let contact = Self::extract_contact_from_path(&file_path);
|
|
log::info!("Extracted contact from path: {:?}", contact);
|
|
|
|
insight_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("date_taken", date_taken.to_string()));
|
|
if let Some(ref c) = contact {
|
|
insight_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("contact", c.clone()));
|
|
}
|
|
|
|
// 4. Get location name from GPS coordinates (needed for RAG query)
|
|
let location = match exif {
|
|
Some(ref exif) => {
|
|
if let (Some(lat), Some(lon)) = (exif.gps_latitude, exif.gps_longitude) {
|
|
let loc = self.reverse_geocode(lat as f64, lon as f64).await;
|
|
if let Some(ref l) = loc {
|
|
insight_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("location", l.clone()));
|
|
Some(l.clone())
|
|
} else {
|
|
// Fallback: If reverse geocoding fails, use coordinates
|
|
log::warn!(
|
|
"Reverse geocoding failed for {}, {}, using coordinates as fallback",
|
|
lat,
|
|
lon
|
|
);
|
|
Some(format!("{:.4}, {:.4}", lat, lon))
|
|
}
|
|
} else {
|
|
None
|
|
}
|
|
}
|
|
None => None,
|
|
};
|
|
|
|
// 5. Intelligent retrieval: Hybrid approach for better context
|
|
let mut sms_summary = None;
|
|
let mut used_rag = false;
|
|
|
|
// TEMPORARY: Set to true to disable RAG and use only time-based retrieval for testing
|
|
let disable_rag_for_testing = false;
|
|
|
|
if disable_rag_for_testing {
|
|
log::warn!("RAG DISABLED FOR TESTING - Using only time-based retrieval (±2 days)");
|
|
// Skip directly to fallback
|
|
} else {
|
|
// ALWAYS use Strategy B: Expanded immediate context + historical RAG
|
|
// This is more reliable than pure semantic search which can match irrelevant messages
|
|
log::info!("Using expanded immediate context + historical RAG approach");
|
|
|
|
// Step 1: Get FULL immediate temporal context (±2 days, ALL messages)
|
|
let immediate_messages = self
|
|
.sms_client
|
|
.fetch_messages_for_contact(contact.as_deref(), timestamp)
|
|
.await
|
|
.unwrap_or_else(|e| {
|
|
log::error!("Failed to fetch immediate messages: {}", e);
|
|
Vec::new()
|
|
});
|
|
|
|
log::info!(
|
|
"Fetched {} messages from ±2 days window (using ALL for immediate context)",
|
|
immediate_messages.len()
|
|
);
|
|
|
|
if !immediate_messages.is_empty() {
|
|
// Step 2: Extract topics from immediate messages to enrich RAG query
|
|
let topics = self
|
|
.extract_topics_from_messages(&immediate_messages, &ollama_client)
|
|
.await;
|
|
|
|
log::info!("Extracted topics for query enrichment: {:?}", topics);
|
|
|
|
// Step 3: Try historical RAG (>30 days ago) using extracted topics
|
|
let topics_slice = if topics.is_empty() {
|
|
None
|
|
} else {
|
|
Some(topics.as_slice())
|
|
};
|
|
match self
|
|
.find_relevant_messages_rag_historical(
|
|
&insight_cx,
|
|
date_taken,
|
|
None,
|
|
contact.as_deref(),
|
|
topics_slice,
|
|
10, // Top 10 historical matches
|
|
)
|
|
.await
|
|
{
|
|
Ok(historical_messages) if !historical_messages.is_empty() => {
|
|
log::info!(
|
|
"Two-context approach: {} immediate (full conversation) + {} historical (similar past moments)",
|
|
immediate_messages.len(),
|
|
historical_messages.len()
|
|
);
|
|
used_rag = true;
|
|
|
|
// Step 4: Summarize contexts separately, then combine
|
|
let immediate_summary = self
|
|
.summarize_context_from_messages(
|
|
&immediate_messages,
|
|
&ollama_client,
|
|
custom_system_prompt.as_deref(),
|
|
)
|
|
.await
|
|
.unwrap_or_else(|| String::from("No immediate context"));
|
|
|
|
let historical_summary = self
|
|
.summarize_messages(
|
|
&historical_messages,
|
|
&ollama_client,
|
|
custom_system_prompt.as_deref(),
|
|
)
|
|
.await
|
|
.unwrap_or_else(|| String::from("No historical context"));
|
|
|
|
// Combine summaries
|
|
sms_summary = Some(format!(
|
|
"Immediate context (±2 days): {}\n\nSimilar moments from the past: {}",
|
|
immediate_summary, historical_summary
|
|
));
|
|
}
|
|
Ok(_) => {
|
|
// RAG found no historical matches, just use immediate context
|
|
log::info!("No historical RAG matches, using immediate context only");
|
|
sms_summary = self
|
|
.summarize_context_from_messages(
|
|
&immediate_messages,
|
|
&ollama_client,
|
|
custom_system_prompt.as_deref(),
|
|
)
|
|
.await;
|
|
}
|
|
Err(e) => {
|
|
log::warn!("Historical RAG failed, using immediate context only: {}", e);
|
|
sms_summary = self
|
|
.summarize_context_from_messages(
|
|
&immediate_messages,
|
|
&ollama_client,
|
|
custom_system_prompt.as_deref(),
|
|
)
|
|
.await;
|
|
}
|
|
}
|
|
} else {
|
|
log::info!("No immediate messages found, trying basic RAG as fallback");
|
|
// Fallback to basic RAG even without strong query
|
|
match self
|
|
.find_relevant_messages_rag(date_taken, None, contact.as_deref(), None, 20)
|
|
.await
|
|
{
|
|
Ok(rag_messages) if !rag_messages.is_empty() => {
|
|
used_rag = true;
|
|
sms_summary = self
|
|
.summarize_messages(
|
|
&rag_messages,
|
|
&ollama_client,
|
|
custom_system_prompt.as_deref(),
|
|
)
|
|
.await;
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
}
|
|
|
|
// 6. Fallback to traditional time-based message retrieval if RAG didn't work
|
|
if !used_rag {
|
|
log::info!("Using traditional time-based message retrieval (±2 days)");
|
|
let sms_messages = self
|
|
.sms_client
|
|
.fetch_messages_for_contact(contact.as_deref(), timestamp)
|
|
.await
|
|
.unwrap_or_else(|e| {
|
|
log::error!("Failed to fetch SMS messages: {}", e);
|
|
Vec::new()
|
|
});
|
|
|
|
log::info!(
|
|
"Fetched {} SMS messages closest to {}",
|
|
sms_messages.len(),
|
|
DateTime::from_timestamp(timestamp, 0)
|
|
.map(|dt| dt.format("%Y-%m-%d %H:%M:%S").to_string())
|
|
.unwrap_or_else(|| "unknown time".to_string())
|
|
);
|
|
|
|
// Summarize time-based messages
|
|
if !sms_messages.is_empty() {
|
|
match self
|
|
.sms_client
|
|
.summarize_context(&sms_messages, &ollama_client)
|
|
.await
|
|
{
|
|
Ok(summary) => {
|
|
sms_summary = Some(summary);
|
|
}
|
|
Err(e) => {
|
|
log::warn!("Failed to summarize SMS context: {}", e);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
let retrieval_method = if used_rag { "RAG" } else { "time-based" };
|
|
insight_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("retrieval_method", retrieval_method));
|
|
insight_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("has_sms_context", sms_summary.is_some()));
|
|
|
|
log::info!(
|
|
"Photo context: date={}, location={:?}, retrieval_method={}",
|
|
date_taken,
|
|
location,
|
|
retrieval_method
|
|
);
|
|
|
|
// 6. Gather Google Takeout context from all sources
|
|
let calendar_context = self
|
|
.gather_calendar_context(&insight_cx, timestamp, location.as_deref())
|
|
.await
|
|
.ok()
|
|
.flatten();
|
|
|
|
let exif_gps = exif.as_ref().and_then(|e| {
|
|
if let (Some(lat), Some(lon)) = (e.gps_latitude, e.gps_longitude) {
|
|
Some((lat as f64, lon as f64))
|
|
} else {
|
|
None
|
|
}
|
|
});
|
|
|
|
let location_context = self
|
|
.gather_location_context(&insight_cx, timestamp, exif_gps)
|
|
.await
|
|
.ok()
|
|
.flatten();
|
|
|
|
let search_context = self
|
|
.gather_search_context(
|
|
&insight_cx,
|
|
timestamp,
|
|
location.as_deref(),
|
|
contact.as_deref(),
|
|
)
|
|
.await
|
|
.ok()
|
|
.flatten();
|
|
|
|
// 7. Combine all context sources with equal weight
|
|
let combined_context = Self::combine_contexts(
|
|
sms_summary,
|
|
calendar_context,
|
|
location_context,
|
|
search_context,
|
|
);
|
|
|
|
log::info!(
|
|
"Combined context from all sources ({} chars)",
|
|
combined_context.len()
|
|
);
|
|
|
|
// 8. Check if the model has vision capabilities
|
|
let model_to_check = ollama_client.primary_model.clone();
|
|
let has_vision = match OllamaClient::check_model_capabilities(
|
|
&ollama_client.primary_url,
|
|
&model_to_check,
|
|
)
|
|
.await
|
|
{
|
|
Ok(capabilities) => {
|
|
log::info!(
|
|
"Model '{}' vision capability: {}",
|
|
model_to_check,
|
|
capabilities.has_vision
|
|
);
|
|
capabilities.has_vision
|
|
}
|
|
Err(e) => {
|
|
log::warn!(
|
|
"Failed to check vision capabilities for model '{}', assuming no vision support: {}",
|
|
model_to_check,
|
|
e
|
|
);
|
|
false
|
|
}
|
|
};
|
|
|
|
insight_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("model_has_vision", has_vision));
|
|
|
|
// 9. Load image and encode as base64 only if model supports vision
|
|
let image_base64 = if has_vision {
|
|
match self.load_image_as_base64(&file_path) {
|
|
Ok(b64) => {
|
|
log::info!(
|
|
"Successfully loaded image for vision-capable model '{}'",
|
|
model_to_check
|
|
);
|
|
Some(b64)
|
|
}
|
|
Err(e) => {
|
|
log::warn!("Failed to load image for vision model: {}", e);
|
|
None
|
|
}
|
|
}
|
|
} else {
|
|
log::info!(
|
|
"Model '{}' does not support vision, skipping image processing",
|
|
model_to_check
|
|
);
|
|
None
|
|
};
|
|
|
|
// 10. Generate title and summary with Ollama (using multi-source context + image if supported)
|
|
let title = ollama_client
|
|
.generate_photo_title(
|
|
date_taken,
|
|
location.as_deref(),
|
|
contact.as_deref(),
|
|
Some(&combined_context),
|
|
custom_system_prompt.as_deref(),
|
|
image_base64.clone(),
|
|
)
|
|
.await?;
|
|
|
|
let summary = ollama_client
|
|
.generate_photo_summary(
|
|
date_taken,
|
|
location.as_deref(),
|
|
contact.as_deref(),
|
|
Some(&combined_context),
|
|
custom_system_prompt.as_deref(),
|
|
image_base64,
|
|
)
|
|
.await?;
|
|
|
|
log::info!("Generated title: {}", title);
|
|
log::info!("Generated summary: {}", summary);
|
|
|
|
insight_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("title_length", title.len() as i64));
|
|
insight_cx
|
|
.span()
|
|
.set_attribute(KeyValue::new("summary_length", summary.len() as i64));
|
|
|
|
// 11. Store in database
|
|
let insight = InsertPhotoInsight {
|
|
file_path: file_path.to_string(),
|
|
title,
|
|
summary,
|
|
generated_at: Utc::now().timestamp(),
|
|
model_version: ollama_client.primary_model.clone(),
|
|
};
|
|
|
|
let mut dao = self.insight_dao.lock().expect("Unable to lock InsightDao");
|
|
let result = dao
|
|
.store_insight(&insight_cx, insight)
|
|
.map_err(|e| anyhow::anyhow!("Failed to store insight: {:?}", e));
|
|
|
|
match &result {
|
|
Ok(_) => {
|
|
log::info!("Successfully stored insight for {}", file_path);
|
|
insight_cx.span().set_status(Status::Ok);
|
|
}
|
|
Err(e) => {
|
|
log::error!("Failed to store insight: {:?}", e);
|
|
insight_cx.span().set_status(Status::error(e.to_string()));
|
|
}
|
|
}
|
|
|
|
result?;
|
|
Ok(())
|
|
}
|
|
|
|
/// Extract key topics/entities from messages using LLM for query enrichment
|
|
async fn extract_topics_from_messages(
|
|
&self,
|
|
messages: &[crate::ai::SmsMessage],
|
|
ollama: &OllamaClient,
|
|
) -> Vec<String> {
|
|
if messages.is_empty() {
|
|
return Vec::new();
|
|
}
|
|
|
|
// Format a sample of messages for topic extraction
|
|
let sample_size = messages.len().min(20);
|
|
let sample_text: Vec<String> = messages
|
|
.iter()
|
|
.take(sample_size)
|
|
.map(|m| format!("{}: {}", if m.is_sent { "Me" } else { &m.contact }, m.body))
|
|
.collect();
|
|
|
|
let prompt = format!(
|
|
r#"Extract important entities from these messages that provide context about what was happening. Focus on:
|
|
|
|
1. **People**: Names of specific people mentioned (first names, nicknames)
|
|
2. **Places**: Locations, cities, buildings, workplaces, parks, restaurants, venues
|
|
3. **Activities**: Specific events, hobbies, groups, organizations (e.g., "drum corps", "auditions")
|
|
4. **Unique terms**: Domain-specific words or phrases that might need explanation (e.g., "Hyland", "Vanguard", "DCI")
|
|
|
|
Messages:
|
|
{}
|
|
|
|
Return a comma-separated list of 3-7 specific entities (people, places, activities, unique terms).
|
|
Focus on proper nouns and specific terms that provide context.
|
|
Return ONLY the comma-separated list, nothing else."#,
|
|
sample_text.join("\n")
|
|
);
|
|
|
|
match ollama
|
|
.generate(&prompt, Some("You are an entity extraction assistant. Extract proper nouns, people, places, and domain-specific terms that provide context."))
|
|
.await
|
|
{
|
|
Ok(response) => {
|
|
log::debug!("Topic extraction raw response: {}", response);
|
|
|
|
// Parse comma-separated topics
|
|
let topics: Vec<String> = response
|
|
.split(',')
|
|
.map(|s| s.trim().to_string())
|
|
.filter(|s| !s.is_empty() && s.len() > 1) // Filter out single chars
|
|
.take(7) // Increased from 5 to 7
|
|
.collect();
|
|
|
|
if topics.is_empty() {
|
|
log::warn!("Topic extraction returned empty list from {} messages", messages.len());
|
|
} else {
|
|
log::info!("Extracted {} topics from {} messages: {}", topics.len(), messages.len(), topics.join(", "));
|
|
}
|
|
|
|
topics
|
|
}
|
|
Err(e) => {
|
|
log::warn!("Failed to extract topics from messages: {}", e);
|
|
Vec::new()
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Summarize pre-formatted message strings using LLM (concise version for historical context)
|
|
async fn summarize_messages(
|
|
&self,
|
|
messages: &[String],
|
|
ollama: &OllamaClient,
|
|
custom_system: Option<&str>,
|
|
) -> Option<String> {
|
|
if messages.is_empty() {
|
|
return None;
|
|
}
|
|
|
|
let messages_text = messages.join("\n");
|
|
|
|
let prompt = format!(
|
|
r#"Summarize the context from these messages in 2-3 sentences. Focus on activities, locations, events, and relationships mentioned.
|
|
|
|
Messages:
|
|
{}
|
|
|
|
Return ONLY the summary, nothing else."#,
|
|
messages_text
|
|
);
|
|
|
|
let system = custom_system
|
|
.unwrap_or("You are a context summarization assistant. Be concise and factual.");
|
|
|
|
match ollama.generate(&prompt, Some(system)).await {
|
|
Ok(summary) => Some(summary),
|
|
Err(e) => {
|
|
log::warn!("Failed to summarize messages: {}", e);
|
|
None
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Convert SmsMessage objects to formatted strings and summarize with more detail
|
|
/// This is used for immediate context (±2 days) to preserve conversation details
|
|
async fn summarize_context_from_messages(
|
|
&self,
|
|
messages: &[crate::ai::SmsMessage],
|
|
ollama: &OllamaClient,
|
|
custom_system: Option<&str>,
|
|
) -> Option<String> {
|
|
if messages.is_empty() {
|
|
return None;
|
|
}
|
|
|
|
// Format messages
|
|
let formatted: Vec<String> = messages
|
|
.iter()
|
|
.map(|m| {
|
|
let sender = if m.is_sent { "Me" } else { &m.contact };
|
|
let timestamp = chrono::DateTime::from_timestamp(m.timestamp, 0)
|
|
.map(|dt| dt.format("%Y-%m-%d %H:%M").to_string())
|
|
.unwrap_or_else(|| "unknown time".to_string());
|
|
format!("[{}] {}: {}", timestamp, sender, m.body)
|
|
})
|
|
.collect();
|
|
|
|
let messages_text = formatted.join("\n");
|
|
|
|
// Use a more detailed prompt for immediate context
|
|
let prompt = format!(
|
|
r#"Provide a detailed summary of the conversation context from these messages. Include:
|
|
- Key activities, events, and plans discussed
|
|
- Important locations or places mentioned
|
|
- Emotional tone and relationship dynamics
|
|
- Any significant details that provide context about what was happening
|
|
|
|
Be thorough but organized. Use 1-2 paragraphs.
|
|
|
|
Messages:
|
|
{}
|
|
|
|
Return ONLY the summary, nothing else."#,
|
|
messages_text
|
|
);
|
|
|
|
let system = custom_system.unwrap_or(
|
|
"You are a context summarization assistant. Be detailed and factual, preserving important context.",
|
|
);
|
|
|
|
match ollama.generate(&prompt, Some(system)).await {
|
|
Ok(summary) => Some(summary),
|
|
Err(e) => {
|
|
log::warn!("Failed to summarize immediate context: {}", e);
|
|
None
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Reverse geocode GPS coordinates to human-readable place names
|
|
async fn reverse_geocode(&self, lat: f64, lon: f64) -> Option<String> {
|
|
let url = format!(
|
|
"https://nominatim.openstreetmap.org/reverse?format=json&lat={}&lon={}",
|
|
lat, lon
|
|
);
|
|
|
|
log::debug!("Reverse geocoding {}, {} via Nominatim", lat, lon);
|
|
|
|
let client = reqwest::Client::new();
|
|
let response = match client
|
|
.get(&url)
|
|
.header("User-Agent", "ImageAPI/1.0") // Nominatim requires User-Agent
|
|
.send()
|
|
.await
|
|
{
|
|
Ok(resp) => resp,
|
|
Err(e) => {
|
|
log::warn!("Geocoding network error for {}, {}: {}", lat, lon, e);
|
|
return None;
|
|
}
|
|
};
|
|
|
|
if !response.status().is_success() {
|
|
log::warn!(
|
|
"Geocoding HTTP error for {}, {}: {}",
|
|
lat,
|
|
lon,
|
|
response.status()
|
|
);
|
|
return None;
|
|
}
|
|
|
|
let data: NominatimResponse = match response.json().await {
|
|
Ok(d) => d,
|
|
Err(e) => {
|
|
log::warn!("Geocoding JSON parse error for {}, {}: {}", lat, lon, e);
|
|
return None;
|
|
}
|
|
};
|
|
|
|
// Try to build a concise location name
|
|
if let Some(addr) = data.address {
|
|
let mut parts = Vec::new();
|
|
|
|
// Prefer city/town/village
|
|
if let Some(city) = addr.city.or(addr.town).or(addr.village) {
|
|
parts.push(city);
|
|
}
|
|
|
|
// Add state if available
|
|
if let Some(state) = addr.state {
|
|
parts.push(state);
|
|
}
|
|
|
|
if !parts.is_empty() {
|
|
log::info!("Reverse geocoded {}, {} -> {}", lat, lon, parts.join(", "));
|
|
return Some(parts.join(", "));
|
|
}
|
|
}
|
|
|
|
// Fallback to display_name if structured address not available
|
|
if let Some(ref display_name) = data.display_name {
|
|
log::info!(
|
|
"Reverse geocoded {}, {} -> {} (display_name)",
|
|
lat,
|
|
lon,
|
|
display_name
|
|
);
|
|
} else {
|
|
log::warn!("Geocoding returned no address data for {}, {}", lat, lon);
|
|
}
|
|
data.display_name
|
|
}
|
|
}
|