Apps/ImageApi
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Enhanced Insights with daily summary embeddings

Browse Source 
Bump to 0.5.0. Added daily summary generation job
This commit is contained in:
Cameron
2026-01-05 09:13:16 -05:00
parent 43b7c2b8ec
commit 11e725c443
18 changed files with 2348 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

589
src/ai/insight_generator.rs
View File

@@ -1,5 +1,7 @@
use anyhow::Result;
use chrono::{DateTime, Utc};
use opentelemetry::trace::{Span, Status, TraceContextExt, Tracer};
use opentelemetry::KeyValue;
use serde::Deserialize;
use std::fs::File;
use std::sync::{Arc, Mutex};
@@ -7,8 +9,9 @@ use std::sync::{Arc, Mutex};
use crate::ai::ollama::OllamaClient;
use crate::ai::sms_client::SmsApiClient;
use crate::database::models::InsertPhotoInsight;
use crate::database::{ExifDao, InsightDao};
use crate::database::{DailySummaryDao, ExifDao, InsightDao};
use crate::memories::extract_date_from_filename;
use crate::otel::global_tracer;
use crate::utils::normalize_path;
#[derive(Deserialize)]
@@ -31,6 +34,7 @@ pub struct InsightGenerator {
sms_client: SmsApiClient,
insight_dao: Arc<Mutex<Box<dyn InsightDao>>>,
exif_dao: Arc<Mutex<Box<dyn ExifDao>>>,
daily_summary_dao: Arc<Mutex<Box<dyn DailySummaryDao>>>,
base_path: String,
}
@@ -40,6 +44,7 @@ impl InsightGenerator {
sms_client: SmsApiClient,
insight_dao: Arc<Mutex<Box<dyn InsightDao>>>,
exif_dao: Arc<Mutex<Box<dyn ExifDao>>>,
daily_summary_dao: Arc<Mutex<Box<dyn DailySummaryDao>>>,
base_path: String,
) -> Self {
Self {
@@ -47,6 +52,7 @@ impl InsightGenerator {
sms_client,
insight_dao,
exif_dao,
daily_summary_dao,
base_path,
}
}
@@ -72,19 +78,174 @@ impl InsightGenerator {
None
}
/// Find relevant messages using RAG, excluding recent messages (>30 days ago)
/// This prevents RAG from returning messages already in the immediate time window
async fn find_relevant_messages_rag_historical(
&self,
parent_cx: &opentelemetry::Context,
date: chrono::NaiveDate,
location: Option<&str>,
contact: Option<&str>,
limit: usize,
) -> Result<Vec<String>> {
let tracer = global_tracer();
let mut span = tracer.start_with_context("ai.rag.filter_historical", parent_cx);
let filter_cx = parent_cx.with_span(span);
filter_cx.span().set_attribute(KeyValue::new("date", date.to_string()));
filter_cx.span().set_attribute(KeyValue::new("limit", limit as i64));
filter_cx.span().set_attribute(KeyValue::new("exclusion_window_days", 30));
let query_results = self.find_relevant_messages_rag(date, location, contact, limit * 2).await?;
filter_cx.span().set_attribute(KeyValue::new("rag_results_count", query_results.len() as i64));
// Filter out messages from within 30 days of the photo date
let photo_timestamp = date.and_hms_opt(12, 0, 0)
.ok_or_else(|| anyhow::anyhow!("Invalid date"))?
.and_utc()
.timestamp();
let exclusion_window = 30 * 86400; // 30 days in seconds
let historical_only: Vec<String> = query_results
.into_iter()
.filter(|msg| {
// Extract date from formatted daily summary "[2024-08-15] Contact ..."
if let Some(bracket_end) = msg.find(']') {
if let Some(date_str) = msg.get(1..bracket_end) {
// Parse just the date (daily summaries don't have time)
if let Ok(msg_date) = chrono::NaiveDate::parse_from_str(date_str, "%Y-%m-%d") {
let msg_timestamp = msg_date
.and_hms_opt(12, 0, 0)
.unwrap()
.and_utc()
.timestamp();
let time_diff = (photo_timestamp - msg_timestamp).abs();
return time_diff > exclusion_window;
}
}
}
false
})
.take(limit)
.collect();
log::info!(
"Found {} historical messages (>30 days from photo date)",
historical_only.len()
);
filter_cx.span().set_attribute(KeyValue::new("historical_results_count", historical_only.len() as i64));
filter_cx.span().set_status(Status::Ok);
Ok(historical_only)
}
/// Find relevant daily summaries using RAG (semantic search)
/// Returns formatted daily summary strings for LLM context
async fn find_relevant_messages_rag(
&self,
date: chrono::NaiveDate,
location: Option<&str>,
contact: Option<&str>,
limit: usize,
) -> Result<Vec<String>> {
let tracer = global_tracer();
let current_cx = opentelemetry::Context::current();
let mut span = tracer.start_with_context("ai.rag.search_daily_summaries", &current_cx);
span.set_attribute(KeyValue::new("date", date.to_string()));
span.set_attribute(KeyValue::new("limit", limit as i64));
if let Some(loc) = location {
span.set_attribute(KeyValue::new("location", loc.to_string()));
}
if let Some(c) = contact {
span.set_attribute(KeyValue::new("contact", c.to_string()));
}
// Build more detailed query string from photo context
let mut query_parts = Vec::new();
// Add temporal context
query_parts.push(format!("On {}", date.format("%B %d, %Y")));
// Add location if available
if let Some(loc) = location {
query_parts.push(format!("at {}", loc));
}
// Add contact context if available
if let Some(c) = contact {
query_parts.push(format!("conversation with {}", c));
}
// Add day of week for temporal context
let weekday = date.format("%A");
query_parts.push(format!("it was a {}", weekday));
let query = query_parts.join(", ");
span.set_attribute(KeyValue::new("query", query.clone()));
// Create context with this span for child operations
let search_cx = current_cx.with_span(span);
log::info!("========================================");
log::info!("RAG QUERY: {}", query);
log::info!("========================================");
// Generate embedding for the query
let query_embedding = self.ollama.generate_embedding(&query).await?;
// Search for similar daily summaries
let mut summary_dao = self
.daily_summary_dao
.lock()
.expect("Unable to lock DailySummaryDao");
let similar_summaries = summary_dao
.find_similar_summaries(&search_cx, &query_embedding, limit)
.map_err(|e| anyhow::anyhow!("Failed to find similar summaries: {:?}", e))?;
log::info!("Found {} relevant daily summaries via RAG", similar_summaries.len());
search_cx.span().set_attribute(KeyValue::new("results_count", similar_summaries.len() as i64));
// Format daily summaries for LLM context
let formatted = similar_summaries
.into_iter()
.map(|s| {
format!(
"[{}] {} ({} messages):\n{}",
s.date, s.contact, s.message_count, s.summary
)
})
.collect();
search_cx.span().set_status(Status::Ok);
Ok(formatted)
}
/// Generate AI insight for a single photo with optional custom model
pub async fn generate_insight_for_photo_with_model(
&self,
file_path: &str,
custom_model: Option<String>,
) -> Result<()> {
let tracer = global_tracer();
let current_cx = opentelemetry::Context::current();
let mut span = tracer.start_with_context("ai.insight.generate", &current_cx);
// Normalize path to ensure consistent forward slashes in database
let file_path = normalize_path(file_path);
log::info!("Generating insight for photo: {}", file_path);
span.set_attribute(KeyValue::new("file_path", file_path.clone()));
// Create custom Ollama client if model is specified
let ollama_client = if let Some(model) = custom_model {
log::info!("Using custom model: {}", model);
span.set_attribute(KeyValue::new("custom_model", model.clone()));
OllamaClient::new(
self.ollama.primary_url.clone(),
self.ollama.fallback_url.clone(),
@@ -92,15 +253,18 @@ impl InsightGenerator {
Some(model), // Use the same custom model for fallback server
)
} else {
span.set_attribute(KeyValue::new("model", self.ollama.primary_model.clone()));
self.ollama.clone()
};
// Create context with this span for child operations
let insight_cx = current_cx.with_span(span);
// 1. Get EXIF data for the photo
let otel_context = opentelemetry::Context::new();
let exif = {
let mut exif_dao = self.exif_dao.lock().expect("Unable to lock ExifDao");
exif_dao
.get_exif(&otel_context, &file_path)
.get_exif(&insight_cx, &file_path)
.map_err(|e| anyhow::anyhow!("Failed to get EXIF: {:?}", e))?
};
@@ -139,47 +303,20 @@ impl InsightGenerator {
let contact = Self::extract_contact_from_path(&file_path);
log::info!("Extracted contact from path: {:?}", contact);
// 4. Fetch SMS messages for the contact (±1 day)
// Pass the full timestamp for proximity sorting
let sms_messages = self
.sms_client
.fetch_messages_for_contact(contact.as_deref(), timestamp)
.await
.unwrap_or_else(|e| {
log::error!("Failed to fetch SMS messages: {}", e);
Vec::new()
});
insight_cx.span().set_attribute(KeyValue::new("date_taken", date_taken.to_string()));
if let Some(ref c) = contact {
insight_cx.span().set_attribute(KeyValue::new("contact", c.clone()));
}
log::info!(
"Fetched {} SMS messages closest to {}",
sms_messages.len(),
chrono::DateTime::from_timestamp(timestamp, 0)
.map(|dt| dt.format("%Y-%m-%d %H:%M:%S").to_string())
.unwrap_or_else(|| "unknown time".to_string())
);
// 5. Summarize SMS context
let sms_summary = if !sms_messages.is_empty() {
match self
.sms_client
.summarize_context(&sms_messages, &ollama_client)
.await
{
Ok(summary) => Some(summary),
Err(e) => {
log::warn!("Failed to summarize SMS context: {}", e);
None
}
}
} else {
None
};
// 6. Get location name from GPS coordinates
// 4. Get location name from GPS coordinates (needed for RAG query)
let location = match exif {
Some(exif) => {
Some(ref exif) => {
if let (Some(lat), Some(lon)) = (exif.gps_latitude, exif.gps_longitude) {
self.reverse_geocode(lat, lon).await
let loc = self.reverse_geocode(lat, lon).await;
if let Some(ref l) = loc {
insight_cx.span().set_attribute(KeyValue::new("location", l.clone()));
}
loc
} else {
None
}
@@ -187,11 +324,171 @@ impl InsightGenerator {
None => None,
};
// 5. Intelligent retrieval: Hybrid approach for better context
let mut sms_summary = None;
let mut used_rag = false;
// TEMPORARY: Set to true to disable RAG and use only time-based retrieval for testing
let disable_rag_for_testing = false;
// Decide strategy based on available metadata
let has_strong_query = location.is_some();
if disable_rag_for_testing {
log::warn!("RAG DISABLED FOR TESTING - Using only time-based retrieval (±1 day)");
// Skip directly to fallback
} else if has_strong_query {
// Strategy A: Pure RAG (we have location for good semantic matching)
log::info!("Using RAG with location-based query");
match self
.find_relevant_messages_rag(
date_taken,
location.as_deref(),
contact.as_deref(),
20,
)
.await
{
Ok(rag_messages) if !rag_messages.is_empty() => {
used_rag = true;
sms_summary = self.summarize_messages(&rag_messages, &ollama_client).await;
}
Ok(_) => log::info!("RAG returned no messages"),
Err(e) => log::warn!("RAG failed: {}", e),
}
} else {
// Strategy B: Expanded immediate context + historical RAG
log::info!("Using expanded immediate context + historical RAG approach");
// Step 1: Get FULL immediate temporal context (±1 day, ALL messages)
let immediate_messages = self
.sms_client
.fetch_messages_for_contact(contact.as_deref(), timestamp)
.await
.unwrap_or_else(|e| {
log::error!("Failed to fetch immediate messages: {}", e);
Vec::new()
});
log::info!(
"Fetched {} messages from ±1 day window (using ALL for immediate context)",
immediate_messages.len()
);
if !immediate_messages.is_empty() {
// Step 2: Extract topics from immediate messages to enrich RAG query
let topics = self.extract_topics_from_messages(&immediate_messages, &ollama_client).await;
log::info!("Extracted topics for query enrichment: {:?}", topics);
// Step 3: Try historical RAG (>30 days ago)
match self
.find_relevant_messages_rag_historical(
&insight_cx,
date_taken,
None,
contact.as_deref(),
10, // Top 10 historical matches
)
.await
{
Ok(historical_messages) if !historical_messages.is_empty() => {
log::info!(
"Two-context approach: {} immediate (full conversation) + {} historical (similar past moments)",
immediate_messages.len(),
historical_messages.len()
);
used_rag = true;
// Step 4: Summarize contexts separately, then combine
let immediate_summary = self
.summarize_context_from_messages(&immediate_messages, &ollama_client)
.await
.unwrap_or_else(|| String::from("No immediate context"));
let historical_summary = self
.summarize_messages(&historical_messages, &ollama_client)
.await
.unwrap_or_else(|| String::from("No historical context"));
// Combine summaries
sms_summary = Some(format!(
"Immediate context (±1 day): {}\n\nSimilar moments from the past: {}",
immediate_summary, historical_summary
));
}
Ok(_) => {
// RAG found no historical matches, just use immediate context
log::info!("No historical RAG matches, using immediate context only");
sms_summary = self.summarize_context_from_messages(&immediate_messages, &ollama_client).await;
}
Err(e) => {
log::warn!("Historical RAG failed, using immediate context only: {}", e);
sms_summary = self.summarize_context_from_messages(&immediate_messages, &ollama_client).await;
}
}
} else {
log::info!("No immediate messages found, trying basic RAG as fallback");
// Fallback to basic RAG even without strong query
match self
.find_relevant_messages_rag(date_taken, None, contact.as_deref(), 20)
.await
{
Ok(rag_messages) if !rag_messages.is_empty() => {
used_rag = true;
sms_summary = self.summarize_messages(&rag_messages, &ollama_client).await;
}
_ => {}
}
}
}
// 6. Fallback to traditional time-based message retrieval if RAG didn't work
if !used_rag {
log::info!("Using traditional time-based message retrieval (±1 day)");
let sms_messages = self
.sms_client
.fetch_messages_for_contact(contact.as_deref(), timestamp)
.await
.unwrap_or_else(|e| {
log::error!("Failed to fetch SMS messages: {}", e);
Vec::new()
});
log::info!(
"Fetched {} SMS messages closest to {}",
sms_messages.len(),
chrono::DateTime::from_timestamp(timestamp, 0)
.map(|dt| dt.format("%Y-%m-%d %H:%M:%S").to_string())
.unwrap_or_else(|| "unknown time".to_string())
);
// Summarize time-based messages
if !sms_messages.is_empty() {
match self
.sms_client
.summarize_context(&sms_messages, &ollama_client)
.await
{
Ok(summary) => {
sms_summary = Some(summary);
}
Err(e) => {
log::warn!("Failed to summarize SMS context: {}", e);
}
}
}
}
let retrieval_method = if used_rag { "RAG" } else { "time-based" };
insight_cx.span().set_attribute(KeyValue::new("retrieval_method", retrieval_method));
insight_cx.span().set_attribute(KeyValue::new("has_sms_context", sms_summary.is_some()));
log::info!(
"Photo context: date={}, location={:?}, sms_messages={}",
"Photo context: date={}, location={:?}, retrieval_method={}",
date_taken,
location,
sms_messages.len()
retrieval_method
);
// 7. Generate title and summary with Ollama
@@ -206,6 +503,9 @@ impl InsightGenerator {
log::info!("Generated title: {}", title);
log::info!("Generated summary: {}", summary);
insight_cx.span().set_attribute(KeyValue::new("title_length", title.len() as i64));
insight_cx.span().set_attribute(KeyValue::new("summary_length", summary.len() as i64));
// 8. Store in database
let insight = InsertPhotoInsight {
file_path: file_path.to_string(),
@@ -216,13 +516,210 @@ impl InsightGenerator {
};
let mut dao = self.insight_dao.lock().expect("Unable to lock InsightDao");
dao.store_insight(&otel_context, insight)
.map_err(|e| anyhow::anyhow!("Failed to store insight: {:?}", e))?;
let result = dao.store_insight(&insight_cx, insight)
.map_err(|e| anyhow::anyhow!("Failed to store insight: {:?}", e));
log::info!("Successfully stored insight for {}", file_path);
match &result {
Ok(_) => {
log::info!("Successfully stored insight for {}", file_path);
insight_cx.span().set_status(Status::Ok);
}
Err(e) => {
log::error!("Failed to store insight: {:?}", e);
insight_cx.span().set_status(Status::error(e.to_string()));
}
}
result?;
Ok(())
}
/// Extract key topics/entities from messages using LLM for query enrichment
async fn extract_topics_from_messages(
&self,
messages: &[crate::ai::SmsMessage],
ollama: &OllamaClient,
) -> Vec<String> {
if messages.is_empty() {
return Vec::new();
}
// Format a sample of messages for topic extraction
let sample_size = messages.len().min(20);
let sample_text: Vec<String> = messages
.iter()
.take(sample_size)
.map(|m| format!("{}: {}", if m.is_sent { "Me" } else { &m.contact }, m.body))
.collect();
let prompt = format!(
r#"Extract important entities from these messages that provide context about what was happening. Focus on:
1. **People**: Names of specific people mentioned (first names, nicknames)
2. **Places**: Locations, cities, buildings, workplaces, parks, restaurants, venues
3. **Activities**: Specific events, hobbies, groups, organizations (e.g., "drum corps", "auditions")
4. **Unique terms**: Domain-specific words or phrases that might need explanation (e.g., "Hyland", "Vanguard", "DCI")
Messages:
{}
Return a comma-separated list of 3-7 specific entities (people, places, activities, unique terms).
Focus on proper nouns and specific terms that provide context.
Return ONLY the comma-separated list, nothing else."#,
sample_text.join("\n")
);
match ollama
.generate(&prompt, Some("You are an entity extraction assistant. Extract proper nouns, people, places, and domain-specific terms that provide context."))
.await
{
Ok(response) => {
// Parse comma-separated topics
response
.split(',')
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty() && s.len() > 1) // Filter out single chars
.take(7) // Increased from 5 to 7
.collect()
}
Err(e) => {
log::warn!("Failed to extract topics from messages: {}", e);
Vec::new()
}
}
}
/// Find relevant messages using RAG with topic-enriched query
async fn find_relevant_messages_rag_enriched(
&self,
date: chrono::NaiveDate,
contact: Option<&str>,
topics: &[String],
limit: usize,
) -> Result<Vec<String>> {
// Build enriched query from date + topics
let mut query_parts = Vec::new();
query_parts.push(format!("On {}", date.format("%B %d, %Y")));
if !topics.is_empty() {
query_parts.push(format!("about {}", topics.join(", ")));
}
if let Some(c) = contact {
query_parts.push(format!("conversation with {}", c));
}
// Add day of week
let weekday = date.format("%A");
query_parts.push(format!("it was a {}", weekday));
let query = query_parts.join(", ");
log::info!("========================================");
log::info!("ENRICHED RAG QUERY: {}", query);
log::info!("Extracted topics: {:?}", topics);
log::info!("========================================");
// Use existing RAG method with enriched query
self.find_relevant_messages_rag(date, None, contact, limit)
.await
}
/// Summarize pre-formatted message strings using LLM (concise version for historical context)
async fn summarize_messages(
&self,
messages: &[String],
ollama: &OllamaClient,
) -> Option<String> {
if messages.is_empty() {
return None;
}
let messages_text = messages.join("\n");
let prompt = format!(
r#"Summarize the context from these messages in 2-3 sentences. Focus on activities, locations, events, and relationships mentioned.
Messages:
{}
Return ONLY the summary, nothing else."#,
messages_text
);
match ollama
.generate(
&prompt,
Some("You are a context summarization assistant. Be concise and factual."),
)
.await
{
Ok(summary) => Some(summary),
Err(e) => {
log::warn!("Failed to summarize messages: {}", e);
None
}
}
}
/// Convert SmsMessage objects to formatted strings and summarize with more detail
/// This is used for immediate context (±1 day) to preserve conversation details
async fn summarize_context_from_messages(
&self,
messages: &[crate::ai::SmsMessage],
ollama: &OllamaClient,
) -> Option<String> {
if messages.is_empty() {
return None;
}
// Format messages
let formatted: Vec<String> = messages
.iter()
.map(|m| {
let sender = if m.is_sent { "Me" } else { &m.contact };
let timestamp = chrono::DateTime::from_timestamp(m.timestamp, 0)
.map(|dt| dt.format("%Y-%m-%d %H:%M").to_string())
.unwrap_or_else(|| "unknown time".to_string());
format!("[{}] {}: {}", timestamp, sender, m.body)
})
.collect();
let messages_text = formatted.join("\n");
// Use a more detailed prompt for immediate context
let prompt = format!(
r#"Provide a detailed summary of the conversation context from these messages. Include:
- Key activities, events, and plans discussed
- Important locations or places mentioned
- Emotional tone and relationship dynamics
- Any significant details that provide context about what was happening
Be thorough but organized. Use 1-2 paragraphs.
Messages:
{}
Return ONLY the summary, nothing else."#,
messages_text
);
match ollama
.generate(
&prompt,
Some("You are a context summarization assistant. Be detailed and factual, preserving important context."),
)
.await
{
Ok(summary) => Some(summary),
Err(e) => {
log::warn!("Failed to summarize immediate context: {}", e);
None
}
}
}
/// Reverse geocode GPS coordinates to human-readable place names
async fn reverse_geocode(&self, lat: f64, lon: f64) -> Option<String> {
let url = format!(