diff --git a/src/ai/insight_generator.rs b/src/ai/insight_generator.rs index dbad90b..be513e8 100644 --- a/src/ai/insight_generator.rs +++ b/src/ai/insight_generator.rs @@ -153,6 +153,7 @@ impl InsightGenerator { contact: Option<&str>, topics: Option<&[String]>, limit: usize, + extra_context: Option<&str>, ) -> Result> { let tracer = global_tracer(); let span = tracer.start_with_context("ai.rag.filter_historical", parent_cx); @@ -174,7 +175,7 @@ impl InsightGenerator { } let query_results = self - .find_relevant_messages_rag(date, location, contact, topics, limit * 2) + .find_relevant_messages_rag(date, location, contact, topics, limit * 2, extra_context) .await?; filter_cx.span().set_attribute(KeyValue::new( @@ -236,6 +237,7 @@ impl InsightGenerator { contact: Option<&str>, topics: Option<&[String]>, limit: usize, + extra_context: Option<&str>, ) -> Result> { let tracer = global_tracer(); let current_cx = opentelemetry::Context::current(); @@ -250,7 +252,7 @@ impl InsightGenerator { } // Build query string - prioritize topics if available (semantically meaningful) - let query = if let Some(topics) = topics { + let base_query = if let Some(topics) = topics { if !topics.is_empty() { // Use topics for semantic search - these are actual content keywords let topic_str = topics.join(", "); @@ -268,6 +270,12 @@ impl InsightGenerator { Self::build_metadata_query(date, location, contact) }; + let query = if let Some(extra) = extra_context { + format!("{}. {}", base_query, extra) + } else { + base_query + }; + span.set_attribute(KeyValue::new("query", query.clone())); // Create context with this span for child operations @@ -718,6 +726,20 @@ impl InsightGenerator { .set_attribute(KeyValue::new("contact", c.clone())); } + // Fetch file tags (used to enrich RAG and final context) + let tag_names: Vec = { + let mut dao = self.tag_dao.lock().expect("Unable to lock TagDao"); + dao.get_tags_for_path(&insight_cx, &file_path) + .unwrap_or_else(|e| { + log::warn!("Failed to fetch tags for insight {}: {}", file_path, e); + Vec::new() + }) + .into_iter() + .map(|t| t.name) + .collect() + }; + log::info!("Fetched {} tags for photo: {:?}", tag_names.len(), tag_names); + // 4. Get location name from GPS coordinates (needed for RAG query) let location = match exif { Some(ref exif) => { @@ -744,6 +766,90 @@ impl InsightGenerator { None => None, }; + // Check if the model has vision capabilities + let model_to_check = ollama_client.primary_model.clone(); + let has_vision = match OllamaClient::check_model_capabilities( + &ollama_client.primary_url, + &model_to_check, + ) + .await + { + Ok(capabilities) => { + log::info!( + "Model '{}' vision capability: {}", + model_to_check, + capabilities.has_vision + ); + capabilities.has_vision + } + Err(e) => { + log::warn!( + "Failed to check vision capabilities for model '{}', assuming no vision support: {}", + model_to_check, + e + ); + false + } + }; + + insight_cx + .span() + .set_attribute(KeyValue::new("model_has_vision", has_vision)); + + // Load image and encode as base64 only if model supports vision + let image_base64 = if has_vision { + match self.load_image_as_base64(&file_path) { + Ok(b64) => { + log::info!( + "Successfully loaded image for vision-capable model '{}'", + model_to_check + ); + Some(b64) + } + Err(e) => { + log::warn!("Failed to load image for vision model: {}", e); + None + } + } + } else { + log::info!( + "Model '{}' does not support vision, skipping image processing", + model_to_check + ); + None + }; + + // Generate brief photo description for RAG enrichment (vision models only) + let photo_description: Option = if let Some(ref img_b64) = image_base64 { + match ollama_client.generate_photo_description(img_b64).await { + Ok(desc) => { + log::info!("Photo description for RAG enrichment: {}", desc); + Some(desc) + } + Err(e) => { + log::warn!("Failed to generate photo description for RAG enrichment: {}", e); + None + } + } + } else { + None + }; + + // Build enriched context string for RAG: photo description + tags + // (SMS topics are passed separately to RAG functions) + let enriched_query: Option = { + let mut parts: Vec = Vec::new(); + if let Some(ref desc) = photo_description { + parts.push(desc.clone()); + } + if !tag_names.is_empty() { + parts.push(format!("tags: {}", tag_names.join(", "))); + } + if parts.is_empty() { None } else { Some(parts.join(". ")) } + }; + + let mut search_enrichment: Option = enriched_query.clone(); + // 5. Intelligent retrieval: Hybrid approach for better context let mut sms_summary = None; let mut used_rag = false; @@ -782,6 +888,21 @@ impl InsightGenerator { log::info!("Extracted topics for query enrichment: {:?}", topics); + // Build full search enrichment: SMS topics + photo description + tag names + search_enrichment = { + let mut parts: Vec = Vec::new(); + if !topics.is_empty() { + parts.push(topics.join(", ")); + } + if let Some(ref desc) = photo_description { + parts.push(desc.clone()); + } + if !tag_names.is_empty() { + parts.push(format!("tags: {}", tag_names.join(", "))); + } + if parts.is_empty() { None } else { Some(parts.join(". ")) } + }; + // Step 3: Try historical RAG (>30 days ago) using extracted topics let topics_slice = if topics.is_empty() { None @@ -796,6 +917,7 @@ impl InsightGenerator { contact.as_deref(), topics_slice, 10, // Top 10 historical matches + enriched_query.as_deref(), ) .await { @@ -858,7 +980,7 @@ impl InsightGenerator { log::info!("No immediate messages found, trying basic RAG as fallback"); // Fallback to basic RAG even without strong query match self - .find_relevant_messages_rag(date_taken, None, contact.as_deref(), None, 20) + .find_relevant_messages_rag(date_taken, None, contact.as_deref(), None, 20, enriched_query.as_deref()) .await { Ok(rag_messages) if !rag_messages.is_empty() => { @@ -955,19 +1077,25 @@ impl InsightGenerator { timestamp, location.as_deref(), contact.as_deref(), - None, // enrichment — wired up in Task 5 + search_enrichment.as_deref(), ) .await .ok() .flatten(); // 7. Combine all context sources with equal weight + let tags_context = if tag_names.is_empty() { + None + } else { + Some(tag_names.join(", ")) + }; + let combined_context = Self::combine_contexts( sms_summary, calendar_context, location_context, search_context, - None, // tags — wired up in Task 5 + tags_context, ); log::info!( @@ -975,59 +1103,6 @@ impl InsightGenerator { combined_context.len() ); - // 8. Check if the model has vision capabilities - let model_to_check = ollama_client.primary_model.clone(); - let has_vision = match OllamaClient::check_model_capabilities( - &ollama_client.primary_url, - &model_to_check, - ) - .await - { - Ok(capabilities) => { - log::info!( - "Model '{}' vision capability: {}", - model_to_check, - capabilities.has_vision - ); - capabilities.has_vision - } - Err(e) => { - log::warn!( - "Failed to check vision capabilities for model '{}', assuming no vision support: {}", - model_to_check, - e - ); - false - } - }; - - insight_cx - .span() - .set_attribute(KeyValue::new("model_has_vision", has_vision)); - - // 9. Load image and encode as base64 only if model supports vision - let image_base64 = if has_vision { - match self.load_image_as_base64(&file_path) { - Ok(b64) => { - log::info!( - "Successfully loaded image for vision-capable model '{}'", - model_to_check - ); - Some(b64) - } - Err(e) => { - log::warn!("Failed to load image for vision model: {}", e); - None - } - } - } else { - log::info!( - "Model '{}' does not support vision, skipping image processing", - model_to_check - ); - None - }; - // 10. Generate summary first, then derive title from the summary let summary = ollama_client .generate_photo_summary( @@ -1036,7 +1111,7 @@ impl InsightGenerator { contact.as_deref(), Some(&combined_context), custom_system_prompt.as_deref(), - image_base64, + image_base64.clone(), ) .await?;