Apps/ImageApi
1
0 Fork 0
Code Issues 7 Pull Requests Releases Wiki Activity

Tracing: LLM clients + streaming spans on tracing, drop cx threading (#140)

Browse Source 
LlmClient methods no longer take an OpenTelemetry Context. llm.request is
now a tracing span (otel.kind=client, llm.provider/operation/model) that
parents on the caller's ambient span; traceparent injection reads the
span's OTel context via OpenTelemetrySpanExt.

The src/ai feature spans (handlers, insight generation, chat, persona
chat, daily summaries, TTS) move to #[instrument] / info_span!, and every
LLM-bearing tokio::spawn carries its parent span with .instrument(): the
streaming chat turn (chat_turn_stream_inner), async chat / persona turns,
background insight jobs and TTS jobs. Sub-spans in the agentic loops
(ai.chat.loop, ai.agentic.loop) instrument each LLM / tool await.

DAO calls keep their explicit Context (derived from the tracing span)
until the DAO migration (#141).

tests/otel_llm_spans.rs pins nesting + traceparent for the blocking and
streaming paths across a spawn on a multi-thread runtime.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Cameron CordesandClaude Opus 5.5 committed 2026-09-29 17:02:40 -04:00
1 parent 7a59afc62b
commit 1c508cdabd
20 files changed
+973 -1030
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/ai/insight_generator.rs
+123 -196
View File
@@ -2,13 +2,14 @@ use anyhow::Result;
use base64::Engine as _;
use chrono::{DateTime, Local, NaiveDate, TimeZone, Utc};
use image::ImageFormat;
use opentelemetry::KeyValue;
use opentelemetry::trace::{Span, Status, TraceContextExt, Tracer};
use opentelemetry::trace::Status;
use serde::Deserialize;
use std::fs::File;
use std::io::Cursor;
use std::str::FromStr;
use std::sync::{Arc, Mutex};
use tracing::Instrument;
use tracing_opentelemetry::OpenTelemetrySpanExt;
use crate::ai::apollo_client::{ApolloClient, ApolloPlace};
use crate::ai::backend::{BackendKind, ResolvedBackend, SamplingOverrides};
@@ -25,7 +26,6 @@ use crate::database::{
};
use crate::libraries::Library;
use crate::memories::extract_date_from_filename;
use crate::otel::global_tracer;
use crate::tags::TagDao;
use crate::utils::{earliest_fs_time, normalize_path, retry_with_backoff, retry_with_backoff_if};
@@ -502,9 +502,9 @@ impl InsightGenerator {
/// Find relevant messages using RAG, excluding recent messages (>30 days ago)
/// This prevents RAG from returning messages already in the immediate time window
#[tracing::instrument(name = "ai.rag.filter_historical", skip_all)]
async fn find_relevant_messages_rag_historical(
&self,
parent_cx: &opentelemetry::Context,
date: chrono::NaiveDate,
location: Option<&str>,
contact: Option<&str>,
@@ -512,33 +512,20 @@ impl InsightGenerator {
limit: usize,
extra_context: Option<&str>,
) -> Result<Vec<String>> {
let tracer = global_tracer();
let span = tracer.start_with_context("ai.rag.filter_historical", parent_cx);
let filter_cx = parent_cx.with_span(span);
let span = tracing::Span::current();
filter_cx
.span()
.set_attribute(KeyValue::new("date", date.to_string()));
filter_cx
.span()
.set_attribute(KeyValue::new("limit", limit as i64));
filter_cx
.span()
.set_attribute(KeyValue::new("exclusion_window_days", 30));
span.set_attribute("date", date.to_string());
span.set_attribute("limit", limit as i64);
span.set_attribute("exclusion_window_days", 30);
if let Some(t) = topics {
filter_cx
.span()
.set_attribute(KeyValue::new("topics", t.join(", ")));
span.set_attribute("topics", t.join(", "));
}
let query_results = self
.find_relevant_messages_rag(date, location, contact, topics, limit * 2, extra_context)
.await?;
filter_cx.span().set_attribute(KeyValue::new(
"rag_results_count",
query_results.len() as i64,
));
span.set_attribute("rag_results_count", query_results.len() as i64);
// Filter out messages from within 30 days of the photo date
let photo_timestamp = date
@@ -576,17 +563,15 @@ impl InsightGenerator {
historical_only.len()
);
filter_cx.span().set_attribute(KeyValue::new(
"historical_results_count",
historical_only.len() as i64,
));
filter_cx.span().set_status(Status::Ok);
span.set_attribute("historical_results_count", historical_only.len() as i64);
span.set_status(Status::Ok);
Ok(historical_only)
}
/// Find relevant daily summaries using RAG (semantic search)
/// Returns formatted daily summary strings for LLM context
#[tracing::instrument(name = "ai.rag.search_daily_summaries", skip_all)]
async fn find_relevant_messages_rag(
&self,
date: chrono::NaiveDate,
@@ -596,16 +581,14 @@ impl InsightGenerator {
limit: usize,
extra_context: Option<&str>,
) -> Result<Vec<String>> {
let tracer = global_tracer();
let current_cx = opentelemetry::Context::current();
let mut span = tracer.start_with_context("ai.rag.search_daily_summaries", &current_cx);
span.set_attribute(KeyValue::new("date", date.to_string()));
span.set_attribute(KeyValue::new("limit", limit as i64));
let span = tracing::Span::current();
span.set_attribute("date", date.to_string());
span.set_attribute("limit", limit as i64);
if let Some(loc) = location {
span.set_attribute(KeyValue::new("location", loc.to_string()));
span.set_attribute("location", loc.to_string());
}
if let Some(c) = contact {
span.set_attribute(KeyValue::new("contact", c.to_string()));
span.set_attribute("contact", c.to_string());
}
// Build query string - prioritize topics if available (semantically meaningful)
@@ -633,10 +616,10 @@ impl InsightGenerator {
base_query
};
span.set_attribute(KeyValue::new("query", query.clone()));
span.set_attribute("query", query.clone());
// Create context with this span for child operations
let search_cx = current_cx.with_span(span);
let search_cx = span.context();
log::info!("========================================");
log::info!("RAG QUERY: {}", query);
@@ -646,8 +629,7 @@ impl InsightGenerator {
// (`LLM_BACKEND` switch). Must match the backend that populated the
// daily-summary embeddings or similarity search will be garbage.
let query_embedding =
crate::ai::embed_query(&search_cx, &self.ollama, self.llamacpp.as_deref(), &query)
.await?;
crate::ai::embed_query(&self.ollama, self.llamacpp.as_deref(), &query).await?;
// Search for similar daily summaries with time-based weighting
// This prioritizes summaries temporally close to the query date
@@ -666,10 +648,7 @@ impl InsightGenerator {
similar_summaries.len()
);
search_cx.span().set_attribute(KeyValue::new(
"results_count",
similar_summaries.len() as i64,
));
span.set_attribute("results_count", similar_summaries.len() as i64);
// Format daily summaries for LLM context
let formatted = similar_summaries
@@ -682,7 +661,7 @@ impl InsightGenerator {
})
.collect();
search_cx.span().set_status(Status::Ok);
span.set_status(Status::Ok);
Ok(formatted)
}
@@ -691,30 +670,28 @@ impl InsightGenerator {
/// tool. Embeds the caller's query as-is (no metadata boilerplate) and
/// only applies time weighting when an anchor date is provided —
/// without one, results rank purely by similarity across all time.
#[tracing::instrument(name = "ai.rag.search_daily_summaries", skip_all)]
async fn search_summaries_semantic(
&self,
query: &str,
date: Option<chrono::NaiveDate>,
limit: usize,
) -> Result<Vec<String>> {
let tracer = global_tracer();
let current_cx = opentelemetry::Context::current();
let mut span = tracer.start_with_context("ai.rag.search_daily_summaries", &current_cx);
span.set_attribute(KeyValue::new("query", query.to_string()));
span.set_attribute(KeyValue::new("limit", limit as i64));
span.set_attribute(KeyValue::new("time_weighted", date.is_some()));
let span = tracing::Span::current();
span.set_attribute("query", query.to_string());
span.set_attribute("limit", limit as i64);
span.set_attribute("time_weighted", date.is_some());
if let Some(d) = date {
span.set_attribute(KeyValue::new("date", d.to_string()));
span.set_attribute("date", d.to_string());
}
let search_cx = current_cx.with_span(span);
let search_cx = span.context();
log::info!("RAG QUERY: {} (anchor date: {:?})", query, date);
// Must use the same backend that populated the daily-summary
// embeddings or similarity search is garbage (see embed_one docs).
let query_embedding =
crate::ai::embed_query(&search_cx, &self.ollama, self.llamacpp.as_deref(), query)
.await?;
crate::ai::embed_query(&self.ollama, self.llamacpp.as_deref(), query).await?;
let mut summary_dao = self
.daily_summary_dao
@@ -732,11 +709,8 @@ impl InsightGenerator {
}
.map_err(|e| anyhow::anyhow!("Failed to find similar summaries: {:?}", e))?;
search_cx.span().set_attribute(KeyValue::new(
"results_count",
similar_summaries.len() as i64,
));
search_cx.span().set_status(Status::Ok);
span.set_attribute("results_count", similar_summaries.len() as i64);
span.set_status(Status::Ok);
Ok(similar_summaries
.into_iter()
@@ -789,20 +763,17 @@ impl InsightGenerator {
/// Gather calendar context for photo timestamp
/// Uses hybrid time + semantic search (±7 days, ranked by relevance)
#[tracing::instrument(name = "ai.context.calendar", skip_all)]
async fn gather_calendar_context(
&self,
parent_cx: &opentelemetry::Context,
timestamp: i64,
location: Option<&str>,
) -> Result<Option<String>> {
let tracer = global_tracer();
let span = tracer.start_with_context("ai.context.calendar", parent_cx);
let calendar_cx = parent_cx.with_span(span);
let span = tracing::Span::current();
let calendar_cx = span.context();
let query_embedding = if let Some(loc) = location {
match crate::ai::embed_query(&calendar_cx, &self.ollama, self.llamacpp.as_deref(), loc)
.await
{
match crate::ai::embed_query(&self.ollama, self.llamacpp.as_deref(), loc).await {
Ok(emb) => Some(emb),
Err(e) => {
log::warn!("Failed to generate embedding for location '{}': {}", loc, e);
@@ -828,7 +799,7 @@ impl InsightGenerator {
.ok()
};
calendar_cx.span().set_status(Status::Ok);
span.set_status(Status::Ok);
if let Some(events) = events {
if events.is_empty() {
@@ -866,15 +837,14 @@ impl InsightGenerator {
/// Gather location context for photo timestamp
/// Finds nearest location record (±30 minutes)
#[tracing::instrument(name = "ai.context.location", skip_all)]
async fn gather_location_context(
&self,
parent_cx: &opentelemetry::Context,
timestamp: i64,
exif_gps: Option<(f64, f64)>,
) -> Result<Option<String>> {
let tracer = global_tracer();
let span = tracer.start_with_context("ai.context.location", parent_cx);
let location_cx = parent_cx.with_span(span);
let span = tracing::Span::current();
let location_cx = span.context();
let nearest = {
let mut dao = self
@@ -890,7 +860,7 @@ impl InsightGenerator {
.flatten()
};
location_cx.span().set_status(Status::Ok);
span.set_status(Status::Ok);
if let Some(loc) = nearest {
// Check if this adds NEW information compared to EXIF
@@ -943,17 +913,16 @@ impl InsightGenerator {
/// Gather search context for photo date
/// Uses semantic search on queries (±30 days, top 5 relevant)
#[tracing::instrument(name = "ai.context.search", skip_all)]
async fn gather_search_context(
&self,
parent_cx: &opentelemetry::Context,
timestamp: i64,
location: Option<&str>,
contact: Option<&str>,
enrichment: Option<&str>,
) -> Result<Option<String>> {
let tracer = global_tracer();
let span = tracer.start_with_context("ai.context.search", parent_cx);
let search_cx = parent_cx.with_span(span);
let span = tracing::Span::current();
let search_cx = span.context();
// Use enrichment (topics + photo description + tags) if available;
// fall back to generic temporal query.
@@ -973,23 +942,18 @@ impl InsightGenerator {
)
};
let query_embedding = match crate::ai::embed_query(
&search_cx,
&self.ollama,
self.llamacpp.as_deref(),
&query_text,
)
.await
{
Ok(emb) => emb,
Err(e) => {
log::warn!("Failed to generate search embedding: {}", e);
search_cx.span().set_status(Status::Error {
description: e.to_string().into(),
});
return Ok(None);
}
};
let query_embedding =
match crate::ai::embed_query(&self.ollama, self.llamacpp.as_deref(), &query_text).await
{
Ok(emb) => emb,
Err(e) => {
log::warn!("Failed to generate search embedding: {}", e);
span.set_status(Status::Error {
description: e.to_string().into(),
});
return Ok(None);
}
};
let searches = {
let mut dao = self
@@ -1006,7 +970,7 @@ impl InsightGenerator {
.ok()
};
search_cx.span().set_status(Status::Ok);
span.set_status(Status::Ok);
if let Some(searches) = searches {
if searches.is_empty() {
@@ -1068,6 +1032,7 @@ impl InsightGenerator {
}
/// Generate AI insight for a single photo with custom configuration
#[tracing::instrument(name = "ai.insight.generate", skip_all)]
pub async fn generate_insight_for_photo_with_config(
&self,
file_path: &str,
@@ -1079,20 +1044,18 @@ impl InsightGenerator {
top_k: Option<i32>,
min_p: Option<f32>,
) -> Result<()> {
let tracer = global_tracer();
let current_cx = opentelemetry::Context::current();
let mut span = tracer.start_with_context("ai.insight.generate", &current_cx);
let span = tracing::Span::current();
// Normalize path to ensure consistent forward slashes in database
let file_path = normalize_path(file_path);
log::info!("Generating insight for photo: {}", file_path);
span.set_attribute(KeyValue::new("file_path", file_path.clone()));
span.set_attribute("file_path", file_path.clone());
// Create custom Ollama client if model is specified
let mut ollama_client = if let Some(model) = custom_model {
log::info!("Using custom model: {}", model);
span.set_attribute(KeyValue::new("custom_model", model.clone()));
span.set_attribute("custom_model", model.clone());
OllamaClient::new(
self.ollama.primary_url.clone(),
self.ollama.fallback_url.clone(),
@@ -1100,14 +1063,14 @@ impl InsightGenerator {
Some(model), // Use the same custom model for fallback server
)
} else {
span.set_attribute(KeyValue::new("model", self.ollama.primary_model.clone()));
span.set_attribute("model", self.ollama.primary_model.clone());
self.ollama.clone()
};
// Set context size if specified
if let Some(ctx) = num_ctx {
log::info!("Using custom context size: {}", ctx);
span.set_attribute(KeyValue::new("num_ctx", ctx as i64));
span.set_attribute("num_ctx", ctx as i64);
ollama_client.set_num_ctx(Some(ctx));
}
@@ -1121,22 +1084,22 @@ impl InsightGenerator {
min_p
);
if let Some(t) = temperature {
span.set_attribute(KeyValue::new("temperature", t as f64));
span.set_attribute("temperature", t as f64);
}
if let Some(p) = top_p {
span.set_attribute(KeyValue::new("top_p", p as f64));
span.set_attribute("top_p", p as f64);
}
if let Some(k) = top_k {
span.set_attribute(KeyValue::new("top_k", k as i64));
span.set_attribute("top_k", k as i64);
}
if let Some(m) = min_p {
span.set_attribute(KeyValue::new("min_p", m as f64));
span.set_attribute("min_p", m as f64);
}
ollama_client.set_sampling_params(temperature, top_p, top_k, min_p);
}
// Create context with this span for child operations
let insight_cx = current_cx.with_span(span);
let insight_cx = span.context();
// 1. Get EXIF data for the photo
let exif = {
@@ -1153,13 +1116,9 @@ impl InsightGenerator {
let contact = Self::extract_contact_from_path(&file_path);
log::info!("Extracted contact from path: {:?}", contact);
insight_cx
.span()
.set_attribute(KeyValue::new("date_taken", date_taken.to_string()));
span.set_attribute("date_taken", date_taken.to_string());
if let Some(ref c) = contact {
insight_cx
.span()
.set_attribute(KeyValue::new("contact", c.clone()));
span.set_attribute("contact", c.clone());
}
// Fetch file tags (used to enrich RAG and final context)
@@ -1197,9 +1156,7 @@ impl InsightGenerator {
.into_iter()
.next();
let combined = compose_location_string(apollo_primary, nominatim, lat, lon);
insight_cx
.span()
.set_attribute(KeyValue::new("location", combined.clone()));
span.set_attribute("location", combined.clone());
Some(combined)
} else {
None
@@ -1234,9 +1191,7 @@ impl InsightGenerator {
}
};
insight_cx
.span()
.set_attribute(KeyValue::new("model_has_vision", has_vision));
span.set_attribute("model_has_vision", has_vision);
// Load image and encode as base64 only if model supports vision
let image_base64 = if has_vision {
@@ -1263,10 +1218,7 @@ impl InsightGenerator {
// Generate brief photo description for RAG enrichment (vision models only)
let photo_description: Option<String> = if let Some(ref img_b64) = image_base64 {
match ollama_client
.generate_photo_description(&insight_cx, img_b64)
.await
{
match ollama_client.generate_photo_description(img_b64).await {
Ok(desc) => {
log::info!("Photo description for RAG enrichment: {}", desc);
Some(desc)
@@ -1335,7 +1287,7 @@ impl InsightGenerator {
if !immediate_messages.is_empty() {
// Step 2: Extract topics from immediate messages to enrich RAG query
let topics = self
.extract_topics_from_messages(&immediate_messages, &ollama_client, &insight_cx)
.extract_topics_from_messages(&immediate_messages, &ollama_client)
.await;
log::info!("Extracted topics for query enrichment: {:?}", topics);
@@ -1367,7 +1319,6 @@ impl InsightGenerator {
};
match self
.find_relevant_messages_rag_historical(
&insight_cx,
date_taken,
None,
contact.as_deref(),
@@ -1391,7 +1342,6 @@ impl InsightGenerator {
&immediate_messages,
&ollama_client,
custom_system_prompt.as_deref(),
&insight_cx,
)
.await
.unwrap_or_else(|| String::from("No immediate context"));
@@ -1401,7 +1351,6 @@ impl InsightGenerator {
&historical_messages,
&ollama_client,
custom_system_prompt.as_deref(),
&insight_cx,
)
.await
.unwrap_or_else(|| String::from("No historical context"));
@@ -1420,7 +1369,6 @@ impl InsightGenerator {
&immediate_messages,
&ollama_client,
custom_system_prompt.as_deref(),
&insight_cx,
)
.await;
}
@@ -1431,7 +1379,6 @@ impl InsightGenerator {
&immediate_messages,
&ollama_client,
custom_system_prompt.as_deref(),
&insight_cx,
)
.await;
}
@@ -1457,7 +1404,6 @@ impl InsightGenerator {
&rag_messages,
&ollama_client,
custom_system_prompt.as_deref(),
&insight_cx,
)
.await;
}
@@ -1490,7 +1436,7 @@ impl InsightGenerator {
if !sms_messages.is_empty() {
match self
.sms_client
.summarize_context(&sms_messages, &ollama_client, &insight_cx)
.summarize_context(&sms_messages, &ollama_client)
.await
{
Ok(summary) => {
@@ -1504,12 +1450,8 @@ impl InsightGenerator {
}
let retrieval_method = if used_rag { "RAG" } else { "time-based" };
insight_cx
.span()
.set_attribute(KeyValue::new("retrieval_method", retrieval_method));
insight_cx
.span()
.set_attribute(KeyValue::new("has_sms_context", sms_summary.is_some()));
span.set_attribute("retrieval_method", retrieval_method);
span.set_attribute("has_sms_context", sms_summary.is_some());
log::info!(
"Photo context: date={}, location={:?}, retrieval_method={}",
@@ -1520,7 +1462,7 @@ impl InsightGenerator {
// 6. Gather Google Takeout context from all sources
let calendar_context = self
.gather_calendar_context(&insight_cx, timestamp, location.as_deref())
.gather_calendar_context(timestamp, location.as_deref())
.await
.ok()
.flatten();
@@ -1534,14 +1476,13 @@ impl InsightGenerator {
});
let location_context = self
.gather_location_context(&insight_cx, timestamp, exif_gps)
.gather_location_context(timestamp, exif_gps)
.await
.ok()
.flatten();
let search_context = self
.gather_search_context(
&insight_cx,
timestamp,
location.as_deref(),
contact.as_deref(),
@@ -1574,7 +1515,6 @@ impl InsightGenerator {
// 10. Generate summary first, then derive title from the summary
let summary = ollama_client
.generate_photo_summary(
&insight_cx,
date_taken,
location.as_deref(),
contact.as_deref(),
@@ -1585,18 +1525,14 @@ impl InsightGenerator {
.await?;
let title = ollama_client
.generate_photo_title(&insight_cx, &summary, custom_system_prompt.as_deref())
.generate_photo_title(&summary, custom_system_prompt.as_deref())
.await?;
log::info!("Generated title: {}", title);
log::info!("Generated summary: {}", summary);
insight_cx
.span()
.set_attribute(KeyValue::new("title_length", title.len() as i64));
insight_cx
.span()
.set_attribute(KeyValue::new("summary_length", summary.len() as i64));
span.set_attribute("title_length", title.len() as i64);
span.set_attribute("summary_length", summary.len() as i64);
// 11. Store in database. content_hash is None here — store_insight
// looks it up from image_exif before persisting; reconciliation
@@ -1635,11 +1571,11 @@ impl InsightGenerator {
match &result {
Ok(_) => {
log::info!("Successfully stored insight for {}", file_path);
insight_cx.span().set_status(Status::Ok);
span.set_status(Status::Ok);
}
Err(e) => {
log::error!("Failed to store insight: {:?}", e);
insight_cx.span().set_status(Status::error(e.to_string()));
span.set_status(Status::error(e.to_string()));
}
}
@@ -1652,7 +1588,6 @@ impl InsightGenerator {
&self,
messages: &[crate::ai::SmsMessage],
ollama: &OllamaClient,
cx: &opentelemetry::Context,
) -> Vec<String> {
if messages.is_empty() {
return Vec::new();
@@ -1688,7 +1623,7 @@ Return ONLY the comma-separated list, nothing else."#,
);
match ollama
.generate(cx, &prompt, Some("You are an entity extraction assistant. Extract proper nouns, people, places, and domain-specific terms that provide context."))
.generate(&prompt, Some("You are an entity extraction assistant. Extract proper nouns, people, places, and domain-specific terms that provide context."))
.await
{
Ok(response) => {
@@ -1723,7 +1658,6 @@ Return ONLY the comma-separated list, nothing else."#,
messages: &[String],
ollama: &OllamaClient,
custom_system: Option<&str>,
cx: &opentelemetry::Context,
) -> Option<String> {
if messages.is_empty() {
return None;
@@ -1744,7 +1678,7 @@ Return ONLY the summary, nothing else."#,
let system = custom_system
.unwrap_or("You are a context summarization assistant. Be concise and factual.");
match ollama.generate(cx, &prompt, Some(system)).await {
match ollama.generate(&prompt, Some(system)).await {
Ok(summary) => Some(summary),
Err(e) => {
log::warn!("Failed to summarize messages: {}", e);
@@ -1760,7 +1694,6 @@ Return ONLY the summary, nothing else."#,
messages: &[crate::ai::SmsMessage],
ollama: &OllamaClient,
custom_system: Option<&str>,
cx: &opentelemetry::Context,
) -> Option<String> {
if messages.is_empty() {
return None;
@@ -1806,7 +1739,7 @@ Return ONLY the summary, nothing else."#,
"You are a context summarization assistant. Be detailed and factual, preserving important context.",
);
match ollama.generate(cx, &prompt, Some(system)).await {
match ollama.generate(&prompt, Some(system)).await {
Ok(summary) => Some(summary),
Err(e) => {
log::warn!("Failed to summarize immediate context: {}", e);
@@ -1842,7 +1775,7 @@ Return ONLY the summary, nothing else."#,
let model = backend.model();
let backend_label = backend.kind.as_str();
let result = match tool_name {
"search_rag" => self.tool_search_rag(arguments, backend.local(), cx).await,
"search_rag" => self.tool_search_rag(arguments, backend.local()).await,
"search_messages" => self.tool_search_messages(arguments, cx).await,
"get_sms_messages" => self.tool_get_sms_messages(arguments, cx).await,
"get_calendar_events" => self.tool_get_calendar_events(arguments, cx).await,
@@ -1850,7 +1783,7 @@ Return ONLY the summary, nothing else."#,
"get_file_tags" => self.tool_get_file_tags(arguments, cx).await,
"get_faces_in_photo" => self.tool_get_faces_in_photo(arguments, cx).await,
"describe_photo" => {
self.tool_describe_photo(backend.local(), image_base64, cx)
self.tool_describe_photo(backend.local(), image_base64)
.await
}
"reverse_geocode" => self.tool_reverse_geocode(arguments).await,
@@ -1897,12 +1830,7 @@ Return ONLY the summary, nothing else."#,
}
/// Tool: search_rag — semantic search over daily summaries
async fn tool_search_rag(
&self,
args: &serde_json::Value,
local: &dyn LlmClient,
cx: &opentelemetry::Context,
) -> String {
async fn tool_search_rag(&self, args: &serde_json::Value, local: &dyn LlmClient) -> String {
let query = match args.get("query").and_then(|v| v.as_str()) {
Some(q) => q.to_string(),
None => return "Error: missing required parameter 'query'".to_string(),
@@ -1965,10 +1893,7 @@ Return ONLY the summary, nothing else."#,
};
let final_results = if rerank_enabled && results.len() > limit {
match self
.rerank_with_llm(cx, &query, &results, limit, local)
.await
{
match self.rerank_with_llm(&query, &results, limit, local).await {
Ok(reordered) => reordered,
Err(e) => {
log::warn!("rerank failed, using vector order: {}", e);
@@ -1991,7 +1916,6 @@ Return ONLY the summary, nothing else."#,
/// tool_result. On parse failure we fall back to the input order.
async fn rerank_with_llm(
&self,
cx: &opentelemetry::Context,
query: &str,
candidates: &[String],
limit: usize,
@@ -2035,7 +1959,7 @@ Return ONLY the summary, nothing else."#,
let started = std::time::Instant::now();
let system =
Some("You are a terse relevance ranker. You output only numbers separated by commas.");
let response = local.generate(cx, &prompt, system, None).await?;
let response = local.generate(&prompt, system, None).await?;
log::info!(
"rerank: finished in {} ms (prompt={} chars)",
started.elapsed().as_millis(),
@@ -2645,11 +2569,10 @@ Return ONLY the summary, nothing else."#,
&self,
local: &dyn LlmClient,
image_base64: &Option<String>,
cx: &opentelemetry::Context,
) -> String {
log::info!("tool_describe_photo: generating visual description");
match image_base64 {
Some(img) => match local.describe_image(cx, img).await {
Some(img) => match local.describe_image(img).await {
Ok(desc) => desc,
Err(e) => format!("Error describing photo: {}", e),
},
@@ -3057,7 +2980,6 @@ Return ONLY the summary, nothing else."#,
// configured local backend.
let embed_text = format!("{} {}", name, description);
let embedding: Option<Vec<u8>> = match crate::ai::embed_document(
cx,
&self.ollama,
self.llamacpp.as_deref(),
&embed_text,
@@ -4175,6 +4097,7 @@ Return ONLY the summary, nothing else."#,
Ok(ResolvedBackend::new(chat, local, kind, images_inline))
}
#[tracing::instrument(name = "ai.insight.generate_agentic", skip_all)]
pub async fn generate_agentic_insight_for_photo(
&self,
file_path: &str,
@@ -4193,19 +4116,17 @@ Return ONLY the summary, nothing else."#,
user_id: i32,
persona_id: String,
) -> Result<(Option<i32>, Option<i32>, Option<i32>)> {
let tracer = global_tracer();
let current_cx = opentelemetry::Context::current();
let mut span = tracer.start_with_context("ai.insight.generate_agentic", &current_cx);
let span = tracing::Span::current();
let file_path = normalize_path(file_path);
log::info!("Generating agentic insight for photo: {}", file_path);
span.set_attribute(KeyValue::new("file_path", file_path.clone()));
span.set_attribute(KeyValue::new("max_iterations", max_iterations as i64));
span.set_attribute("file_path", file_path.clone());
span.set_attribute("max_iterations", max_iterations as i64);
// 1. Resolve backend + build clients.
let kind = BackendKind::parse(backend.as_deref().unwrap_or("local"))?;
span.set_attribute(KeyValue::new("backend", kind.as_str()));
span.set_attribute("backend", kind.as_str());
let overrides = SamplingOverrides {
model: custom_model,
num_ctx,
@@ -4216,10 +4137,10 @@ Return ONLY the summary, nothing else."#,
enable_thinking,
};
let backend = self.resolve_backend(kind, &overrides).await?;
span.set_attribute(KeyValue::new("model", backend.model().to_string()));
span.set_attribute(KeyValue::new("images_inline", backend.images_inline));
span.set_attribute("model", backend.model().to_string());
span.set_attribute("images_inline", backend.images_inline);
let insight_cx = current_cx.with_span(span);
let insight_cx = span.context();
// 3. Fetch EXIF
let exif = {
@@ -4304,7 +4225,7 @@ Return ONLY the summary, nothing else."#,
// the local backend so non-text work stays off OpenRouter.
let inlined_visual_description: Option<String> = if !backend.images_inline {
match image_base64.as_deref() {
Some(b64) => match backend.local().describe_image(&insight_cx, b64).await {
Some(b64) => match backend.local().describe_image(b64).await {
Ok(desc) => {
log::info!("{}: vision describe succeeded ({} chars)", kind, desc.len());
Some(desc)
@@ -4410,8 +4331,10 @@ Return ONLY the summary, nothing else."#,
let mut messages = vec![system_msg, user_msg];
let loop_span = tracer.start_with_context("ai.agentic.loop", &insight_cx);
let loop_cx = insight_cx.with_span(loop_span);
// LLM / tool awaits below run inside `loop_span` so their spans
// nest under it.
let loop_span = tracing::info_span!("ai.agentic.loop");
let loop_cx = loop_span.context();
let mut final_content = String::new();
let mut iterations_used = 0usize;
@@ -4424,7 +4347,8 @@ Return ONLY the summary, nothing else."#,
let (response, prompt_tokens, eval_tokens) = backend
.chat()
.chat_with_tools(&loop_cx, messages.clone(), tools.clone())
.chat_with_tools(messages.clone(), tools.clone())
.instrument(loop_span.clone())
.await?;
last_prompt_eval_count = prompt_tokens;
@@ -4470,6 +4394,7 @@ Return ONLY the summary, nothing else."#,
&persona_id,
&loop_cx,
)
.instrument(loop_span.clone())
.await;
messages.push(ChatMessage::tool_result(result));
}
@@ -4493,7 +4418,8 @@ Return ONLY the summary, nothing else."#,
)));
let (final_response, prompt_tokens, eval_tokens) = backend
.chat()
.chat_with_tools(&loop_cx, messages.clone(), tools.clone())
.chat_with_tools(messages.clone(), tools.clone())
.instrument(loop_span.clone())
.await?;
last_prompt_eval_count = prompt_tokens;
last_eval_count = eval_tokens;
@@ -4521,12 +4447,14 @@ Return ONLY the summary, nothing else."#,
&persona_id,
&loop_cx,
)
.instrument(loop_span.clone())
.await;
messages.push(ChatMessage::tool_result(result));
}
let (resp2, pt2, et2) = backend
.chat()
.chat_with_tools(&loop_cx, messages.clone(), vec![])
.chat_with_tools(messages.clone(), vec![])
.instrument(loop_span.clone())
.await?;
last_prompt_eval_count = pt2;
last_eval_count = et2;
@@ -4538,10 +4466,9 @@ Return ONLY the summary, nothing else."#,
}
}
loop_cx
.span()
.set_attribute(KeyValue::new("iterations_used", iterations_used as i64));
loop_cx.span().set_status(Status::Ok);
loop_span.set_attribute("iterations_used", iterations_used as i64);
loop_span.set_status(Status::Ok);
drop(loop_span);
// 13. Strip any leaked <think>…</think> reasoning block (thinking
// models emit it ahead of the answer; the raw transcript in
@@ -4608,11 +4535,11 @@ Return ONLY the summary, nothing else."#,
match &stored {
Ok(_) => {
log::info!("Successfully stored agentic insight for {}", file_path);
insight_cx.span().set_status(Status::Ok);
span.set_status(Status::Ok);
}
Err(e) => {
log::error!("Failed to store agentic insight: {:?}", e);
insight_cx.span().set_status(Status::error(e.to_string()));
span.set_status(Status::error(e.to_string()));
}
}
@@ -4672,7 +4599,7 @@ Return ONLY the summary, nothing else."#,
let (response, _prompt_tokens, _eval_tokens) = backend
.chat()
.chat_with_tools(&loop_cx, messages.clone(), tools.clone())
.chat_with_tools(messages.clone(), tools.clone())
.await?;
// Sanitize tool call arguments before pushing back into history.
@@ -4738,7 +4665,7 @@ Return ONLY the summary, nothing else."#,
));
let (final_response, _, _) = backend
.chat()
.chat_with_tools(&loop_cx, messages.clone(), tools.clone())
.chat_with_tools(messages.clone(), tools.clone())
.await?;
// Same bounded tool round as the insight loop: execute any last
// tool call, then force a tool-less turn for the actual answer.
@@ -4763,7 +4690,7 @@ Return ONLY the summary, nothing else."#,
}
let (resp2, _, _) = backend
.chat()
.chat_with_tools(&loop_cx, messages.clone(), vec![])
.chat_with_tools(messages.clone(), vec![])
.await?;
final_content = resp2.content;
} else {