Apps/ImageApi
1
0 Fork 0
Code Issues 7 Pull Requests Releases Wiki Activity

Otel: llm.request client spans + traceparent on LLM calls (#130)

Browse Source 
Add OpenTelemetry tracing to every LLM client call so an insight/chat
turn is a single trace from HTTP entry to each downstream inference.

- LlmClient trait: thread a &opentelemetry::Context through all 5
  methods (generate, chat_with_tools, chat_with_tools_stream,
  generate_embeddings, describe_image), matching the DAO convention of
  explicit context threading rather than ambient Context::current().
- otel::start_llm_request_span: single shared helper that every
  provider (openrouter / llamacpp / ollama) funnels through, so the
  span shape is identical: name llm.request, SpanKind::Client,
  llm.provider / llm.operation / llm.model.
- Per-method spans in all three clients with the spec's operation enum
  (chat | chat_stream | embeddings | describe_image | generate); stream
  spans are owned and moved into the async_stream generator so they end
  when the response drains, not when headers arrive.
- llm.tokens.input / llm.tokens.output recorded on non-stream and
  stream completion; llm.cost_usd from usage.cost (OpenRouter);
  http.status_code + error status on failure.
- W3C traceparent injected on outgoing LLM HTTP requests (llama-swap,
  Ollama, OpenRouter) gated on span_context.is_valid() so debug
  no-op-tracer builds don't send an all-zero parent.
- Turn-level spans (ai.chat.turn.execute, ai.persona_chat.turn.execute)
  now carry llm.backend + llm.model; with_entry streaming paths thread
  the turn context (turn_cx) to child describe_image / agentic-loop
  calls instead of the ambient context.

clippy clean; 690 lib + 775 integration + 4 doc tests green.
This commit is contained in:
Cameron Cordes committed 2026-09-28 01:21:46 -04:00
1 parent e84ac82f86
commit 696d9fa8f3
17 files changed
+910 -234
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/ai/insight_generator.rs
+64 -36
View File
@@ -646,7 +646,8 @@ impl InsightGenerator {
// (`LLM_BACKEND` switch). Must match the backend that populated the
// daily-summary embeddings or similarity search will be garbage.
let query_embedding =
crate::ai::embed_query(&self.ollama, self.llamacpp.as_deref(), &query).await?;
crate::ai::embed_query(&search_cx, &self.ollama, self.llamacpp.as_deref(), &query)
.await?;
// Search for similar daily summaries with time-based weighting
// This prioritizes summaries temporally close to the query date
@@ -712,7 +713,8 @@ impl InsightGenerator {
// Must use the same backend that populated the daily-summary
// embeddings or similarity search is garbage (see embed_one docs).
let query_embedding =
crate::ai::embed_query(&self.ollama, self.llamacpp.as_deref(), query).await?;
crate::ai::embed_query(&search_cx, &self.ollama, self.llamacpp.as_deref(), query)
.await?;
let mut summary_dao = self
.daily_summary_dao
@@ -798,7 +800,9 @@ impl InsightGenerator {
let calendar_cx = parent_cx.with_span(span);
let query_embedding = if let Some(loc) = location {
match crate::ai::embed_query(&self.ollama, self.llamacpp.as_deref(), loc).await {
match crate::ai::embed_query(&calendar_cx, &self.ollama, self.llamacpp.as_deref(), loc)
.await
{
Ok(emb) => Some(emb),
Err(e) => {
log::warn!("Failed to generate embedding for location '{}': {}", loc, e);
@@ -969,18 +973,23 @@ impl InsightGenerator {
)
};
let query_embedding =
match crate::ai::embed_query(&self.ollama, self.llamacpp.as_deref(), &query_text).await
{
Ok(emb) => emb,
Err(e) => {
log::warn!("Failed to generate search embedding: {}", e);
search_cx.span().set_status(Status::Error {
description: e.to_string().into(),
});
return Ok(None);
}
};
let query_embedding = match crate::ai::embed_query(
&search_cx,
&self.ollama,
self.llamacpp.as_deref(),
&query_text,
)
.await
{
Ok(emb) => emb,
Err(e) => {
log::warn!("Failed to generate search embedding: {}", e);
search_cx.span().set_status(Status::Error {
description: e.to_string().into(),
});
return Ok(None);
}
};
let searches = {
let mut dao = self
@@ -1254,7 +1263,10 @@ impl InsightGenerator {
// Generate brief photo description for RAG enrichment (vision models only)
let photo_description: Option<String> = if let Some(ref img_b64) = image_base64 {
match ollama_client.generate_photo_description(img_b64).await {
match ollama_client
.generate_photo_description(&insight_cx, img_b64)
.await
{
Ok(desc) => {
log::info!("Photo description for RAG enrichment: {}", desc);
Some(desc)
@@ -1323,7 +1335,7 @@ impl InsightGenerator {
if !immediate_messages.is_empty() {
// Step 2: Extract topics from immediate messages to enrich RAG query
let topics = self
.extract_topics_from_messages(&immediate_messages, &ollama_client)
.extract_topics_from_messages(&immediate_messages, &ollama_client, &insight_cx)
.await;
log::info!("Extracted topics for query enrichment: {:?}", topics);
@@ -1379,6 +1391,7 @@ impl InsightGenerator {
&immediate_messages,
&ollama_client,
custom_system_prompt.as_deref(),
&insight_cx,
)
.await
.unwrap_or_else(|| String::from("No immediate context"));
@@ -1388,6 +1401,7 @@ impl InsightGenerator {
&historical_messages,
&ollama_client,
custom_system_prompt.as_deref(),
&insight_cx,
)
.await
.unwrap_or_else(|| String::from("No historical context"));
@@ -1406,6 +1420,7 @@ impl InsightGenerator {
&immediate_messages,
&ollama_client,
custom_system_prompt.as_deref(),
&insight_cx,
)
.await;
}
@@ -1416,6 +1431,7 @@ impl InsightGenerator {
&immediate_messages,
&ollama_client,
custom_system_prompt.as_deref(),
&insight_cx,
)
.await;
}
@@ -1441,6 +1457,7 @@ impl InsightGenerator {
&rag_messages,
&ollama_client,
custom_system_prompt.as_deref(),
&insight_cx,
)
.await;
}
@@ -1473,7 +1490,7 @@ impl InsightGenerator {
if !sms_messages.is_empty() {
match self
.sms_client
.summarize_context(&sms_messages, &ollama_client)
.summarize_context(&sms_messages, &ollama_client, &insight_cx)
.await
{
Ok(summary) => {
@@ -1557,6 +1574,7 @@ impl InsightGenerator {
// 10. Generate summary first, then derive title from the summary
let summary = ollama_client
.generate_photo_summary(
&insight_cx,
date_taken,
location.as_deref(),
contact.as_deref(),
@@ -1567,7 +1585,7 @@ impl InsightGenerator {
.await?;
let title = ollama_client
.generate_photo_title(&summary, custom_system_prompt.as_deref())
.generate_photo_title(&insight_cx, &summary, custom_system_prompt.as_deref())
.await?;
log::info!("Generated title: {}", title);
@@ -1634,6 +1652,7 @@ impl InsightGenerator {
&self,
messages: &[crate::ai::SmsMessage],
ollama: &OllamaClient,
cx: &opentelemetry::Context,
) -> Vec<String> {
if messages.is_empty() {
return Vec::new();
@@ -1669,7 +1688,7 @@ Return ONLY the comma-separated list, nothing else."#,
);
match ollama
.generate(&prompt, Some("You are an entity extraction assistant. Extract proper nouns, people, places, and domain-specific terms that provide context."))
.generate(cx, &prompt, Some("You are an entity extraction assistant. Extract proper nouns, people, places, and domain-specific terms that provide context."))
.await
{
Ok(response) => {
@@ -1704,6 +1723,7 @@ Return ONLY the comma-separated list, nothing else."#,
messages: &[String],
ollama: &OllamaClient,
custom_system: Option<&str>,
cx: &opentelemetry::Context,
) -> Option<String> {
if messages.is_empty() {
return None;
@@ -1724,7 +1744,7 @@ Return ONLY the summary, nothing else."#,
let system = custom_system
.unwrap_or("You are a context summarization assistant. Be concise and factual.");
match ollama.generate(&prompt, Some(system)).await {
match ollama.generate(cx, &prompt, Some(system)).await {
Ok(summary) => Some(summary),
Err(e) => {
log::warn!("Failed to summarize messages: {}", e);
@@ -1740,6 +1760,7 @@ Return ONLY the summary, nothing else."#,
messages: &[crate::ai::SmsMessage],
ollama: &OllamaClient,
custom_system: Option<&str>,
cx: &opentelemetry::Context,
) -> Option<String> {
if messages.is_empty() {
return None;
@@ -1785,7 +1806,7 @@ Return ONLY the summary, nothing else."#,
"You are a context summarization assistant. Be detailed and factual, preserving important context.",
);
match ollama.generate(&prompt, Some(system)).await {
match ollama.generate(cx, &prompt, Some(system)).await {
Ok(summary) => Some(summary),
Err(e) => {
log::warn!("Failed to summarize immediate context: {}", e);
@@ -1829,7 +1850,7 @@ Return ONLY the summary, nothing else."#,
"get_file_tags" => self.tool_get_file_tags(arguments, cx).await,
"get_faces_in_photo" => self.tool_get_faces_in_photo(arguments, cx).await,
"describe_photo" => {
self.tool_describe_photo(backend.local(), image_base64)
self.tool_describe_photo(backend.local(), image_base64, cx)
.await
}
"reverse_geocode" => self.tool_reverse_geocode(arguments).await,
@@ -1880,7 +1901,7 @@ Return ONLY the summary, nothing else."#,
&self,
args: &serde_json::Value,
local: &dyn LlmClient,
_cx: &opentelemetry::Context,
cx: &opentelemetry::Context,
) -> String {
let query = match args.get("query").and_then(|v| v.as_str()) {
Some(q) => q.to_string(),
@@ -1944,7 +1965,10 @@ Return ONLY the summary, nothing else."#,
};
let final_results = if rerank_enabled && results.len() > limit {
match self.rerank_with_llm(&query, &results, limit, local).await {
match self
.rerank_with_llm(cx, &query, &results, limit, local)
.await
{
Ok(reordered) => reordered,
Err(e) => {
log::warn!("rerank failed, using vector order: {}", e);
@@ -1967,6 +1991,7 @@ Return ONLY the summary, nothing else."#,
/// tool_result. On parse failure we fall back to the input order.
async fn rerank_with_llm(
&self,
cx: &opentelemetry::Context,
query: &str,
candidates: &[String],
limit: usize,
@@ -2010,7 +2035,7 @@ Return ONLY the summary, nothing else."#,
let started = std::time::Instant::now();
let system =
Some("You are a terse relevance ranker. You output only numbers separated by commas.");
let response = local.generate(&prompt, system, None).await?;
let response = local.generate(cx, &prompt, system, None).await?;
log::info!(
"rerank: finished in {} ms (prompt={} chars)",
started.elapsed().as_millis(),
@@ -2620,10 +2645,11 @@ Return ONLY the summary, nothing else."#,
&self,
local: &dyn LlmClient,
image_base64: &Option<String>,
cx: &opentelemetry::Context,
) -> String {
log::info!("tool_describe_photo: generating visual description");
match image_base64 {
Some(img) => match local.describe_image(img).await {
Some(img) => match local.describe_image(cx, img).await {
Ok(desc) => desc,
Err(e) => format!("Error describing photo: {}", e),
},
@@ -3031,6 +3057,7 @@ Return ONLY the summary, nothing else."#,
// configured local backend.
let embed_text = format!("{} {}", name, description);
let embedding: Option<Vec<u8>> = match crate::ai::embed_document(
cx,
&self.ollama,
self.llamacpp.as_deref(),
&embed_text,
@@ -4277,7 +4304,7 @@ Return ONLY the summary, nothing else."#,
// the local backend so non-text work stays off OpenRouter.
let inlined_visual_description: Option<String> = if !backend.images_inline {
match image_base64.as_deref() {
Some(b64) => match backend.local().describe_image(b64).await {
Some(b64) => match backend.local().describe_image(&insight_cx, b64).await {
Ok(desc) => {
log::info!("{}: vision describe succeeded ({} chars)", kind, desc.len());
Some(desc)
@@ -4397,7 +4424,7 @@ Return ONLY the summary, nothing else."#,
let (response, prompt_tokens, eval_tokens) = backend
.chat()
.chat_with_tools(messages.clone(), tools.clone())
.chat_with_tools(&loop_cx, messages.clone(), tools.clone())
.await?;
last_prompt_eval_count = prompt_tokens;
@@ -4466,7 +4493,7 @@ Return ONLY the summary, nothing else."#,
)));
let (final_response, prompt_tokens, eval_tokens) = backend
.chat()
.chat_with_tools(messages.clone(), tools.clone())
.chat_with_tools(&loop_cx, messages.clone(), tools.clone())
.await?;
last_prompt_eval_count = prompt_tokens;
last_eval_count = eval_tokens;
@@ -4499,7 +4526,7 @@ Return ONLY the summary, nothing else."#,
}
let (resp2, pt2, et2) = backend
.chat()
.chat_with_tools(messages.clone(), vec![])
.chat_with_tools(&loop_cx, messages.clone(), vec![])
.await?;
last_prompt_eval_count = pt2;
last_eval_count = et2;
@@ -4637,6 +4664,7 @@ Return ONLY the summary, nothing else."#,
tools: Vec<Tool>,
max_iter: usize,
) -> Result<String> {
let loop_cx = opentelemetry::Context::current();
let mut final_content = String::new();
for iteration in 0..max_iter {
@@ -4644,7 +4672,7 @@ Return ONLY the summary, nothing else."#,
let (response, _prompt_tokens, _eval_tokens) = backend
.chat()
.chat_with_tools(messages.clone(), tools.clone())
.chat_with_tools(&loop_cx, messages.clone(), tools.clone())
.await?;
// Sanitize tool call arguments before pushing back into history.
@@ -4686,7 +4714,7 @@ Return ONLY the summary, nothing else."#,
"",
0,
"",
&opentelemetry::Context::new(),
&loop_cx,
)
.await;
messages.push(ChatMessage::tool_result(result));
@@ -4710,7 +4738,7 @@ Return ONLY the summary, nothing else."#,
));
let (final_response, _, _) = backend
.chat()
.chat_with_tools(messages.clone(), tools.clone())
.chat_with_tools(&loop_cx, messages.clone(), tools.clone())
.await?;
// Same bounded tool round as the insight loop: execute any last
// tool call, then force a tool-less turn for the actual answer.
@@ -4728,14 +4756,14 @@ Return ONLY the summary, nothing else."#,
"",
0,
"",
&opentelemetry::Context::new(),
&loop_cx,
)
.await;
messages.push(ChatMessage::tool_result(result));
}
let (resp2, _, _) = backend
.chat()
.chat_with_tools(messages.clone(), vec![])
.chat_with_tools(&loop_cx, messages.clone(), vec![])
.await?;
final_content = resp2.content;
} else {