Apps/ImageApi
1
0 Fork 0
Code Issues 7 Pull Requests Releases Wiki Activity

Otel: llm.request client spans + traceparent on LLM calls (#130)

Browse Source 
Add OpenTelemetry tracing to every LLM client call so an insight/chat
turn is a single trace from HTTP entry to each downstream inference.

- LlmClient trait: thread a &opentelemetry::Context through all 5
  methods (generate, chat_with_tools, chat_with_tools_stream,
  generate_embeddings, describe_image), matching the DAO convention of
  explicit context threading rather than ambient Context::current().
- otel::start_llm_request_span: single shared helper that every
  provider (openrouter / llamacpp / ollama) funnels through, so the
  span shape is identical: name llm.request, SpanKind::Client,
  llm.provider / llm.operation / llm.model.
- Per-method spans in all three clients with the spec's operation enum
  (chat | chat_stream | embeddings | describe_image | generate); stream
  spans are owned and moved into the async_stream generator so they end
  when the response drains, not when headers arrive.
- llm.tokens.input / llm.tokens.output recorded on non-stream and
  stream completion; llm.cost_usd from usage.cost (OpenRouter);
  http.status_code + error status on failure.
- W3C traceparent injected on outgoing LLM HTTP requests (llama-swap,
  Ollama, OpenRouter) gated on span_context.is_valid() so debug
  no-op-tracer builds don't send an all-zero parent.
- Turn-level spans (ai.chat.turn.execute, ai.persona_chat.turn.execute)
  now carry llm.backend + llm.model; with_entry streaming paths thread
  the turn context (turn_cx) to child describe_image / agentic-loop
  calls instead of the ambient context.

clippy clean; 690 lib + 775 integration + 4 doc tests green.
This commit is contained in:
Cameron Cordes committed 2026-09-28 01:21:46 -04:00
1 parent e84ac82f86
commit 696d9fa8f3
17 files changed
+910 -234
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/ai/insight_chat.rs
+38 -11
View File
@@ -377,7 +377,7 @@ impl InsightChatService {
let (response, prompt_tokens, eval_tokens) = backend
.chat()
.chat_with_tools(messages.clone(), tools.clone())
.chat_with_tools(&loop_cx, messages.clone(), tools.clone())
.await?;
last_prompt_eval_count = prompt_tokens;
last_eval_count = eval_tokens;
@@ -434,7 +434,7 @@ impl InsightChatService {
let synthetic_idx = push_synthetic_final_prompt(&mut messages);
let (final_response, prompt_tokens, eval_tokens) = backend
.chat()
.chat_with_tools(messages.clone(), vec![])
.chat_with_tools(&loop_cx, messages.clone(), vec![])
.await?;
last_prompt_eval_count = prompt_tokens;
last_eval_count = eval_tokens;
@@ -492,6 +492,7 @@ impl InsightChatService {
let title_raw = backend
.chat()
.generate(
&loop_cx,
&title_prompt,
Some(
"You are my long term memory assistant. Use only the information provided. Do not invent details.",
@@ -852,7 +853,7 @@ impl InsightChatService {
span.set_attribute(KeyValue::new("library_id", library_id as i64));
let result = svc
.run_streaming_turn_with_entry(req, entry_clone.clone())
.run_streaming_turn_with_entry(req, entry_clone.clone(), &mut span)
.await;
if let Err(ref e) = result {
span.set_attribute(KeyValue::new("status", "error"));
@@ -882,6 +883,7 @@ impl InsightChatService {
self: Arc<Self>,
req: ChatTurnRequest,
entry: Arc<TurnEntry>,
turn_span: &mut opentelemetry::global::BoxedSpan,
) -> Result<()> {
if req.user_message.trim().is_empty() {
bail!("user_message must not be empty");
@@ -911,11 +913,11 @@ impl InsightChatService {
if req.regenerate || existing_insight.is_none() {
return self
.run_bootstrap_streaming_with_entry(req, normalized, entry)
.run_bootstrap_streaming_with_entry(req, normalized, entry, turn_span)
.await;
}
let insight = existing_insight.expect("just checked Some above");
self.run_continuation_streaming_with_entry(req, normalized, insight, entry)
self.run_continuation_streaming_with_entry(req, normalized, insight, entry, turn_span)
.await
}
@@ -926,6 +928,7 @@ impl InsightChatService {
normalized: String,
insight: crate::database::models::PhotoInsight,
entry: Arc<TurnEntry>,
turn_span: &mut opentelemetry::global::BoxedSpan,
) -> Result<()> {
let active_persona = req
.persona_id
@@ -981,6 +984,12 @@ impl InsightChatService {
};
let backend = self.generator.resolve_backend(kind, &overrides).await?;
let model_used = backend.model().to_string();
// llm.* attributes on the turn-level span (issue #130): clients can't
// know local-vs-hybrid, but the turn knows its resolved backend.
turn_span.set_attribute(KeyValue::new("llm.backend", kind.as_str()));
turn_span.set_attribute(KeyValue::new("llm.model", model_used.clone()));
let turn_cx = opentelemetry::Context::new()
.with_remote_span_context(turn_span.span_context().clone());
let local_first_user_has_image = messages
.iter()
@@ -1030,6 +1039,7 @@ impl InsightChatService {
&active_persona,
max_iterations,
&entry,
&turn_cx,
)
.await?;
let AgenticLoopOutcome {
@@ -1146,6 +1156,7 @@ impl InsightChatService {
req: ChatTurnRequest,
normalized: String,
entry: Arc<TurnEntry>,
turn_span: &mut opentelemetry::global::BoxedSpan,
) -> Result<()> {
let active_persona = req
.persona_id
@@ -1171,6 +1182,11 @@ impl InsightChatService {
};
let backend = self.generator.resolve_backend(kind, &overrides).await?;
let model_used = backend.model().to_string();
// llm.* attributes on the turn-level span (issue #130).
turn_span.set_attribute(KeyValue::new("llm.backend", kind.as_str()));
turn_span.set_attribute(KeyValue::new("llm.model", model_used.clone()));
let turn_cx = opentelemetry::Context::new()
.with_remote_span_context(turn_span.span_context().clone());
let image_base64: Option<String> = self.generator.load_image_as_base64(&normalized).ok();
@@ -1190,7 +1206,7 @@ impl InsightChatService {
let visual_block = if !backend.images_inline {
match image_base64.as_deref() {
Some(b64) => match backend.local().describe_image(b64).await {
Some(b64) => match backend.local().describe_image(&turn_cx, b64).await {
Ok(desc) => {
format!("Visual description (from local vision model):\n{}\n", desc)
}
@@ -1245,6 +1261,7 @@ impl InsightChatService {
&active_persona,
max_iterations,
&entry,
&turn_cx,
)
.await?;
let AgenticLoopOutcome {
@@ -1333,6 +1350,7 @@ impl InsightChatService {
active_persona: &str,
max_iterations: usize,
entry: &Arc<TurnEntry>,
cx: &opentelemetry::Context,
) -> Result<AgenticLoopOutcome> {
crate::ai::insight_chat::run_streaming_agentic_loop_with_entry(
&self.generator,
@@ -1345,6 +1363,7 @@ impl InsightChatService {
active_persona,
max_iterations,
entry,
cx,
)
.await
}
@@ -1500,6 +1519,7 @@ impl InsightChatService {
&active_persona,
max_iterations,
&tx,
&opentelemetry::Context::current(),
)
.await?;
let AgenticLoopOutcome {
@@ -1667,7 +1687,11 @@ impl InsightChatService {
// images_inline backends send images directly to the chat model.
let visual_block = if !backend.images_inline {
match image_base64.as_deref() {
Some(b64) => match backend.local().describe_image(b64).await {
Some(b64) => match backend
.local()
.describe_image(&opentelemetry::Context::current(), b64)
.await
{
Ok(desc) => {
format!("Visual description (from local vision model):\n{}\n", desc)
}
@@ -1734,6 +1758,7 @@ impl InsightChatService {
&active_persona,
max_iterations,
&tx,
&opentelemetry::Context::current(),
)
.await?;
let AgenticLoopOutcome {
@@ -1816,6 +1841,7 @@ impl InsightChatService {
active_persona: &str,
max_iterations: usize,
tx: &tokio::sync::mpsc::Sender<ChatStreamEvent>,
cx: &opentelemetry::Context,
) -> Result<AgenticLoopOutcome> {
let mut tool_calls_made = 0usize;
let mut iterations_used = 0usize;
@@ -1834,7 +1860,7 @@ impl InsightChatService {
let mut stream = backend
.chat()
.chat_with_tools_stream(messages.clone(), tools.clone())
.chat_with_tools_stream(cx, messages.clone(), tools.clone())
.await?;
let mut final_message: Option<ChatMessage> = None;
@@ -1928,7 +1954,7 @@ impl InsightChatService {
let synthetic_idx = push_synthetic_final_prompt(messages);
let mut stream = backend
.chat()
.chat_with_tools_stream(messages.clone(), vec![])
.chat_with_tools_stream(cx, messages.clone(), vec![])
.await?;
let mut final_message: Option<ChatMessage> = None;
while let Some(ev) = stream.next().await {
@@ -2308,6 +2334,7 @@ pub async fn run_streaming_agentic_loop_with_entry(
active_persona: &str,
max_iterations: usize,
entry: &Arc<TurnEntry>,
cx: &opentelemetry::Context,
) -> Result<AgenticLoopOutcome> {
let mut tool_calls_made = 0usize;
let mut iterations_used = 0usize;
@@ -2337,7 +2364,7 @@ pub async fn run_streaming_agentic_loop_with_entry(
let mut stream = backend
.chat()
.chat_with_tools_stream(messages.clone(), tools.clone())
.chat_with_tools_stream(cx, messages.clone(), tools.clone())
.await?;
let mut final_message: Option<ChatMessage> = None;
@@ -2420,7 +2447,7 @@ pub async fn run_streaming_agentic_loop_with_entry(
let synthetic_idx = push_synthetic_final_prompt(messages);
let mut stream = backend
.chat()
.chat_with_tools_stream(messages.clone(), vec![])
.chat_with_tools_stream(cx, messages.clone(), vec![])
.await?;
let mut final_message: Option<ChatMessage> = None;
while let Some(ev) = stream.next().await {