Otel: llm.request client spans + traceparent on LLM calls (#130)

Add OpenTelemetry tracing to every LLM client call so an insight/chat
turn is a single trace from HTTP entry to each downstream inference.

- LlmClient trait: thread a &opentelemetry::Context through all 5
  methods (generate, chat_with_tools, chat_with_tools_stream,
  generate_embeddings, describe_image), matching the DAO convention of
  explicit context threading rather than ambient Context::current().
- otel::start_llm_request_span: single shared helper that every
  provider (openrouter / llamacpp / ollama) funnels through, so the
  span shape is identical: name llm.request, SpanKind::Client,
  llm.provider / llm.operation / llm.model.
- Per-method spans in all three clients with the spec's operation enum
  (chat | chat_stream | embeddings | describe_image | generate); stream
  spans are owned and moved into the async_stream generator so they end
  when the response drains, not when headers arrive.
- llm.tokens.input / llm.tokens.output recorded on non-stream and
  stream completion; llm.cost_usd from usage.cost (OpenRouter);
  http.status_code + error status on failure.
- W3C traceparent injected on outgoing LLM HTTP requests (llama-swap,
  Ollama, OpenRouter) gated on span_context.is_valid() so debug
  no-op-tracer builds don't send an all-zero parent.
- Turn-level spans (ai.chat.turn.execute, ai.persona_chat.turn.execute)
  now carry llm.backend + llm.model; with_entry streaming paths thread
  the turn context (turn_cx) to child describe_image / agentic-loop
  calls instead of the ambient context.

clippy clean; 690 lib + 775 integration + 4 doc tests green.

src/ai/openrouter.rs

@@ -2,8 +2,11 @@
 // the translation helpers directly.
 #![allow(dead_code)]
 
-use anyhow::{Context, Result, anyhow, bail};
+use anyhow::Context as _;
+use anyhow::{Result, anyhow, bail};
 use async_trait::async_trait;
+use opentelemetry::trace::{Span, Status, TraceContextExt};
+use opentelemetry::{Context, KeyValue};
 use reqwest::Client;
 use serde::Deserialize;
 use serde_json::{Value, json};
@@ -14,6 +17,7 @@ use std::time::{Duration, Instant};
 use crate::ai::llm_client::{
     ChatMessage, LlmClient, LlmStreamEvent, ModelCapabilities, Tool, ToolCall, ToolCallFunction,
 };
+
 use futures::stream::{BoxStream, StreamExt};
 
 const DEFAULT_BASE_URL: &str = "https://openrouter.ai/api/v1";
@@ -144,6 +148,19 @@ impl OpenRouterClient {
         b
     }
 
+    /// Start the `llm.request` span for an outgoing call. `operation` is
+    /// `chat`, `embeddings`, or `describe` — the call shape, not the entry
+    /// point (a `describe_image` funneled through `chat_with_tools` is
+    /// still a `describe`).
+    fn start_request_span(
+        &self,
+        cx: &Context,
+        operation: &str,
+        model: &str,
+    ) -> opentelemetry::global::BoxedSpan {
+        crate::otel::start_llm_request_span(cx, "openrouter", operation, model)
+    }
+
     /// Translate canonical messages to the OpenAI-compatible wire shape.
     ///
     /// Walks in order so it can attach `tool_call_id` to `role=tool` messages
@@ -312,6 +329,7 @@ impl OpenRouterClient {
 impl LlmClient for OpenRouterClient {
     async fn generate(
         &self,
+        cx: &Context,
         prompt: &str,
         system: Option<&str>,
         images: Option<Vec<String>>,
@@ -324,90 +342,34 @@ impl LlmClient for OpenRouterClient {
         user.images = images;
         messages.push(user);
 
-        let (reply, _, _) = self.chat_with_tools(messages, Vec::new()).await?;
+        // Delegates to chat_with_tools, which starts the llm.request span
+        // (operation=chat) — one span per HTTP request.
+        let (reply, _, _) = self.chat_with_tools(cx, messages, Vec::new()).await?;
         Ok(reply.content)
     }
 
     async fn chat_with_tools(
         &self,
+        cx: &Context,
         messages: Vec<ChatMessage>,
         tools: Vec<Tool>,
     ) -> Result<(ChatMessage, Option<i32>, Option<i32>)> {
-        let url = format!("{}/chat/completions", self.base_url);
-        let mut body = serde_json::Map::new();
-        body.insert("model".into(), Value::String(self.primary_model.clone()));
-        body.insert(
-            "messages".into(),
-            Value::Array(Self::messages_to_openai(&messages)),
-        );
-        body.insert("stream".into(), Value::Bool(false));
-        if !tools.is_empty() {
-            body.insert(
-                "tools".into(),
-                serde_json::to_value(&tools).context("serializing tools")?,
-            );
-        }
-        for (k, v) in self.build_options() {
-            body.insert(k.into(), v);
-        }
-
-        log::info!(
-            "OpenRouter chat_with_tools: model={} messages={} tools={}",
-            self.primary_model,
-            messages.len(),
-            tools.len()
-        );
-
-        let resp = self
-            .authed(self.client.post(&url))
-            .json(&Value::Object(body))
-            .send()
+        let mut span = self.start_request_span(cx, "chat", &self.primary_model);
+        self.chat_with_tools_impl(Some(&mut span), messages, tools)
             .await
-            .with_context(|| format!("POST {} failed", url))?;
-
-        if !resp.status().is_success() {
-            let status = resp.status();
-            let body = resp.text().await.unwrap_or_default();
-            bail!("OpenRouter chat request failed: {} — {}", status, body);
-        }
-
-        let parsed: Value = resp.json().await.context("parsing chat response")?;
-        let choice = parsed
-            .get("choices")
-            .and_then(|v| v.as_array())
-            .and_then(|a| a.first())
-            .ok_or_else(|| {
-                anyhow!(
-                    "response missing choices[0]: {}",
-                    extract_openrouter_error_detail(&parsed)
-                )
-            })?;
-        let msg = choice.get("message").ok_or_else(|| {
-            anyhow!(
-                "choices[0] missing message: {}",
-                extract_openrouter_error_detail(&parsed)
-            )
-        })?;
-        let chat_msg = Self::openai_message_to_chat(msg)?;
-
-        let usage = parsed.get("usage");
-        let prompt_tokens = usage
-            .and_then(|u| u.get("prompt_tokens"))
-            .and_then(|v| v.as_i64())
-            .map(|n| n as i32);
-        let completion_tokens = usage
-            .and_then(|u| u.get("completion_tokens"))
-            .and_then(|v| v.as_i64())
-            .map(|n| n as i32);
-
-        Ok((chat_msg, prompt_tokens, completion_tokens))
     }
 
     async fn chat_with_tools_stream(
         &self,
+        cx: &Context,
         messages: Vec<ChatMessage>,
         tools: Vec<Tool>,
     ) -> Result<BoxStream<'static, Result<LlmStreamEvent>>> {
+        // The span outlives this method — the stream yields until the
+        // response fully drains, so the owned span moves into the
+        // generator and is finalized there.
+        let mut span = self.start_request_span(cx, "chat_stream", &self.primary_model);
+
         let url = format!("{}/chat/completions", self.base_url);
         let mut body = serde_json::Map::new();
         body.insert("model".into(), Value::String(self.primary_model.clone()));
@@ -432,18 +394,32 @@ impl LlmClient for OpenRouterClient {
             body.insert(k.into(), v);
         }
 
-        let resp = self
+        let mut request = self
             .authed(self.client.post(&url))
-            .json(&Value::Object(body))
+            .json(&Value::Object(body));
+        for (k, v) in crate::otel::traceparent_headers(
+            &Context::new().with_remote_span_context(span.span_context().clone()),
+        ) {
+            request = request.header(k, v);
+        }
+        let resp = request
             .send()
             .await
             .with_context(|| format!("POST {} failed", url))?;
 
         if !resp.status().is_success() {
             let status = resp.status();
+            span.set_attribute(KeyValue::new("http.status_code", status.as_u16() as i64));
+            span.set_status(Status::error(format!(
+                "OpenRouter stream request failed: {status}"
+            )));
             let body = resp.text().await.unwrap_or_default();
             bail!("OpenRouter stream request failed: {} — {}", status, body);
         }
+        span.set_attribute(KeyValue::new(
+            "http.status_code",
+            resp.status().as_u16() as i64,
+        ));
 
         // OpenAI-compat SSE stream. Each event is `data: <json>\n\n`, with
         // `data: [DONE]` signalling completion. Tool calls arrive as
@@ -467,6 +443,7 @@ impl LlmClient for OpenRouterClient {
                 let chunk = match chunk {
                     Ok(b) => b,
                     Err(e) => {
+                        span.set_status(Status::error(format!("stream read failed: {e}")));
                         yield Err(anyhow!("stream read failed: {}", e));
                         return;
                     }
@@ -517,6 +494,9 @@ impl LlmClient for OpenRouterClient {
                                 .get("completion_tokens")
                                 .and_then(|n| n.as_i64())
                                 .map(|n| n as i32);
+                            if let Some(cost) = usage.get("cost").and_then(|c| c.as_f64()) {
+                                span.set_attribute(KeyValue::new("llm.cost_usd", cost));
+                            }
                         }
 
                         let Some(choices) = v.get("choices").and_then(|c| c.as_array())
@@ -606,6 +586,13 @@ impl LlmClient for OpenRouterClient {
                 tool_calls,
                 images: None,
             };
+            if let Some(pt) = prompt_tokens {
+                span.set_attribute(KeyValue::new("llm.tokens.input", pt as i64));
+            }
+            if let Some(ct) = completion_tokens {
+                span.set_attribute(KeyValue::new("llm.tokens.output", ct as i64));
+            }
+            span.set_status(Status::Ok);
             yield Ok(LlmStreamEvent::Done {
                 message,
                 prompt_eval_count: prompt_tokens,
@@ -616,25 +603,41 @@ impl LlmClient for OpenRouterClient {
         Ok(Box::pin(stream))
     }
 
-    async fn generate_embeddings(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
+    async fn generate_embeddings(&self, cx: &Context, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
+        let mut span = self.start_request_span(cx, "embeddings", &self.embedding_model);
         let url = format!("{}/embeddings", self.base_url);
         let body = json!({
             "model": self.embedding_model,
             "input": texts,
         });
 
-        let resp = self
-            .authed(self.client.post(&url))
-            .json(&body)
+        let mut request = self.authed(self.client.post(&url)).json(&body);
+        for (k, v) in crate::otel::traceparent_headers(
+            &Context::new().with_remote_span_context(span.span_context().clone()),
+        ) {
+            request = request.header(k, v);
+        }
+        let resp = request
             .send()
             .await
-            .with_context(|| format!("POST {} failed", url))?;
+            .with_context(|| format!("POST {} failed", url))
+            .inspect_err(|_| {
+                span.set_status(Status::error("OpenRouter embedding request failed"));
+            })?;
 
         if !resp.status().is_success() {
             let status = resp.status();
+            span.set_attribute(KeyValue::new("http.status_code", status.as_u16() as i64));
+            span.set_status(Status::error(format!(
+                "OpenRouter embedding request failed: {status}"
+            )));
             let body = resp.text().await.unwrap_or_default();
             bail!("OpenRouter embedding request failed: {} — {}", status, body);
         }
+        span.set_attribute(KeyValue::new(
+            "http.status_code",
+            resp.status().as_u16() as i64,
+        ));
 
         #[derive(Deserialize)]
         struct EmbedResponse {
@@ -646,18 +649,28 @@ impl LlmClient for OpenRouterClient {
         }
 
         let parsed: EmbedResponse = resp.json().await.context("parsing embed response")?;
+        span.set_status(Status::Ok);
         Ok(parsed.data.into_iter().map(|i| i.embedding).collect())
     }
 
-    async fn describe_image(&self, image_base64: &str) -> Result<String> {
+    async fn describe_image(&self, cx: &Context, image_base64: &str) -> Result<String> {
+        // Own span with operation=describe_image — the funnel into
+        // chat_with_tools_impl keeps the request on a single span.
+        let mut span = self.start_request_span(cx, "describe_image", &self.primary_model);
         let prompt = "Briefly describe what you see in this image in 1-2 sentences. \
                       Focus on the people, location, and activity.";
-        self.generate(
-            prompt,
-            Some("You are a scene description assistant. Be concise and factual."),
-            Some(vec![image_base64.to_string()]),
-        )
-        .await
+        let mut messages: Vec<ChatMessage> = Vec::new();
+        messages.push(ChatMessage::system(
+            "You are a scene description assistant. Be concise and factual.",
+        ));
+        let mut user = ChatMessage::user(prompt);
+        user.images = Some(vec![image_base64.to_string()]);
+        messages.push(user);
+
+        let (reply, _, _) = self
+            .chat_with_tools_impl(Some(&mut span), messages, Vec::new())
+            .await?;
+        Ok(reply.content)
     }
 
     async fn list_models(&self) -> Result<Vec<ModelCapabilities>> {
@@ -711,6 +724,134 @@ impl LlmClient for OpenRouterClient {
     }
 }
 
+impl OpenRouterClient {
+    /// HTTP core of `chat_with_tools`. `span` is the `llm.request` span
+    /// started by the caller (or a funnelling entry point like
+    /// `describe_image`) so attributes from the response land on the right
+    /// operation label.
+    async fn chat_with_tools_impl(
+        &self,
+        mut span: Option<&mut opentelemetry::global::BoxedSpan>,
+        messages: Vec<ChatMessage>,
+        tools: Vec<Tool>,
+    ) -> Result<(ChatMessage, Option<i32>, Option<i32>)> {
+        let url = format!("{}/chat/completions", self.base_url);
+        let mut body = serde_json::Map::new();
+        body.insert("model".into(), Value::String(self.primary_model.clone()));
+        body.insert(
+            "messages".into(),
+            Value::Array(Self::messages_to_openai(&messages)),
+        );
+        body.insert("stream".into(), Value::Bool(false));
+        if !tools.is_empty() {
+            body.insert(
+                "tools".into(),
+                serde_json::to_value(&tools).context("serializing tools")?,
+            );
+        }
+        for (k, v) in self.build_options() {
+            body.insert(k.into(), v);
+        }
+
+        log::info!(
+            "OpenRouter chat_with_tools: model={} messages={} tools={}",
+            self.primary_model,
+            messages.len(),
+            tools.len()
+        );
+
+        let mut request = self
+            .authed(self.client.post(&url))
+            .json(&Value::Object(body));
+        if let Some(s) = span.as_mut() {
+            for (k, v) in crate::otel::traceparent_headers(
+                &Context::new().with_remote_span_context(s.span_context().clone()),
+            ) {
+                request = request.header(k, v);
+            }
+        }
+        let resp = match request.send().await {
+            Ok(r) => r,
+            Err(e) => {
+                if let Some(s) = span.as_mut() {
+                    s.set_status(Status::error("OpenRouter request failed"));
+                }
+                return Err(e).with_context(|| format!("POST {} failed", url));
+            }
+        };
+
+        if !resp.status().is_success() {
+            let status = resp.status();
+            if let Some(s) = span.as_mut() {
+                s.set_attribute(KeyValue::new("http.status_code", status.as_u16() as i64));
+                s.set_status(Status::error(format!(
+                    "OpenRouter chat request failed: {status}"
+                )));
+            }
+            let body = resp.text().await.unwrap_or_default();
+            bail!("OpenRouter chat request failed: {} — {}", status, body);
+        }
+        if let Some(s) = span.as_mut() {
+            s.set_attribute(KeyValue::new(
+                "http.status_code",
+                resp.status().as_u16() as i64,
+            ));
+        }
+
+        let parsed: Value = match resp.json().await {
+            Ok(v) => v,
+            Err(e) => {
+                if let Some(s) = span.as_mut() {
+                    s.set_status(Status::error("parsing chat response failed"));
+                }
+                return Err(e).context("parsing chat response");
+            }
+        };
+        let choice = parsed
+            .get("choices")
+            .and_then(|v| v.as_array())
+            .and_then(|a| a.first())
+            .ok_or_else(|| {
+                anyhow!(
+                    "response missing choices[0]: {}",
+                    extract_openrouter_error_detail(&parsed)
+                )
+            })?;
+        let msg = choice.get("message").ok_or_else(|| {
+            anyhow!(
+                "choices[0] missing message: {}",
+                extract_openrouter_error_detail(&parsed)
+            )
+        })?;
+        let chat_msg = Self::openai_message_to_chat(msg)?;
+
+        let usage = parsed.get("usage");
+        let prompt_tokens = usage
+            .and_then(|u| u.get("prompt_tokens"))
+            .and_then(|v| v.as_i64())
+            .map(|n| n as i32);
+        let completion_tokens = usage
+            .and_then(|u| u.get("completion_tokens"))
+            .and_then(|v| v.as_i64())
+            .map(|n| n as i32);
+        // OpenRouter reports per-request USD cost in `usage.cost`.
+        if let Some(s) = span.as_mut() {
+            if let Some(pt) = prompt_tokens {
+                s.set_attribute(KeyValue::new("llm.tokens.input", pt as i64));
+            }
+            if let Some(ct) = completion_tokens {
+                s.set_attribute(KeyValue::new("llm.tokens.output", ct as i64));
+            }
+            if let Some(cost) = usage.and_then(|u| u.get("cost")).and_then(|c| c.as_f64()) {
+                s.set_attribute(KeyValue::new("llm.cost_usd", cost));
+            }
+            s.set_status(Status::Ok);
+        }
+
+        Ok((chat_msg, prompt_tokens, completion_tokens))
+    }
+}
+
 /// Extract a diagnostic fragment from an OpenRouter response body that
 /// doesn't match the expected `{choices: [...]}` shape. OpenRouter will
 /// sometimes return 200 OK with `{"error": {"message": "...", "code": ...}}`