Otel: llm.request client spans + traceparent on LLM calls (#130)

Add OpenTelemetry tracing to every LLM client call so an insight/chat
turn is a single trace from HTTP entry to each downstream inference.

- LlmClient trait: thread a &opentelemetry::Context through all 5
  methods (generate, chat_with_tools, chat_with_tools_stream,
  generate_embeddings, describe_image), matching the DAO convention of
  explicit context threading rather than ambient Context::current().
- otel::start_llm_request_span: single shared helper that every
  provider (openrouter / llamacpp / ollama) funnels through, so the
  span shape is identical: name llm.request, SpanKind::Client,
  llm.provider / llm.operation / llm.model.
- Per-method spans in all three clients with the spec's operation enum
  (chat | chat_stream | embeddings | describe_image | generate); stream
  spans are owned and moved into the async_stream generator so they end
  when the response drains, not when headers arrive.
- llm.tokens.input / llm.tokens.output recorded on non-stream and
  stream completion; llm.cost_usd from usage.cost (OpenRouter);
  http.status_code + error status on failure.
- W3C traceparent injected on outgoing LLM HTTP requests (llama-swap,
  Ollama, OpenRouter) gated on span_context.is_valid() so debug
  no-op-tracer builds don't send an all-zero parent.
- Turn-level spans (ai.chat.turn.execute, ai.persona_chat.turn.execute)
  now carry llm.backend + llm.model; with_entry streaming paths thread
  the turn context (turn_cx) to child describe_image / agentic-loop
  calls instead of the ambient context.

clippy clean; 690 lib + 775 integration + 4 doc tests green.

src/otel.rs

@@ -2,7 +2,7 @@ use actix_web::HttpRequest;
 use actix_web::http::header::HeaderMap;
 use opentelemetry::global::{BoxedSpan, BoxedTracer};
 use opentelemetry::propagation::TextMapPropagator;
-use opentelemetry::trace::{Span, Status, Tracer};
+use opentelemetry::trace::{Span, SpanBuilder, SpanKind, Status, TraceContextExt, Tracer};
 use opentelemetry::{Context, KeyValue, global};
 use opentelemetry_appender_log::OpenTelemetryLogBridge;
 use opentelemetry_otlp::WithExportConfig;
@@ -77,11 +77,46 @@ impl<'a> opentelemetry::propagation::Extractor for HeaderExtractor<'a> {
         self.0.keys().map(|k| k.as_str()).collect()
     }
 }
+/// Start a client span for a single LLM call. Every provider funnels its
+/// LlmClient methods through this so the span shape (name, `SpanKind::Client`,
+/// provider / operation / model attributes) is identical across
+/// openrouter / llamacpp / ollama. `operation` is one of `chat`,
+/// `chat_stream`, `embeddings`, `describe_image`, `generate`.
+pub fn start_llm_request_span(
+    cx: &Context,
+    provider: &str,
+    operation: &str,
+    model: &str,
+) -> BoxedSpan {
+    let builder = SpanBuilder::from_name("llm.request").with_kind(SpanKind::Client);
+    let mut span = global_tracer().build_with_context(builder, cx);
+    span.set_attribute(KeyValue::new("llm.provider", provider.to_string()));
+    span.set_attribute(KeyValue::new("llm.operation", operation.to_string()));
+    span.set_attribute(KeyValue::new("llm.model", model.to_string()));
+    span
+}
+
 pub fn extract_context_from_request(req: &HttpRequest) -> Context {
     let propagator = TraceContextPropagator::new();
     propagator.extract(&HeaderExtractor(req.headers()))
 }
 
+/// Build the W3C trace-context headers to attach to an outgoing HTTP
+/// request so a downstream service (llama-swap, Ollama) can parent its own
+/// spans onto this trace. Returns an empty vec when the context carries no
+/// valid span — e.g. debug builds, where the global tracer is a no-op and
+/// injecting an all-zero traceparent would produce garbage parent spans.
+pub fn traceparent_headers(context: &Context) -> Vec<(String, String)> {
+    if !context.span().span_context().is_valid() {
+        return Vec::new();
+    }
+    let mut headers: std::collections::HashMap<String, String> = std::collections::HashMap::new();
+    TraceContextPropagator::new().inject_context(context, &mut headers);
+    // The propagator writes an empty tracestate header when there is no
+    // state — drop empty-valued headers rather than sending noise.
+    headers.into_iter().filter(|(_, v)| !v.is_empty()).collect()
+}
+
 pub fn trace_db_call<F, O>(
     context: &Context,
     query_type: &str,
@@ -110,3 +145,59 @@ where
 
     result
 }
+
+#[cfg(test)]
+mod traceparent_tests {
+    use super::*;
+    use opentelemetry::trace::{
+        SpanContext, SpanId, TraceContextExt, TraceFlags, TraceId, TraceState,
+    };
+
+    /// A context carrying a span context with known ids — no global
+    /// provider needed, so tests can't pollute process state.
+    fn context_with_flags(flags: TraceFlags) -> Context {
+        let trace_id = TraceId::from_bytes([
+            0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e,
+            0x0f, 0x10,
+        ]);
+        let span_id = SpanId::from_bytes([0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]);
+        let span_context = SpanContext::new(trace_id, span_id, flags, false, TraceState::default());
+        Context::new().with_remote_span_context(span_context)
+    }
+
+    #[test]
+    fn traceparent_headers_sampled_context() {
+        let cx = context_with_flags(TraceFlags::SAMPLED);
+        let headers = traceparent_headers(&cx);
+        assert_eq!(
+            headers,
+            vec![(
+                "traceparent".to_string(),
+                "00-0102030405060708090a0b0c0d0e0f10-0102030405060708-01".to_string(),
+            )]
+        );
+    }
+
+    #[test]
+    fn traceparent_headers_not_sampled_context() {
+        // A valid-but-unsampled context still propagates; the flags byte
+        // carries the state.
+        let cx = context_with_flags(TraceFlags::NOT_SAMPLED);
+        let headers = traceparent_headers(&cx);
+        assert_eq!(
+            headers,
+            vec![(
+                "traceparent".to_string(),
+                "00-0102030405060708090a0b0c0d0e0f10-0102030405060708-00".to_string(),
+            )]
+        );
+    }
+
+    #[test]
+    fn traceparent_headers_invalid_context_is_empty() {
+        // No span (debug-build no-op tracer) → no header, never an
+        // all-zero traceparent.
+        let cx = Context::new();
+        assert!(traceparent_headers(&cx).is_empty());
+    }
+}