Tracing: LLM clients + streaming spans on tracing, drop cx threading (#140)

LlmClient methods no longer take an OpenTelemetry Context. llm.request is
now a tracing span (otel.kind=client, llm.provider/operation/model) that
parents on the caller's ambient span; traceparent injection reads the
span's OTel context via OpenTelemetrySpanExt.

The src/ai feature spans (handlers, insight generation, chat, persona
chat, daily summaries, TTS) move to #[instrument] / info_span!, and every
LLM-bearing tokio::spawn carries its parent span with .instrument(): the
streaming chat turn (chat_turn_stream_inner), async chat / persona turns,
background insight jobs and TTS jobs. Sub-spans in the agentic loops
(ai.chat.loop, ai.agentic.loop) instrument each LLM / tool await.

DAO calls keep their explicit Context (derived from the tracing span)
until the DAO migration (#141).

tests/otel_llm_spans.rs pins nesting + traceparent for the blocking and
streaming paths across a spawn on a multi-thread runtime.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/ai/llamacpp.rs

@@ -23,12 +23,12 @@
 use anyhow::Context as _;
 use anyhow::{Result, anyhow, bail};
 use async_trait::async_trait;
-use opentelemetry::trace::{Span, Status};
-use opentelemetry::{Context, KeyValue};
+use opentelemetry::trace::Status;
 use reqwest::Client;
 use serde::Deserialize;
 use serde_json::{Value, json};
 use std::time::Duration;
+use tracing_opentelemetry::OpenTelemetrySpanExt as _;
 
 use crate::ai::llm_client::{
     ChatMessage, LlmClient, LlmStreamEvent, ModelCapabilities, Tool, ToolCall, ToolCallFunction,
@@ -489,7 +489,7 @@ impl LlamaCppClient {
     async fn chat_completion_with_model(
         &self,
         model: &str,
-        mut span: Option<&mut opentelemetry::global::BoxedSpan>,
+        span: Option<&tracing::Span>,
         messages: Vec<ChatMessage>,
         tools: Vec<Tool>,
     ) -> Result<(ChatMessage, Option<i32>, Option<i32>)> {
@@ -522,13 +522,13 @@ impl LlamaCppClient {
         // timeout starts (see ai::gpu).
         let _gpu = crate::ai::gpu::llm_lease().await;
         let mut request = self.client.post(&url).json(&Value::Object(body));
-        if let Some(s) = span.as_mut() {
-            request = crate::otel::with_traceparent(request, s.span_context());
+        if let Some(s) = span {
+            request = crate::otel::with_traceparent(request, s);
         }
         let resp = match request.send().await {
             Ok(r) => r,
             Err(e) => {
-                if let Some(s) = span.as_mut() {
+                if let Some(s) = span {
                     s.set_status(Status::error("llama-swap request failed"));
                 }
                 return Err(e).with_context(|| format!("POST {} failed", url));
@@ -538,14 +538,14 @@ impl LlamaCppClient {
         if !resp.status().is_success() {
             let status = resp.status();
             crate::otel::record_http_failure(
-                span.as_deref_mut(),
+                span,
                 status.as_u16(),
                 &format!("llama-swap chat request failed: {status}"),
             );
             let body = resp.text().await.unwrap_or_default();
             bail!("llama-swap chat request failed: {} — {}", status, body);
         }
-        crate::otel::set_http_status(span.as_deref_mut(), resp.status().as_u16());
+        crate::otel::set_http_status(span, resp.status().as_u16());
 
         let parsed: Value = resp.json().await.context("parsing chat response")?;
         let choice = parsed
@@ -578,12 +578,12 @@ impl LlamaCppClient {
 
         log_timings(&parsed, prompt_tokens, completion_tokens);
 
-        if let Some(s) = span.as_mut() {
+        if let Some(s) = span {
             if let Some(pt) = prompt_tokens {
-                s.set_attribute(KeyValue::new("llm.tokens.input", pt as i64));
+                s.set_attribute("llm.tokens.input", pt as i64);
             }
             if let Some(ct) = completion_tokens {
-                s.set_attribute(KeyValue::new("llm.tokens.output", ct as i64));
+                s.set_attribute("llm.tokens.output", ct as i64);
             }
             s.set_status(Status::Ok);
         }
@@ -592,13 +592,8 @@ impl LlamaCppClient {
     }
 
     /// Start the `llm.request` span for an outgoing llama-swap call.
-    fn start_request_span(
-        &self,
-        cx: &Context,
-        operation: &str,
-        model: &str,
-    ) -> opentelemetry::global::BoxedSpan {
-        crate::otel::start_llm_request_span(cx, "llamacpp", operation, model)
+    fn start_request_span(&self, operation: &str, model: &str) -> tracing::Span {
+        crate::otel::start_llm_request_span("llamacpp", operation, model)
     }
 }
 
@@ -606,7 +601,6 @@ impl LlamaCppClient {
 impl LlmClient for LlamaCppClient {
     async fn generate(
         &self,
-        cx: &Context,
         prompt: &str,
         system: Option<&str>,
         images: Option<Vec<String>>,
@@ -621,13 +615,12 @@ impl LlmClient for LlamaCppClient {
 
         // Delegates to chat_with_tools, which starts the llm.request span
         // (operation=chat) — one span per HTTP request.
-        let (reply, _, _) = self.chat_with_tools(cx, messages, Vec::new()).await?;
+        let (reply, _, _) = self.chat_with_tools(messages, Vec::new()).await?;
         Ok(reply.content)
     }
 
     async fn chat_with_tools(
         &self,
-        cx: &Context,
         messages: Vec<ChatMessage>,
         tools: Vec<Tool>,
     ) -> Result<(ChatMessage, Option<i32>, Option<i32>)> {
@@ -637,25 +630,19 @@ impl LlmClient for LlamaCppClient {
             messages.len(),
             tools.len()
         );
-        let mut span = self.start_request_span(cx, "chat", &self.primary_model);
-        self.chat_completion_with_model(
-            &self.primary_model.clone(),
-            Some(&mut span),
-            messages,
-            tools,
-        )
-        .await
+        let span = self.start_request_span("chat", &self.primary_model);
+        self.chat_completion_with_model(&self.primary_model.clone(), Some(&span), messages, tools)
+            .await
     }
 
     async fn chat_with_tools_stream(
         &self,
-        cx: &Context,
         messages: Vec<ChatMessage>,
         tools: Vec<Tool>,
     ) -> Result<BoxStream<'static, Result<LlmStreamEvent>>> {
         // The span outlives this method — it moves into the stream
         // generator and is finalized when the response drains.
-        let mut span = self.start_request_span(cx, "chat_stream", &self.primary_model);
+        let span = self.start_request_span("chat_stream", &self.primary_model);
         let url = format!("{}/chat/completions", self.base_url);
         let roles: Vec<&str> = messages.iter().map(|m| m.role.as_str()).collect();
         log::debug!(
@@ -690,7 +677,7 @@ impl LlmClient for LlamaCppClient {
         // below so the lease spans the whole generation, not just the send.
         let gpu = crate::ai::gpu::llm_lease().await;
         let mut request = self.client.post(&url).json(&Value::Object(body));
-        request = crate::otel::with_traceparent(request, span.span_context());
+        request = crate::otel::with_traceparent(request, &span);
         let resp = request
             .send()
             .await
@@ -699,14 +686,14 @@ impl LlmClient for LlamaCppClient {
         if !resp.status().is_success() {
             let status = resp.status();
             crate::otel::record_http_failure(
-                Some(&mut span),
+                Some(&span),
                 status.as_u16(),
                 &format!("llama-swap stream request failed: {status}"),
             );
             let body = resp.text().await.unwrap_or_default();
             bail!("llama-swap stream request failed: {} — {}", status, body);
         }
-        crate::otel::set_http_status(Some(&mut span), resp.status().as_u16());
+        crate::otel::set_http_status(Some(&span), resp.status().as_u16());
 
         let byte_stream = resp.bytes_stream();
         let stream = async_stream::stream! {
@@ -816,10 +803,10 @@ impl LlmClient for LlamaCppClient {
                 images: None,
             };
             if let Some(pt) = prompt_tokens {
-                span.set_attribute(KeyValue::new("llm.tokens.input", pt as i64));
+                span.set_attribute("llm.tokens.input", pt as i64);
             }
             if let Some(ct) = completion_tokens {
-                span.set_attribute(KeyValue::new("llm.tokens.output", ct as i64));
+                span.set_attribute("llm.tokens.output", ct as i64);
             }
             span.set_status(Status::Ok);
             yield Ok(LlmStreamEvent::Done {
@@ -832,8 +819,8 @@ impl LlmClient for LlamaCppClient {
         Ok(Box::pin(stream))
     }
 
-    async fn generate_embeddings(&self, cx: &Context, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
-        let mut span = self.start_request_span(cx, "embeddings", &self.embedding_model);
+    async fn generate_embeddings(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
+        let span = self.start_request_span("embeddings", &self.embedding_model);
         // Deliberately NO GPU lease: the embed slot sits in llama-swap's
         // always-resident group and never participates in a model swap, so
         // leasing here would only stall searches behind a queued synthesis.
@@ -844,7 +831,7 @@ impl LlmClient for LlamaCppClient {
         });
 
         let mut request = self.client.post(&url).json(&body);
-        request = crate::otel::with_traceparent(request, span.span_context());
+        request = crate::otel::with_traceparent(request, &span);
         let resp = match request.send().await {
             Ok(r) => r,
             Err(e) => {
@@ -856,14 +843,14 @@ impl LlmClient for LlamaCppClient {
         if !resp.status().is_success() {
             let status = resp.status();
             crate::otel::record_http_failure(
-                Some(&mut span),
+                Some(&span),
                 status.as_u16(),
                 &format!("llama-swap embedding request failed: {status}"),
             );
             let body = resp.text().await.unwrap_or_default();
             bail!("llama-swap embedding request failed: {} — {}", status, body);
         }
-        crate::otel::set_http_status(Some(&mut span), resp.status().as_u16());
+        crate::otel::set_http_status(Some(&span), resp.status().as_u16());
 
         #[derive(Deserialize)]
         struct EmbedResponse {
@@ -885,9 +872,9 @@ impl LlmClient for LlamaCppClient {
         Ok(parsed.data.into_iter().map(|i| i.embedding).collect())
     }
 
-    async fn describe_image(&self, cx: &Context, image_base64: &str) -> Result<String> {
+    async fn describe_image(&self, image_base64: &str) -> Result<String> {
         // Own span with operation=describe_image — routes through the vision slot.
-        let mut span = self.start_request_span(cx, "describe_image", &self.vision_model);
+        let span = self.start_request_span("describe_image", &self.vision_model);
         let prompt = "Briefly describe what you see in this image in 1-2 sentences. \
                       Focus on the people, location, and activity.";
         let system = "You are a scene description assistant. Be concise and factual.";
@@ -905,7 +892,7 @@ impl LlmClient for LlamaCppClient {
         let (reply, _, _) = self
             .chat_completion_with_model(
                 &self.vision_model.clone(),
-                Some(&mut span),
+                Some(&span),
                 messages,
                 Vec::new(),
             )