Tracing: LLM clients + streaming spans on tracing, drop cx threading (#140)

LlmClient methods no longer take an OpenTelemetry Context. llm.request is
now a tracing span (otel.kind=client, llm.provider/operation/model) that
parents on the caller's ambient span; traceparent injection reads the
span's OTel context via OpenTelemetrySpanExt.

The src/ai feature spans (handlers, insight generation, chat, persona
chat, daily summaries, TTS) move to #[instrument] / info_span!, and every
LLM-bearing tokio::spawn carries its parent span with .instrument(): the
streaming chat turn (chat_turn_stream_inner), async chat / persona turns,
background insight jobs and TTS jobs. Sub-spans in the agentic loops
(ai.chat.loop, ai.agentic.loop) instrument each LLM / tool await.

DAO calls keep their explicit Context (derived from the tracing span)
until the DAO migration (#141).

tests/otel_llm_spans.rs pins nesting + traceparent for the blocking and
streaming paths across a spawn on a multi-thread runtime.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/ai/persona_chat.rs

@@ -22,15 +22,16 @@
 //!   registry.
 //! - The HTTP handlers under ` /persona_chat/*`.
 
-use actix_web::{HttpRequest, HttpResponse, Responder, delete, get, post, web};
+use actix_web::{HttpResponse, Responder, delete, get, post, web};
 use anyhow::{Context, Result, anyhow, bail};
 use chrono::Utc;
-use opentelemetry::KeyValue;
-use opentelemetry::trace::{Span, Status, TraceContextExt, Tracer};
+use opentelemetry::trace::Status;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::sync::{Arc, Mutex as StdMutex};
 use tokio::sync::Mutex as TokioMutex;
+use tracing::Instrument;
+use tracing_opentelemetry::OpenTelemetrySpanExt;
 use uuid::Uuid;
 
 use crate::ai::backend::{BackendKind, SamplingOverrides};
@@ -45,7 +46,6 @@ use crate::ai::turn_registry::{TurnEntry, TurnRegistry};
 use crate::data::Claims;
 use crate::database::PersonaDao;
 use crate::libraries;
-use crate::otel::{extract_context_from_request, global_tracer};
 use crate::state::AppState;
 
 // ─────────────────────────────────────────────────────────────────────
@@ -660,7 +660,6 @@ impl PersonaChatSession {
         let svc = self.clone();
         let persona_for_task = persona.clone();
         let entry_clone = entry.clone();
-        let turn_id_for_task = turn_id.clone();
         let req_for_task = PersonaChatTurnRequest {
             user_message: trimmed.clone(),
             ..req
@@ -669,44 +668,50 @@ impl PersonaChatSession {
             in_flight: self.in_flight.clone(),
             conversation_id: conversation_id.clone(),
         };
-        let conversation_for_span = conversation_id.clone();
-        let handle = tokio::spawn(async move {
-            let tracer = global_tracer();
-            let mut span = tracer.start("ai.persona_chat.turn.execute");
-            span.set_attribute(KeyValue::new("turn_id", turn_id_for_task.clone()));
-            span.set_attribute(KeyValue::new("persona_id", persona_id.clone()));
-            span.set_attribute(KeyValue::new("conversation_id", conversation_for_span));
-            span.set_attribute(KeyValue::new("library_id", library_id as i64));
-
-            let result = svc
-                .run_streaming_turn(
-                    persona_for_task,
-                    req_for_task,
-                    entry_clone.clone(),
-                    user_id,
-                    library_id,
-                    &mut span,
-                )
-                .await;
-            match result {
-                Ok(()) => {
-                    span.set_attribute(KeyValue::new("status", "done"));
-                    span.set_status(Status::Ok);
-                }
-                Err(e) => {
-                    span.set_attribute(KeyValue::new("status", "error"));
-                    span.set_status(Status::error(format!("{e}")));
-                    let _ = entry_clone
-                        .push_event(ChatStreamEvent::Error(format!("{e}")))
-                        .await;
+        // Created before the spawn so it parents on the dispatching request,
+        // then carried into the task with `.instrument` (tracing context is
+        // thread-local and does not cross `tokio::spawn` on its own).
+        let turn_span = tracing::info_span!(
+            "ai.persona_chat.turn.execute",
+            turn_id = %turn_id,
+            persona_id = %persona_id,
+            conversation_id = %conversation_id,
+            library_id = library_id,
+        );
+        let handle = tokio::spawn(
+            async move {
+                let span = tracing::Span::current();
+                let result = svc
+                    .run_streaming_turn(
+                        persona_for_task,
+                        req_for_task,
+                        entry_clone.clone(),
+                        user_id,
+                        library_id,
+                        &span,
+                    )
+                    .await;
+                match result {
+                    Ok(()) => {
+                        span.set_attribute("status", "done");
+                        span.set_status(Status::Ok);
+                    }
+                    Err(e) => {
+                        span.set_attribute("status", "error");
+                        span.set_status(Status::error(format!("{e}")));
+                        let _ = entry_clone
+                            .push_event(ChatStreamEvent::Error(format!("{e}")))
+                            .await;
+                    }
                 }
+                entry_clone.set_terminal_status(crate::ai::turn_registry::TurnStatus::Done);
+                // Frees the in-flight slot. Placed explicitly so the guard is
+                // captured by the spawned task: it must drop when the task ends
+                // (completion, error, or abort) — not when dispatch returns.
+                drop(in_flight_guard);
             }
-            entry_clone.set_terminal_status(crate::ai::turn_registry::TurnStatus::Done);
-            // Frees the in-flight slot. Placed explicitly so the guard is
-            // captured by the spawned task: it must drop when the task ends
-            // (completion, error, or abort) — not when dispatch returns.
-            drop(in_flight_guard);
-        });
+            .instrument(turn_span),
+        );
 
         entry.set_abort_handle(handle.abort_handle());
         Ok(turn_id)
@@ -722,7 +727,7 @@ impl PersonaChatSession {
         entry: Arc<TurnEntry>,
         user_id: i32,
         _library_id: i32,
-        turn_span: &mut opentelemetry::global::BoxedSpan,
+        turn_span: &tracing::Span,
     ) -> Result<()> {
         // Per-conversation mutex — serializes this turn against a
         // concurrent rewind or branch switch on the same transcript.
@@ -810,12 +815,10 @@ impl PersonaChatSession {
             .resolve_backend(backend_kind, &overrides)
             .await?;
         let model_used = backend.model().to_string();
-        // llm.* attributes on the turn-level span (issue #130), plus the
-        // parent context for the child llm.request spans.
-        turn_span.set_attribute(KeyValue::new("llm.backend", backend.kind.as_str()));
-        turn_span.set_attribute(KeyValue::new("llm.model", model_used.clone()));
-        let turn_cx = opentelemetry::Context::new()
-            .with_remote_span_context(turn_span.span_context().clone());
+        // llm.* attributes on the turn-level span (issue #130). The child
+        // llm.request spans parent on it ambiently (the task is instrumented).
+        turn_span.set_attribute("llm.backend", backend.kind.as_str());
+        turn_span.set_attribute("llm.model", model_used.clone());
 
         // Build a no-photo tool catalog. `current_gate_opts_for_persona`
         // already probes each tool's backing table for presence and
@@ -882,7 +885,6 @@ impl PersonaChatSession {
             &persona.persona_id,
             max_iterations,
             &entry,
-            &turn_cx,
         )
         .await
         .map_err(|e| anyhow!("persona chat loop failed: {e:?}"))?;
@@ -951,7 +953,7 @@ impl PersonaChatSession {
         // list falls back to the persona name while the title is empty.
         if needs_title {
             let title = self
-                .generate_title(&backend, &messages, &turn_cx)
+                .generate_title(&backend, &messages)
                 .await
                 .unwrap_or_else(|| fallback_title(&messages));
             if !title.is_empty() {
@@ -1056,7 +1058,6 @@ impl PersonaChatSession {
         &self,
         backend: &crate::ai::backend::ResolvedBackend,
         messages: &[ChatMessage],
-        cx: &opentelemetry::Context,
     ) -> Option<String> {
         let user_message = messages.iter().find(|m| m.role == "user")?;
         // The last assistant message with actual prose — earlier ones may be
@@ -1067,7 +1068,7 @@ impl PersonaChatSession {
             .find(|m| m.role == "assistant" && !m.content.trim().is_empty())?;
 
         let prompt = title_prompt(&user_message.content, &reply.content);
-        match backend.chat().generate(cx, &prompt, None, None).await {
+        match backend.chat().generate(&prompt, None, None).await {
             Ok(raw) => {
                 let title = sanitize_title(&crate::ai::llm_client::strip_think_blocks(&raw));
                 if title.is_empty() { None } else { Some(title) }
@@ -1276,19 +1277,14 @@ pub async fn persona_chat_history_handler(
 /// the `turn_id` immediately; the streaming agent loop runs in a
 /// background task and the client opens the SSE replay stream with it.
 #[post("/persona_chat/turn")]
+#[tracing::instrument(name = "http.persona_chat.turn", skip_all)]
 pub async fn persona_chat_turn_handler(
-    http_request: HttpRequest,
     claims: Claims,
     request: web::Json<PersonaChatTurnRequest>,
     app_state: web::Data<AppState>,
 ) -> impl Responder {
-    let parent_context = extract_context_from_request(&http_request);
-    let tracer = global_tracer();
-    let mut span = tracer.start_with_context("http.persona_chat.turn", &parent_context);
-    span.set_attribute(KeyValue::new(
-        "conversation_id",
-        request.conversation_id.clone(),
-    ));
+    let span = tracing::Span::current();
+    span.set_attribute("conversation_id", request.conversation_id.clone());
 
     // The transcript is not library-scoped, but the turn's tool catalog
     // (memories, SMS, places) resolves against a library — mirror the
@@ -1313,7 +1309,7 @@ pub async fn persona_chat_turn_handler(
         .await
     {
         Ok(turn_id) => {
-            span.set_attribute(KeyValue::new("turn_id", turn_id.clone()));
+            span.set_attribute("turn_id", turn_id.clone());
             span.set_status(Status::Ok);
             HttpResponse::Accepted().json(serde_json::json!({
                 "turn_id": turn_id,
@@ -1540,22 +1536,20 @@ pub async fn persona_chat_branches_handler(
 /// shared renderer.
 #[get("/persona_chat/turn/{turn_id}")]
 pub async fn persona_turn_replay_handler(
-    http_request: HttpRequest,
     path: web::Path<String>,
     query: web::Query<ReplayQuery>,
     app_state: web::Data<AppState>,
 ) -> HttpResponse {
-    crate::ai::handlers::turn_replay_impl(http_request, path, query, app_state).await
+    crate::ai::handlers::turn_replay_impl(path, query, app_state).await
 }
 
 /// DELETE /persona_chat/turn/{turn_id} — cancel a running persona turn.
 #[delete("/persona_chat/turn/{turn_id}")]
 pub async fn persona_turn_cancel_handler(
-    http_request: HttpRequest,
     path: web::Path<String>,
     app_state: web::Data<AppState>,
 ) -> impl Responder {
-    crate::ai::handlers::cancel_turn_impl(http_request, path, app_state).await
+    crate::ai::handlers::cancel_turn_impl(path, app_state).await
 }
 
 // ─────────────────────────────────────────────────────────────────────