Apps/ImageApi
1
0 Fork 0
Code Issues 7 Pull Requests Releases Wiki Activity

Tracing: LLM clients + streaming spans on tracing, drop cx threading (#140)

Browse Source 
LlmClient methods no longer take an OpenTelemetry Context. llm.request is
now a tracing span (otel.kind=client, llm.provider/operation/model) that
parents on the caller's ambient span; traceparent injection reads the
span's OTel context via OpenTelemetrySpanExt.

The src/ai feature spans (handlers, insight generation, chat, persona
chat, daily summaries, TTS) move to #[instrument] / info_span!, and every
LLM-bearing tokio::spawn carries its parent span with .instrument(): the
streaming chat turn (chat_turn_stream_inner), async chat / persona turns,
background insight jobs and TTS jobs. Sub-spans in the agentic loops
(ai.chat.loop, ai.agentic.loop) instrument each LLM / tool await.

DAO calls keep their explicit Context (derived from the tracing span)
until the DAO migration (#141).

tests/otel_llm_spans.rs pins nesting + traceparent for the blocking and
streaming paths across a spawn on a multi-thread runtime.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Cameron CordesandClaude Opus 5.5 committed 2026-09-29 17:02:40 -04:00
1 parent 7a59afc62b
commit 1c508cdabd
20 files changed
+973 -1030
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/ai/tts.rs
+99 -113
View File
@@ -6,13 +6,12 @@
// (audio read directly; video has its audio track extracted via ffmpeg).
use actix_multipart::Multipart;
use actix_web::{HttpRequest, HttpResponse, Responder, delete, get, post, web};
use actix_web::{HttpResponse, Responder, delete, get, post, web};
use anyhow::Context;
use base64::Engine;
use bytes::{BufMut, BytesMut};
use futures::StreamExt;
use opentelemetry::KeyValue;
use opentelemetry::trace::{Span, Status, Tracer};
use opentelemetry::trace::Status;
use regex::Regex;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
@@ -21,6 +20,8 @@ use std::path::Path;
use std::sync::{LazyLock, Mutex as StdMutex};
use std::time::{Duration, Instant};
use tokio::sync::Semaphore;
use tracing::Instrument;
use tracing_opentelemetry::OpenTelemetrySpanExt;
use uuid::Uuid;
use crate::ai::llamacpp::LlamaCppClient;
@@ -28,7 +29,6 @@ use crate::data::Claims;
use crate::file_types::{is_audio_file, is_video_file};
use crate::files::is_valid_full_path;
use crate::libraries;
use crate::otel::{extract_context_from_request, global_tracer};
use crate::state::AppState;
/// Hard cap on an uploaded voice-reference clip. Chatterbox itself caps the
@@ -382,14 +382,13 @@ pub struct TtsSpeechResponse {
/// POST /tts/speech — synthesize `text` (optionally in a named `voice`) and
/// return base64-encoded audio for `data:` URI playback on the client.
#[post("/tts/speech")]
#[tracing::instrument(name = "http.tts.speech", skip_all)]
pub async fn tts_speech_handler(
http_request: HttpRequest,
_claims: Claims,
req: web::Json<TtsSpeechRequest>,
app_state: web::Data<AppState>,
) -> impl Responder {
let parent_context = extract_context_from_request(&http_request);
let mut span = global_tracer().start_with_context("http.tts.speech", &parent_context);
let span = tracing::Span::current();
let text = prepare_for_tts(&req.text);
if text.is_empty() {
@@ -414,10 +413,10 @@ pub async fn tts_speech_handler(
.filter(|s| !s.is_empty())
.or(dv.as_deref());
span.set_attribute(KeyValue::new("tts.model", client.tts_model.clone()));
span.set_attribute(KeyValue::new("tts.format", format.to_string()));
span.set_attribute(KeyValue::new("tts.has_voice", voice.is_some()));
span.set_attribute(KeyValue::new("tts.text_len", text.len() as i64));
span.set_attribute("tts.model", client.tts_model.clone());
span.set_attribute("tts.format", format.to_string());
span.set_attribute("tts.has_voice", voice.is_some());
span.set_attribute("tts.text_len", text.len() as i64);
// Clamp generation knobs to Chatterbox's documented ranges before forwarding.
let exaggeration = req.exaggeration.map(|x| x.clamp(0.25, 2.0));
@@ -441,7 +440,7 @@ pub async fn tts_speech_handler(
.await
{
Ok(bytes) => {
span.set_attribute(KeyValue::new("tts.audio_bytes", bytes.len() as i64));
span.set_attribute("tts.audio_bytes", bytes.len() as i64);
span.set_status(Status::Ok);
let audio_base64 = base64::engine::general_purpose::STANDARD.encode(&bytes);
HttpResponse::Ok().json(TtsSpeechResponse {
@@ -513,15 +512,13 @@ pub async fn synthesize_serialized(
/// permit (instead of fast-failing 429) and the client polls the job until
/// the audio is ready.
#[post("/tts/speech/jobs")]
#[tracing::instrument(name = "http.tts.speech_job.create", skip_all)]
pub async fn create_speech_job_handler(
http_request: HttpRequest,
_claims: Claims,
req: web::Json<TtsSpeechRequest>,
app_state: web::Data<AppState>,
) -> impl Responder {
let parent_context = extract_context_from_request(&http_request);
let mut span =
global_tracer().start_with_context("http.tts.speech_job.create", &parent_context);
let span = tracing::Span::current();
let text = prepare_for_tts(&req.text);
if text.is_empty() {
@@ -550,9 +547,9 @@ pub async fn create_speech_job_handler(
let cfg_weight = req.cfg_weight.map(|x| x.clamp(0.0, 1.0));
let temperature = req.temperature.map(|x| x.clamp(0.05, 5.0));
span.set_attribute(KeyValue::new("tts.format", format.clone()));
span.set_attribute(KeyValue::new("tts.has_voice", voice.is_some()));
span.set_attribute(KeyValue::new("tts.text_len", text.len() as i64));
span.set_attribute("tts.format", format.clone());
span.set_attribute("tts.has_voice", voice.is_some());
span.set_attribute("tts.text_len", text.len() as i64);
let job_id = Uuid::new_v4();
{
@@ -573,75 +570,79 @@ pub async fn create_speech_job_handler(
}
let state = app_state.clone();
let handle = tokio::spawn(async move {
// Queue rather than fast-fail: jobs wait their turn for the GPU.
let _permit = match TTS_PERMIT.acquire().await {
Ok(p) => p,
Err(_) => {
finish_job(
job_id,
TtsJobStatus::Error,
None,
Some("TTS queue closed".to_string()),
);
let handle = tokio::spawn(
async move {
// Queue rather than fast-fail: jobs wait their turn for the GPU.
let _permit = match TTS_PERMIT.acquire().await {
Ok(p) => p,
Err(_) => {
finish_job(
job_id,
TtsJobStatus::Error,
None,
Some("TTS queue closed".to_string()),
);
return;
}
};
// Wait for the LLM side to release the GPU too (see ai::gpu) — only
// then does the job count as running. The synthesis timeout starts at
// the HTTP send below, so neither wait burns it, and the client can
// anchor its own deadline to the queued→running transition.
let _gpu = crate::ai::gpu::tts_lease().await;
// Cancelled while queued — release the permits without synthesizing.
let cancelled = with_job(job_id, |job| {
if job.status == TtsJobStatus::Queued {
job.status = TtsJobStatus::Running;
false
} else {
true
}
})
.unwrap_or(true);
if cancelled {
return;
}
};
// Wait for the LLM side to release the GPU too (see ai::gpu) — only
// then does the job count as running. The synthesis timeout starts at
// the HTTP send below, so neither wait burns it, and the client can
// anchor its own deadline to the queued→running transition.
let _gpu = crate::ai::gpu::tts_lease().await;
// Cancelled while queued — release the permits without synthesizing.
let cancelled = with_job(job_id, |job| {
if job.status == TtsJobStatus::Queued {
job.status = TtsJobStatus::Running;
false
} else {
true
}
})
.unwrap_or(true);
if cancelled {
return;
}
let Some(client) = state.llamacpp.as_ref() else {
finish_job(
job_id,
TtsJobStatus::Error,
None,
Some("TTS backend not configured".to_string()),
);
return;
};
match client
.text_to_speech(
&text,
voice.as_deref(),
&format,
exaggeration,
cfg_weight,
temperature,
)
.await
{
Ok(bytes) => {
let audio = base64::engine::general_purpose::STANDARD.encode(&bytes);
finish_job(job_id, TtsJobStatus::Done, Some(audio), None);
}
Err(e) => {
log::error!("TTS job {job_id} failed: {:?}", e);
let Some(client) = state.llamacpp.as_ref() else {
finish_job(
job_id,
TtsJobStatus::Error,
None,
Some(format!("TTS failed: {e}")),
Some("TTS backend not configured".to_string()),
);
return;
};
match client
.text_to_speech(
&text,
voice.as_deref(),
&format,
exaggeration,
cfg_weight,
temperature,
)
.await
{
Ok(bytes) => {
let audio = base64::engine::general_purpose::STANDARD.encode(&bytes);
finish_job(job_id, TtsJobStatus::Done, Some(audio), None);
}
Err(e) => {
log::error!("TTS job {job_id} failed: {:?}", e);
finish_job(
job_id,
TtsJobStatus::Error,
None,
Some(format!("TTS failed: {e}")),
);
}
}
}
});
// Carry the handler span into the job (tracing context is thread-local).
.instrument(tracing::Span::current()),
);
// Aborting an already-finished task is a no-op, so this late install is
// safe even if the job raced to completion.
with_job(job_id, |job| {
@@ -660,14 +661,9 @@ pub async fn create_speech_job_handler(
/// GET /tts/speech/jobs/{id} — poll a speech job; returns the audio once done.
/// 404s after the job expires (results are kept ~10 min past completion).
#[get("/tts/speech/jobs/{id}")]
pub async fn speech_job_status_handler(
http_request: HttpRequest,
_claims: Claims,
path: web::Path<String>,
) -> impl Responder {
let parent_context = extract_context_from_request(&http_request);
let mut span =
global_tracer().start_with_context("http.tts.speech_job.status", &parent_context);
#[tracing::instrument(name = "http.tts.speech_job.status", skip_all)]
pub async fn speech_job_status_handler(_claims: Claims, path: web::Path<String>) -> impl Responder {
let span = tracing::Span::current();
let Ok(id) = Uuid::parse_str(&path.into_inner()) else {
span.set_status(Status::error("invalid job id"));
@@ -701,14 +697,9 @@ pub async fn speech_job_status_handler(
/// limitation as the sync path); cancelling stops the wait and discards the
/// result. Cancelling an already-finished job leaves it terminal.
#[delete("/tts/speech/jobs/{id}")]
pub async fn cancel_speech_job_handler(
http_request: HttpRequest,
_claims: Claims,
path: web::Path<String>,
) -> impl Responder {
let parent_context = extract_context_from_request(&http_request);
let mut span =
global_tracer().start_with_context("http.tts.speech_job.cancel", &parent_context);
#[tracing::instrument(name = "http.tts.speech_job.cancel", skip_all)]
pub async fn cancel_speech_job_handler(_claims: Claims, path: web::Path<String>) -> impl Responder {
let span = tracing::Span::current();
let Ok(id) = Uuid::parse_str(&path.into_inner()) else {
span.set_status(Status::error("invalid job id"));
@@ -749,21 +740,20 @@ pub struct ListVoicesQuery {
/// in-memory cache when possible so browsing settings doesn't make llama-swap
/// load the TTS model (and evict the resident LLM); see VOICES_CACHE.
#[get("/tts/voices")]
#[tracing::instrument(name = "http.tts.voices.list", skip_all)]
pub async fn list_voices_handler(
http_request: HttpRequest,
_claims: Claims,
query: web::Query<ListVoicesQuery>,
app_state: web::Data<AppState>,
) -> impl Responder {
let parent_context = extract_context_from_request(&http_request);
let mut span = global_tracer().start_with_context("http.tts.voices.list", &parent_context);
let span = tracing::Span::current();
let force = query
.refresh
.as_deref()
.is_some_and(|v| matches!(v, "1" | "true" | "yes"));
if !force && let Some(v) = cached_voices() {
span.set_attribute(KeyValue::new("tts.voices_cache_hit", true));
span.set_attribute("tts.voices_cache_hit", true);
span.set_status(Status::Ok);
return HttpResponse::Ok().json(v);
}
@@ -776,7 +766,7 @@ pub async fn list_voices_handler(
match client.list_voices().await {
Ok(v) => {
store_voices_cache(&v);
span.set_attribute(KeyValue::new("tts.voices_cache_hit", false));
span.set_attribute("tts.voices_cache_hit", false);
span.set_status(Status::Ok);
HttpResponse::Ok().json(v)
}
@@ -790,14 +780,13 @@ pub async fn list_voices_handler(
/// DELETE /tts/voices/{name} — remove a cloned voice from the library.
#[delete("/tts/voices/{name}")]
#[tracing::instrument(name = "http.tts.voices.delete", skip_all)]
pub async fn delete_voice_handler(
http_request: HttpRequest,
_claims: Claims,
path: web::Path<String>,
app_state: web::Data<AppState>,
) -> impl Responder {
let parent_context = extract_context_from_request(&http_request);
let mut span = global_tracer().start_with_context("http.tts.voices.delete", &parent_context);
let span = tracing::Span::current();
let Some(client) = app_state.llamacpp.as_ref() else {
span.set_status(Status::error("tts backend not configured"));
@@ -814,7 +803,7 @@ pub async fn delete_voice_handler(
return HttpResponse::BadRequest().json(json!({ "error": "invalid voice name" }));
}
};
span.set_attribute(KeyValue::new("tts.voice_name", name.clone()));
span.set_attribute("tts.voice_name", name.clone());
match client.delete_voice(&name).await {
Ok(v) => {
@@ -835,14 +824,13 @@ pub async fn delete_voice_handler(
/// plus optional `start_seconds` / `duration_seconds` (text) selecting which
/// window of a longer recording becomes the reference clip.
#[post("/tts/voices/upload")]
#[tracing::instrument(name = "http.tts.voices.upload", skip_all)]
pub async fn create_voice_upload_handler(
http_request: HttpRequest,
_claims: Claims,
mut payload: Multipart,
app_state: web::Data<AppState>,
) -> impl Responder {
let parent_context = extract_context_from_request(&http_request);
let mut span = global_tracer().start_with_context("http.tts.voices.upload", &parent_context);
let span = tracing::Span::current();
let Some(client) = app_state.llamacpp.as_ref() else {
span.set_status(Status::error("tts backend not configured"));
@@ -932,8 +920,8 @@ pub async fn create_voice_upload_handler(
span.set_status(Status::error("voice_file is required"));
return HttpResponse::BadRequest().json(json!({ "error": "voice_file is required" }));
}
span.set_attribute(KeyValue::new("tts.voice_name", name.clone()));
span.set_attribute(KeyValue::new("tts.upload_bytes", file_bytes.len() as i64));
span.set_attribute("tts.voice_name", name.clone());
span.set_attribute("tts.upload_bytes", file_bytes.len() as i64);
// Normalize to WAV so any device format (e.g. .aac / .opus, which Chatterbox
// rejects by extension) is accepted.
@@ -987,15 +975,13 @@ pub struct CreateVoiceFromLibraryRequest {
/// WAV reference clip (window selected by start/duration_seconds, length
/// capped by LLAMA_SWAP_TTS_REF_SECONDS).
#[post("/tts/voices/from-library")]
#[tracing::instrument(name = "http.tts.voices.from_library", skip_all)]
pub async fn create_voice_from_library_handler(
http_request: HttpRequest,
_claims: Claims,
req: web::Json<CreateVoiceFromLibraryRequest>,
app_state: web::Data<AppState>,
) -> impl Responder {
let parent_context = extract_context_from_request(&http_request);
let mut span =
global_tracer().start_with_context("http.tts.voices.from_library", &parent_context);
let span = tracing::Span::current();
let Some(client) = app_state.llamacpp.as_ref() else {
span.set_status(Status::error("tts backend not configured"));
@@ -1045,7 +1031,7 @@ pub async fn create_voice_from_library_handler(
return HttpResponse::BadRequest()
.json(json!({ "error": "file is not an audio or video file" }));
}
span.set_attribute(KeyValue::new("tts.voice_name", voice_name.clone()));
span.set_attribute("tts.voice_name", voice_name.clone());
let wav = match prepare_reference_audio(&abs, ref_start, ref_duration).await {
Ok(b) => b,