Apps/ImageApi
1
0 Fork 0
Code Issues 7 Pull Requests Releases Wiki Activity

Tracing: LLM clients + streaming spans on tracing, drop cx threading (#140)

Browse Source 
LlmClient methods no longer take an OpenTelemetry Context. llm.request is
now a tracing span (otel.kind=client, llm.provider/operation/model) that
parents on the caller's ambient span; traceparent injection reads the
span's OTel context via OpenTelemetrySpanExt.

The src/ai feature spans (handlers, insight generation, chat, persona
chat, daily summaries, TTS) move to #[instrument] / info_span!, and every
LLM-bearing tokio::spawn carries its parent span with .instrument(): the
streaming chat turn (chat_turn_stream_inner), async chat / persona turns,
background insight jobs and TTS jobs. Sub-spans in the agentic loops
(ai.chat.loop, ai.agentic.loop) instrument each LLM / tool await.

DAO calls keep their explicit Context (derived from the tracing span)
until the DAO migration (#141).

tests/otel_llm_spans.rs pins nesting + traceparent for the blocking and
streaming paths across a spawn on a multi-thread runtime.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Cameron CordesandClaude Opus 5.5 committed 2026-09-29 17:02:40 -04:00
1 parent 7a59afc62b
commit 1c508cdabd
20 files changed
+973 -1030
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/ai/openrouter.rs
+33 -41
View File
@@ -5,14 +5,14 @@
use anyhow::Context as _;
use anyhow::{Result, anyhow, bail};
use async_trait::async_trait;
use opentelemetry::trace::{Span, Status};
use opentelemetry::{Context, KeyValue};
use opentelemetry::trace::Status;
use reqwest::Client;
use serde::Deserialize;
use serde_json::{Value, json};
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use tracing_opentelemetry::OpenTelemetrySpanExt as _;
use crate::ai::llm_client::{
ChatMessage, LlmClient, LlmStreamEvent, ModelCapabilities, Tool, ToolCall, ToolCallFunction,
@@ -152,13 +152,8 @@ impl OpenRouterClient {
/// `chat`, `embeddings`, or `describe` — the call shape, not the entry
/// point (a `describe_image` funneled through `chat_with_tools` is
/// still a `describe`).
fn start_request_span(
&self,
cx: &Context,
operation: &str,
model: &str,
) -> opentelemetry::global::BoxedSpan {
crate::otel::start_llm_request_span(cx, "openrouter", operation, model)
fn start_request_span(&self, operation: &str, model: &str) -> tracing::Span {
crate::otel::start_llm_request_span("openrouter", operation, model)
}
/// Translate canonical messages to the OpenAI-compatible wire shape.
@@ -329,7 +324,6 @@ impl OpenRouterClient {
impl LlmClient for OpenRouterClient {
async fn generate(
&self,
cx: &Context,
prompt: &str,
system: Option<&str>,
images: Option<Vec<String>>,
@@ -344,31 +338,29 @@ impl LlmClient for OpenRouterClient {
// Delegates to chat_with_tools, which starts the llm.request span
// (operation=chat) — one span per HTTP request.
let (reply, _, _) = self.chat_with_tools(cx, messages, Vec::new()).await?;
let (reply, _, _) = self.chat_with_tools(messages, Vec::new()).await?;
Ok(reply.content)
}
async fn chat_with_tools(
&self,
cx: &Context,
messages: Vec<ChatMessage>,
tools: Vec<Tool>,
) -> Result<(ChatMessage, Option<i32>, Option<i32>)> {
let mut span = self.start_request_span(cx, "chat", &self.primary_model);
self.chat_with_tools_impl(Some(&mut span), messages, tools)
let span = self.start_request_span("chat", &self.primary_model);
self.chat_with_tools_impl(Some(&span), messages, tools)
.await
}
async fn chat_with_tools_stream(
&self,
cx: &Context,
messages: Vec<ChatMessage>,
tools: Vec<Tool>,
) -> Result<BoxStream<'static, Result<LlmStreamEvent>>> {
// The span outlives this method — the stream yields until the
// response fully drains, so the owned span moves into the
// generator and is finalized there.
let mut span = self.start_request_span(cx, "chat_stream", &self.primary_model);
let span = self.start_request_span("chat_stream", &self.primary_model);
let url = format!("{}/chat/completions", self.base_url);
let mut body = serde_json::Map::new();
@@ -397,7 +389,7 @@ impl LlmClient for OpenRouterClient {
let mut request = self
.authed(self.client.post(&url))
.json(&Value::Object(body));
request = crate::otel::with_traceparent(request, span.span_context());
request = crate::otel::with_traceparent(request, &span);
let resp = request
.send()
.await
@@ -406,14 +398,14 @@ impl LlmClient for OpenRouterClient {
if !resp.status().is_success() {
let status = resp.status();
crate::otel::record_http_failure(
Some(&mut span),
Some(&span),
status.as_u16(),
&format!("OpenRouter stream request failed: {status}"),
);
let body = resp.text().await.unwrap_or_default();
bail!("OpenRouter stream request failed: {} — {}", status, body);
}
crate::otel::set_http_status(Some(&mut span), resp.status().as_u16());
crate::otel::set_http_status(Some(&span), resp.status().as_u16());
// OpenAI-compat SSE stream. Each event is `data: <json>\n\n`, with
// `data: [DONE]` signalling completion. Tool calls arrive as
@@ -489,7 +481,7 @@ impl LlmClient for OpenRouterClient {
.and_then(|n| n.as_i64())
.map(|n| n as i32);
if let Some(cost) = usage.get("cost").and_then(|c| c.as_f64()) {
span.set_attribute(KeyValue::new("llm.cost_usd", cost));
span.set_attribute("llm.cost_usd", cost);
}
}
@@ -581,10 +573,10 @@ impl LlmClient for OpenRouterClient {
images: None,
};
if let Some(pt) = prompt_tokens {
span.set_attribute(KeyValue::new("llm.tokens.input", pt as i64));
span.set_attribute("llm.tokens.input", pt as i64);
}
if let Some(ct) = completion_tokens {
span.set_attribute(KeyValue::new("llm.tokens.output", ct as i64));
span.set_attribute("llm.tokens.output", ct as i64);
}
span.set_status(Status::Ok);
yield Ok(LlmStreamEvent::Done {
@@ -597,8 +589,8 @@ impl LlmClient for OpenRouterClient {
Ok(Box::pin(stream))
}
async fn generate_embeddings(&self, cx: &Context, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
let mut span = self.start_request_span(cx, "embeddings", &self.embedding_model);
async fn generate_embeddings(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
let span = self.start_request_span("embeddings", &self.embedding_model);
let url = format!("{}/embeddings", self.base_url);
let body = json!({
"model": self.embedding_model,
@@ -606,7 +598,7 @@ impl LlmClient for OpenRouterClient {
});
let mut request = self.authed(self.client.post(&url)).json(&body);
request = crate::otel::with_traceparent(request, span.span_context());
request = crate::otel::with_traceparent(request, &span);
let resp = request
.send()
.await
@@ -618,14 +610,14 @@ impl LlmClient for OpenRouterClient {
if !resp.status().is_success() {
let status = resp.status();
crate::otel::record_http_failure(
Some(&mut span),
Some(&span),
status.as_u16(),
&format!("OpenRouter embedding request failed: {status}"),
);
let body = resp.text().await.unwrap_or_default();
bail!("OpenRouter embedding request failed: {} — {}", status, body);
}
crate::otel::set_http_status(Some(&mut span), resp.status().as_u16());
crate::otel::set_http_status(Some(&span), resp.status().as_u16());
#[derive(Deserialize)]
struct EmbedResponse {
@@ -641,10 +633,10 @@ impl LlmClient for OpenRouterClient {
Ok(parsed.data.into_iter().map(|i| i.embedding).collect())
}
async fn describe_image(&self, cx: &Context, image_base64: &str) -> Result<String> {
async fn describe_image(&self, image_base64: &str) -> Result<String> {
// Own span with operation=describe_image — the funnel into
// chat_with_tools_impl keeps the request on a single span.
let mut span = self.start_request_span(cx, "describe_image", &self.primary_model);
let span = self.start_request_span("describe_image", &self.primary_model);
let prompt = "Briefly describe what you see in this image in 1-2 sentences. \
Focus on the people, location, and activity.";
let mut messages: Vec<ChatMessage> = Vec::new();
@@ -656,7 +648,7 @@ impl LlmClient for OpenRouterClient {
messages.push(user);
let (reply, _, _) = self
.chat_with_tools_impl(Some(&mut span), messages, Vec::new())
.chat_with_tools_impl(Some(&span), messages, Vec::new())
.await?;
Ok(reply.content)
}
@@ -719,7 +711,7 @@ impl OpenRouterClient {
/// operation label.
async fn chat_with_tools_impl(
&self,
mut span: Option<&mut opentelemetry::global::BoxedSpan>,
span: Option<&tracing::Span>,
messages: Vec<ChatMessage>,
tools: Vec<Tool>,
) -> Result<(ChatMessage, Option<i32>, Option<i32>)> {
@@ -751,13 +743,13 @@ impl OpenRouterClient {
let mut request = self
.authed(self.client.post(&url))
.json(&Value::Object(body));
if let Some(s) = span.as_mut() {
request = crate::otel::with_traceparent(request, s.span_context());
if let Some(s) = span {
request = crate::otel::with_traceparent(request, s);
}
let resp = match request.send().await {
Ok(r) => r,
Err(e) => {
if let Some(s) = span.as_mut() {
if let Some(s) = span {
s.set_status(Status::error("OpenRouter request failed"));
}
return Err(e).with_context(|| format!("POST {} failed", url));
@@ -767,19 +759,19 @@ impl OpenRouterClient {
if !resp.status().is_success() {
let status = resp.status();
crate::otel::record_http_failure(
span.as_deref_mut(),
span,
status.as_u16(),
&format!("OpenRouter chat request failed: {status}"),
);
let body = resp.text().await.unwrap_or_default();
bail!("OpenRouter chat request failed: {} — {}", status, body);
}
crate::otel::set_http_status(span.as_deref_mut(), resp.status().as_u16());
crate::otel::set_http_status(span, resp.status().as_u16());
let parsed: Value = match resp.json().await {
Ok(v) => v,
Err(e) => {
if let Some(s) = span.as_mut() {
if let Some(s) = span {
s.set_status(Status::error("parsing chat response failed"));
}
return Err(e).context("parsing chat response");
@@ -813,15 +805,15 @@ impl OpenRouterClient {
.and_then(|v| v.as_i64())
.map(|n| n as i32);
// OpenRouter reports per-request USD cost in `usage.cost`.
if let Some(s) = span.as_mut() {
if let Some(s) = span {
if let Some(pt) = prompt_tokens {
s.set_attribute(KeyValue::new("llm.tokens.input", pt as i64));
s.set_attribute("llm.tokens.input", pt as i64);
}
if let Some(ct) = completion_tokens {
s.set_attribute(KeyValue::new("llm.tokens.output", ct as i64));
s.set_attribute("llm.tokens.output", ct as i64);
}
if let Some(cost) = usage.and_then(|u| u.get("cost")).and_then(|c| c.as_f64()) {
s.set_attribute(KeyValue::new("llm.cost_usd", cost));
s.set_attribute("llm.cost_usd", cost);
}
s.set_status(Status::Ok);
}