Apps/ImageApi
1
0 Fork 0
Code Issues 7 Pull Requests Releases Wiki Activity

Otel: llm.request client spans + traceparent on LLM calls (#130)

Browse Source 
Add OpenTelemetry tracing to every LLM client call so an insight/chat
turn is a single trace from HTTP entry to each downstream inference.

- LlmClient trait: thread a &opentelemetry::Context through all 5
  methods (generate, chat_with_tools, chat_with_tools_stream,
  generate_embeddings, describe_image), matching the DAO convention of
  explicit context threading rather than ambient Context::current().
- otel::start_llm_request_span: single shared helper that every
  provider (openrouter / llamacpp / ollama) funnels through, so the
  span shape is identical: name llm.request, SpanKind::Client,
  llm.provider / llm.operation / llm.model.
- Per-method spans in all three clients with the spec's operation enum
  (chat | chat_stream | embeddings | describe_image | generate); stream
  spans are owned and moved into the async_stream generator so they end
  when the response drains, not when headers arrive.
- llm.tokens.input / llm.tokens.output recorded on non-stream and
  stream completion; llm.cost_usd from usage.cost (OpenRouter);
  http.status_code + error status on failure.
- W3C traceparent injected on outgoing LLM HTTP requests (llama-swap,
  Ollama, OpenRouter) gated on span_context.is_valid() so debug
  no-op-tracer builds don't send an all-zero parent.
- Turn-level spans (ai.chat.turn.execute, ai.persona_chat.turn.execute)
  now carry llm.backend + llm.model; with_entry streaming paths thread
  the turn context (turn_cx) to child describe_image / agentic-loop
  calls instead of the ambient context.

clippy clean; 690 lib + 775 integration + 4 doc tests green.
This commit is contained in:
Cameron Cordes committed 2026-09-28 01:21:46 -04:00
1 parent e84ac82f86
commit 696d9fa8f3
17 files changed
+910 -234
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/ai/llamacpp.rs
+133 -26
View File
@@ -20,8 +20,11 @@
// insight_generator / insight_chat.
#![allow(dead_code)]
use anyhow::{Context, Result, anyhow, bail};
use anyhow::Context as _;
use anyhow::{Result, anyhow, bail};
use async_trait::async_trait;
use opentelemetry::trace::{Span, Status, TraceContextExt};
use opentelemetry::{Context, KeyValue};
use reqwest::Client;
use serde::Deserialize;
use serde_json::{Value, json};
@@ -486,6 +489,7 @@ impl LlamaCppClient {
async fn chat_completion_with_model(
&self,
model: &str,
mut span: Option<&mut opentelemetry::global::BoxedSpan>,
messages: Vec<ChatMessage>,
tools: Vec<Tool>,
) -> Result<(ChatMessage, Option<i32>, Option<i32>)> {
@@ -517,19 +521,41 @@ impl LlamaCppClient {
// Wait for any TTS synthesis to release the GPU before the request
// timeout starts (see ai::gpu).
let _gpu = crate::ai::gpu::llm_lease().await;
let resp = self
.client
.post(&url)
.json(&Value::Object(body))
.send()
.await
.with_context(|| format!("POST {} failed", url))?;
let mut request = self.client.post(&url).json(&Value::Object(body));
if let Some(s) = span.as_mut() {
for (k, v) in crate::otel::traceparent_headers(
&Context::new().with_remote_span_context(s.span_context().clone()),
) {
request = request.header(k, v);
}
}
let resp = match request.send().await {
Ok(r) => r,
Err(e) => {
if let Some(s) = span.as_mut() {
s.set_status(Status::error("llama-swap request failed"));
}
return Err(e).with_context(|| format!("POST {} failed", url));
}
};
if !resp.status().is_success() {
let status = resp.status();
if let Some(s) = span.as_mut() {
s.set_attribute(KeyValue::new("http.status_code", status.as_u16() as i64));
s.set_status(Status::error(format!(
"llama-swap chat request failed: {status}"
)));
}
let body = resp.text().await.unwrap_or_default();
bail!("llama-swap chat request failed: {} — {}", status, body);
}
if let Some(s) = span.as_mut() {
s.set_attribute(KeyValue::new(
"http.status_code",
resp.status().as_u16() as i64,
));
}
let parsed: Value = resp.json().await.context("parsing chat response")?;
let choice = parsed
@@ -562,14 +588,35 @@ impl LlamaCppClient {
log_timings(&parsed, prompt_tokens, completion_tokens);
if let Some(s) = span.as_mut() {
if let Some(pt) = prompt_tokens {
s.set_attribute(KeyValue::new("llm.tokens.input", pt as i64));
}
if let Some(ct) = completion_tokens {
s.set_attribute(KeyValue::new("llm.tokens.output", ct as i64));
}
s.set_status(Status::Ok);
}
Ok((chat_msg, prompt_tokens, completion_tokens))
}
/// Start the `llm.request` span for an outgoing llama-swap call.
fn start_request_span(
&self,
cx: &Context,
operation: &str,
model: &str,
) -> opentelemetry::global::BoxedSpan {
crate::otel::start_llm_request_span(cx, "llamacpp", operation, model)
}
}
#[async_trait]
impl LlmClient for LlamaCppClient {
async fn generate(
&self,
cx: &Context,
prompt: &str,
system: Option<&str>,
images: Option<Vec<String>>,
@@ -582,12 +629,15 @@ impl LlmClient for LlamaCppClient {
user.images = images;
messages.push(user);
let (reply, _, _) = self.chat_with_tools(messages, Vec::new()).await?;
// Delegates to chat_with_tools, which starts the llm.request span
// (operation=chat) — one span per HTTP request.
let (reply, _, _) = self.chat_with_tools(cx, messages, Vec::new()).await?;
Ok(reply.content)
}
async fn chat_with_tools(
&self,
cx: &Context,
messages: Vec<ChatMessage>,
tools: Vec<Tool>,
) -> Result<(ChatMessage, Option<i32>, Option<i32>)> {
@@ -597,15 +647,25 @@ impl LlmClient for LlamaCppClient {
messages.len(),
tools.len()
);
self.chat_completion_with_model(&self.primary_model.clone(), messages, tools)
.await
let mut span = self.start_request_span(cx, "chat", &self.primary_model);
self.chat_completion_with_model(
&self.primary_model.clone(),
Some(&mut span),
messages,
tools,
)
.await
}
async fn chat_with_tools_stream(
&self,
cx: &Context,
messages: Vec<ChatMessage>,
tools: Vec<Tool>,
) -> Result<BoxStream<'static, Result<LlmStreamEvent>>> {
// The span outlives this method — it moves into the stream
// generator and is finalized when the response drains.
let mut span = self.start_request_span(cx, "chat_stream", &self.primary_model);
let url = format!("{}/chat/completions", self.base_url);
let roles: Vec<&str> = messages.iter().map(|m| m.role.as_str()).collect();
log::debug!(
@@ -639,19 +699,30 @@ impl LlmClient for LlamaCppClient {
// timeout starts (see ai::gpu). The guard is moved into the stream
// below so the lease spans the whole generation, not just the send.
let gpu = crate::ai::gpu::llm_lease().await;
let resp = self
.client
.post(&url)
.json(&Value::Object(body))
let mut request = self.client.post(&url).json(&Value::Object(body));
for (k, v) in crate::otel::traceparent_headers(
&Context::new().with_remote_span_context(span.span_context().clone()),
) {
request = request.header(k, v);
}
let resp = request
.send()
.await
.with_context(|| format!("POST {} failed", url))?;
if !resp.status().is_success() {
let status = resp.status();
span.set_attribute(KeyValue::new("http.status_code", status.as_u16() as i64));
span.set_status(Status::error(format!(
"llama-swap stream request failed: {status}"
)));
let body = resp.text().await.unwrap_or_default();
bail!("llama-swap stream request failed: {} — {}", status, body);
}
span.set_attribute(KeyValue::new(
"http.status_code",
resp.status().as_u16() as i64,
));
let byte_stream = resp.bytes_stream();
let stream = async_stream::stream! {
@@ -760,6 +831,13 @@ impl LlmClient for LlamaCppClient {
tool_calls,
images: None,
};
if let Some(pt) = prompt_tokens {
span.set_attribute(KeyValue::new("llm.tokens.input", pt as i64));
}
if let Some(ct) = completion_tokens {
span.set_attribute(KeyValue::new("llm.tokens.output", ct as i64));
}
span.set_status(Status::Ok);
yield Ok(LlmStreamEvent::Done {
message,
prompt_eval_count: prompt_tokens,
@@ -770,7 +848,8 @@ impl LlmClient for LlamaCppClient {
Ok(Box::pin(stream))
}
async fn generate_embeddings(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
async fn generate_embeddings(&self, cx: &Context, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
let mut span = self.start_request_span(cx, "embeddings", &self.embedding_model);
// Deliberately NO GPU lease: the embed slot sits in llama-swap's
// always-resident group and never participates in a model swap, so
// leasing here would only stall searches behind a queued synthesis.
@@ -780,19 +859,33 @@ impl LlmClient for LlamaCppClient {
"input": texts,
});
let resp = self
.client
.post(&url)
.json(&body)
.send()
.await
.with_context(|| format!("POST {} failed", url))?;
let mut request = self.client.post(&url).json(&body);
for (k, v) in crate::otel::traceparent_headers(
&Context::new().with_remote_span_context(span.span_context().clone()),
) {
request = request.header(k, v);
}
let resp = match request.send().await {
Ok(r) => r,
Err(e) => {
span.set_status(Status::error("llama-swap embedding request failed"));
return Err(e).with_context(|| format!("POST {} failed", url));
}
};
if !resp.status().is_success() {
let status = resp.status();
span.set_attribute(KeyValue::new("http.status_code", status.as_u16() as i64));
span.set_status(Status::error(format!(
"llama-swap embedding request failed: {status}"
)));
let body = resp.text().await.unwrap_or_default();
bail!("llama-swap embedding request failed: {} — {}", status, body);
}
span.set_attribute(KeyValue::new(
"http.status_code",
resp.status().as_u16() as i64,
));
#[derive(Deserialize)]
struct EmbedResponse {
@@ -803,11 +896,20 @@ impl LlmClient for LlamaCppClient {
embedding: Vec<f32>,
}
let parsed: EmbedResponse = resp.json().await.context("parsing embed response")?;
let parsed: EmbedResponse = match resp.json().await {
Ok(p) => p,
Err(e) => {
span.set_status(Status::error("parsing embed response failed"));
return Err(e).context("parsing embed response");
}
};
span.set_status(Status::Ok);
Ok(parsed.data.into_iter().map(|i| i.embedding).collect())
}
async fn describe_image(&self, image_base64: &str) -> Result<String> {
async fn describe_image(&self, cx: &Context, image_base64: &str) -> Result<String> {
// Own span with operation=describe_image — routes through the vision slot.
let mut span = self.start_request_span(cx, "describe_image", &self.vision_model);
let prompt = "Briefly describe what you see in this image in 1-2 sentences. \
Focus on the people, location, and activity.";
let system = "You are a scene description assistant. Be concise and factual.";
@@ -823,7 +925,12 @@ impl LlmClient for LlamaCppClient {
];
let (reply, _, _) = self
.chat_completion_with_model(&self.vision_model.clone(), messages, Vec::new())
.chat_completion_with_model(
&self.vision_model.clone(),
Some(&mut span),
messages,
Vec::new(),
)
.await?;
Ok(reply.content)
}