Apps/ImageApi
1
0 Fork 0
Code Issues 7 Pull Requests Releases Wiki Activity

Otel: llm.request client spans + traceparent on LLM calls (#130)

Browse Source 
Add OpenTelemetry tracing to every LLM client call so an insight/chat
turn is a single trace from HTTP entry to each downstream inference.

- LlmClient trait: thread a &opentelemetry::Context through all 5
  methods (generate, chat_with_tools, chat_with_tools_stream,
  generate_embeddings, describe_image), matching the DAO convention of
  explicit context threading rather than ambient Context::current().
- otel::start_llm_request_span: single shared helper that every
  provider (openrouter / llamacpp / ollama) funnels through, so the
  span shape is identical: name llm.request, SpanKind::Client,
  llm.provider / llm.operation / llm.model.
- Per-method spans in all three clients with the spec's operation enum
  (chat | chat_stream | embeddings | describe_image | generate); stream
  spans are owned and moved into the async_stream generator so they end
  when the response drains, not when headers arrive.
- llm.tokens.input / llm.tokens.output recorded on non-stream and
  stream completion; llm.cost_usd from usage.cost (OpenRouter);
  http.status_code + error status on failure.
- W3C traceparent injected on outgoing LLM HTTP requests (llama-swap,
  Ollama, OpenRouter) gated on span_context.is_valid() so debug
  no-op-tracer builds don't send an all-zero parent.
- Turn-level spans (ai.chat.turn.execute, ai.persona_chat.turn.execute)
  now carry llm.backend + llm.model; with_entry streaming paths thread
  the turn context (turn_cx) to child describe_image / agentic-loop
  calls instead of the ambient context.

clippy clean; 690 lib + 775 integration + 4 doc tests green.
This commit is contained in:
Cameron Cordes committed 2026-09-28 01:21:46 -04:00
1 parent e84ac82f86
commit 696d9fa8f3
17 files changed
+910 -234
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/ai/ollama.rs
+273 -55
View File
@@ -1,6 +1,9 @@
use anyhow::{Context, Result};
use anyhow::Context as _;
use anyhow::Result;
use async_trait::async_trait;
use chrono::NaiveDate;
use opentelemetry::trace::{Span, Status, TraceContextExt};
use opentelemetry::{Context, KeyValue};
use reqwest::Client;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
@@ -9,6 +12,7 @@ use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use crate::ai::llm_client::{LlmClient, LlmStreamEvent};
use crate::otel::traceparent_headers;
use futures::stream::{BoxStream, StreamExt};
// Re-export shared types so existing `crate::ai::ollama::{...}` imports
@@ -365,6 +369,7 @@ impl OllamaClient {
async fn try_generate(
&self,
mut span: Option<&mut opentelemetry::global::BoxedSpan>,
url: &str,
model: &str,
prompt: &str,
@@ -382,15 +387,29 @@ impl OllamaClient {
think,
};
let response = self
let mut request_builder = self
.client
.post(format!("{}/api/generate", url))
.json(&request)
.send()
.await?;
.json(&request);
if let Some(s) = span.as_mut() {
request_builder = self.inject_traceparent(request_builder, s);
}
let response = match request_builder.send().await {
Ok(r) => r,
Err(e) => {
if let Some(s) = span.as_mut() {
s.set_status(Status::error("Ollama request failed"));
}
return Err(e.into());
}
};
if !response.status().is_success() {
let status = response.status();
if let Some(s) = span.as_mut() {
s.set_attribute(KeyValue::new("http.status_code", status.as_u16() as i64));
s.set_status(Status::error(format!("Ollama request failed: {status}")));
}
let error_body = response.text().await.unwrap_or_default();
return Err(anyhow::anyhow!(
"Ollama request failed: {} - {}",
@@ -398,8 +417,31 @@ impl OllamaClient {
error_body
));
}
if let Some(s) = span.as_mut() {
s.set_attribute(KeyValue::new(
"http.status_code",
response.status().as_u16() as i64,
));
}
let result: OllamaResponse = response.json().await?;
let result: OllamaResponse = match response.json().await {
Ok(r) => r,
Err(e) => {
if let Some(s) = span.as_mut() {
s.set_status(Status::error("parsing Ollama response failed"));
}
return Err(e.into());
}
};
if let Some(s) = span.as_mut() {
if let Some(pt) = result.prompt_eval_count {
s.set_attribute(KeyValue::new("llm.tokens.input", pt as i64));
}
if let Some(ct) = result.eval_count {
s.set_attribute(KeyValue::new("llm.tokens.output", ct as i64));
}
s.set_status(Status::Ok);
}
log_chat_metrics(
result.prompt_eval_count,
result.prompt_eval_duration,
@@ -409,32 +451,71 @@ impl OllamaClient {
Ok(result.response)
}
pub async fn generate(&self, prompt: &str, system: Option<&str>) -> Result<String> {
self.generate_with_images(prompt, system, None).await
/// Attach the W3C traceparent header to an outgoing Ollama request so
/// a server that understands it can parent its own spans onto ours.
fn inject_traceparent(
&self,
builder: reqwest::RequestBuilder,
span: &opentelemetry::global::BoxedSpan,
) -> reqwest::RequestBuilder {
let mut builder = builder;
for (k, v) in traceparent_headers(
&Context::new().with_remote_span_context(span.span_context().clone()),
) {
builder = builder.header(k, v);
}
builder
}
/// Start the `llm.request` span for an outgoing Ollama call.
fn start_request_span(
&self,
cx: &Context,
operation: &str,
model: &str,
) -> opentelemetry::global::BoxedSpan {
crate::otel::start_llm_request_span(cx, "ollama", operation, model)
}
pub async fn generate(
&self,
cx: &Context,
prompt: &str,
system: Option<&str>,
) -> Result<String> {
self.generate_with_images(cx, prompt, system, None).await
}
#[allow(dead_code)]
pub async fn generate_no_think(&self, prompt: &str, system: Option<&str>) -> Result<String> {
self.generate_with_options(prompt, system, None, Some(false))
pub async fn generate_no_think(
&self,
cx: &Context,
prompt: &str,
system: Option<&str>,
) -> Result<String> {
self.generate_with_options(cx, prompt, system, None, Some(false), "generate")
.await
}
pub async fn generate_with_images(
&self,
cx: &Context,
prompt: &str,
system: Option<&str>,
images: Option<Vec<String>>,
) -> Result<String> {
self.generate_with_options(prompt, system, images, None)
self.generate_with_options(cx, prompt, system, images, None, "generate")
.await
}
async fn generate_with_options(
&self,
cx: &Context,
prompt: &str,
system: Option<&str>,
images: Option<Vec<String>>,
think: Option<bool>,
operation: &str,
) -> Result<String> {
log::debug!("=== Ollama Request ===");
log::debug!("Primary model: {}", self.primary_model);
@@ -447,6 +528,10 @@ impl OllamaClient {
}
log::debug!("=====================");
// Span for the whole generate attempt; the model attribute
// reflects whichever server actually answered.
let mut span = self.start_request_span(cx, operation, &self.primary_model);
// Try primary server first with primary model
log::info!(
"Attempting to generate with primary server: {} (model: {})",
@@ -455,6 +540,7 @@ impl OllamaClient {
);
let primary_result = self
.try_generate(
Some(&mut span),
&self.primary_url,
&self.primary_model,
prompt,
@@ -483,8 +569,10 @@ impl OllamaClient {
fallback_url,
fallback_model
);
span.set_attribute(KeyValue::new("llm.model", fallback_model.to_string()));
match self
.try_generate(
Some(&mut span),
fallback_url,
fallback_model,
prompt,
@@ -531,6 +619,7 @@ impl OllamaClient {
/// Generate a title for a single photo based on its generated summary
pub async fn generate_photo_title(
&self,
cx: &Context,
summary: &str,
custom_system: Option<&str>,
) -> Result<String> {
@@ -546,7 +635,7 @@ Capture the key moment or theme. Return ONLY the title, nothing else."#,
let system = custom_system.unwrap_or("You are my long term memory assistant. Use only the information provided. Do not invent details.");
let title = self
.generate_with_images(&prompt, Some(system), None)
.generate_with_images(cx, &prompt, Some(system), None)
.await?;
// Models decorate despite "Return ONLY the title": quotes, bold
// markers, sometimes a "Title:" label.
@@ -563,6 +652,7 @@ Capture the key moment or theme. Return ONLY the title, nothing else."#,
/// Generate a summary for a single photo based on its context
pub async fn generate_photo_summary(
&self,
cx: &Context,
date: NaiveDate,
location: Option<&str>,
contact: Option<&str>,
@@ -645,21 +735,32 @@ Analyze the image and use specific details from both the visual content and the
let system = custom_system.unwrap_or("You are a memory refreshing assistant who is able to provide insights through analyzing past conversations. Use only the information provided. Do not invent details.");
let images = image_base64.map(|img| vec![img]);
self.generate_with_images(&prompt, Some(system), images)
self.generate_with_images(cx, &prompt, Some(system), images)
.await
}
/// Generate a brief visual description of a photo for use in RAG query enrichment.
/// Returns 1-2 sentences describing people, location, and activity visible in the image.
/// Only called when the model has vision capabilities.
pub async fn generate_photo_description(&self, image_base64: &str) -> Result<String> {
pub async fn generate_photo_description(
&self,
cx: &Context,
image_base64: &str,
) -> Result<String> {
let prompt = "Briefly describe what you see in this image in 1-2 sentences. \
Focus on the people, location, and activity.";
let system = "You are a scene description assistant. Be concise and factual.";
let images = vec![image_base64.to_string()];
let description = self
.generate_with_images(prompt, Some(system), Some(images))
.generate_with_options(
cx,
prompt,
Some(system),
Some(images),
None,
"describe_image",
)
.await?;
Ok(description.trim().to_string())
@@ -672,9 +773,11 @@ Analyze the image and use specific details from both the visual content and the
/// iteration of the agent loop.
pub async fn chat_with_tools(
&self,
cx: &Context,
messages: Vec<ChatMessage>,
tools: Vec<Tool>,
) -> Result<(ChatMessage, Option<i32>, Option<i32>)> {
let mut span = self.start_request_span(cx, "chat", &self.primary_model);
let order = self.attempt_order();
let mut errors: Vec<String> = Vec::new();
@@ -685,8 +788,11 @@ Analyze the image and use specific details from both the visual content and the
url,
model
);
if *label == "fallback" {
span.set_attribute(KeyValue::new("llm.model", model.to_string()));
}
match self
.try_chat_with_tools(url, messages.clone(), tools.clone())
.try_chat_with_tools(Some(&mut span), url, messages.clone(), tools.clone())
.await
{
Ok(result) => {
@@ -728,18 +834,22 @@ Analyze the image and use specific details from both the visual content and the
/// any, live on the final message).
pub async fn chat_with_tools_stream(
&self,
cx: &Context,
messages: Vec<ChatMessage>,
tools: Vec<Tool>,
) -> Result<BoxStream<'static, Result<LlmStreamEvent>>> {
// Same preference logic as `chat_with_tools`. Only the initial
// connection is retried across servers — once the stream begins,
// mid-stream errors propagate to the caller.
// mid-stream errors propagate to the caller. Each attempt gets its
// own `llm.request` span, created inside
// `try_chat_with_tools_stream` so the winning attempt's span moves
// into the stream and is finalized when the response drains.
let order = self.attempt_order();
let mut last_err: Option<anyhow::Error> = None;
for (label, url, _model) in &order {
match self
.try_chat_with_tools_stream(url, messages.clone(), tools.clone())
.try_chat_with_tools_stream(cx, url, messages.clone(), tools.clone())
.await
{
Ok(s) => {
@@ -759,6 +869,7 @@ Analyze the image and use specific details from both the visual content and the
async fn try_chat_with_tools_stream(
&self,
cx: &Context,
base_url: &str,
messages: Vec<ChatMessage>,
tools: Vec<Tool>,
@@ -773,6 +884,11 @@ Analyze the image and use specific details from both the visual content and the
};
let options = self.build_options();
// The span outlives this method — the stream yields until the
// response fully drains, so the owned span moves into the
// generator and is finalized there.
let mut span = self.start_request_span(cx, "chat_stream", model);
let request_body = OllamaChatRequest {
model,
messages: &messages,
@@ -781,16 +897,25 @@ Analyze the image and use specific details from both the visual content and the
options,
};
let response = self
.client
.post(&url)
.json(&request_body)
.send()
.await
.with_context(|| format!("Failed to connect to Ollama at {}", url))?;
let response = {
let mut rb = self.client.post(&url).json(&request_body);
rb = self.inject_traceparent(rb, &span);
match rb.send().await {
Ok(r) => r,
Err(e) => {
span.set_status(Status::error("Ollama stream connect failed"));
return Err(e)
.with_context(|| format!("Failed to connect to Ollama at {}", url));
}
}
};
if !response.status().is_success() {
let status = response.status();
span.set_attribute(KeyValue::new("http.status_code", status.as_u16() as i64));
span.set_status(Status::error(format!(
"Ollama stream request failed: {status}"
)));
let body = response.text().await.unwrap_or_default();
anyhow::bail!(
"Ollama stream request failed with status {}: {}",
@@ -798,6 +923,10 @@ Analyze the image and use specific details from both the visual content and the
body
);
}
span.set_attribute(KeyValue::new(
"http.status_code",
response.status().as_u16() as i64,
));
// Ollama streams NDJSON: each line is a full `OllamaStreamChunk`.
// We buffer partial lines across chunks from the byte stream.
@@ -879,6 +1008,13 @@ Analyze the image and use specific details from both the visual content and the
eval_count,
eval_duration,
);
if let Some(pt) = prompt_eval_count {
span.set_attribute(KeyValue::new("llm.tokens.input", pt as i64));
}
if let Some(ct) = eval_count {
span.set_attribute(KeyValue::new("llm.tokens.output", ct as i64));
}
span.set_status(Status::Ok);
let message = ChatMessage {
role,
content: accumulated,
@@ -897,6 +1033,7 @@ Analyze the image and use specific details from both the visual content and the
async fn try_chat_with_tools(
&self,
mut span: Option<&mut opentelemetry::global::BoxedSpan>,
base_url: &str,
messages: Vec<ChatMessage>,
tools: Vec<Tool>,
@@ -924,16 +1061,31 @@ Analyze the image and use specific details from both the visual content and the
.unwrap_or_else(|e| format!("<serialization error: {}>", e));
log::debug!("chat_with_tools request body: {}", request_json);
let response = self
.client
.post(&url)
.json(&request_body)
.send()
.await
.with_context(|| format!("Failed to connect to Ollama at {}", url))?;
let response = {
let mut rb = self.client.post(&url).json(&request_body);
if let Some(s) = span.as_mut() {
rb = self.inject_traceparent(rb, s);
}
match rb.send().await {
Ok(r) => r,
Err(e) => {
if let Some(s) = span.as_mut() {
s.set_status(Status::error("Ollama chat connect failed"));
}
return Err(e)
.with_context(|| format!("Failed to connect to Ollama at {}", url));
}
}
};
if !response.status().is_success() {
let status = response.status();
if let Some(s) = span.as_mut() {
s.set_attribute(KeyValue::new("http.status_code", status.as_u16() as i64));
s.set_status(Status::error(format!(
"Ollama chat request failed: {status}"
)));
}
let body = response.text().await.unwrap_or_default();
// warn, not error — the outer `chat_with_tools` may recover via
// the fallback server. When both fail, the outer layer emits the
@@ -950,11 +1102,31 @@ Analyze the image and use specific details from both the visual content and the
body
);
}
if let Some(s) = span.as_mut() {
s.set_attribute(KeyValue::new(
"http.status_code",
response.status().as_u16() as i64,
));
}
let chat_response: OllamaChatResponse = response
.json()
.await
.with_context(|| "Failed to parse Ollama chat response")?;
let chat_response: OllamaChatResponse = match response.json().await {
Ok(r) => r,
Err(e) => {
if let Some(s) = span.as_mut() {
s.set_status(Status::error("parsing Ollama chat response failed"));
}
return Err(e).with_context(|| "Failed to parse Ollama chat response");
}
};
if let Some(s) = span.as_mut() {
if let Some(pt) = chat_response.prompt_eval_count {
s.set_attribute(KeyValue::new("llm.tokens.input", pt as i64));
}
if let Some(ct) = chat_response.eval_count {
s.set_attribute(KeyValue::new("llm.tokens.output", ct as i64));
}
s.set_status(Status::Ok);
}
// Log performance counters returned by Ollama. Durations are
// reported in nanoseconds; we render ms + tokens/sec for skim-ability
@@ -976,8 +1148,8 @@ Analyze the image and use specific details from both the visual content and the
/// Generate an embedding vector for text using nomic-embed-text:v1.5
/// Returns a 768-dimensional vector as Vec<f32>
pub async fn generate_embedding(&self, text: &str) -> Result<Vec<f32>> {
let embeddings = self.generate_embeddings(&[text]).await?;
pub async fn generate_embedding(&self, cx: &Context, text: &str) -> Result<Vec<f32>> {
let embeddings = self.generate_embeddings(cx, &[text]).await?;
embeddings
.into_iter()
.next()
@@ -987,9 +1159,11 @@ Analyze the image and use specific details from both the visual content and the
/// Generate embeddings for multiple texts in a single API call (batch mode)
/// Returns a vector of 768-dimensional vectors
/// This is much more efficient than calling generate_embedding multiple times
pub async fn generate_embeddings(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
pub async fn generate_embeddings(&self, cx: &Context, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
let embedding_model = EMBEDDING_MODEL;
let mut span = self.start_request_span(cx, "embeddings", embedding_model);
log::debug!("=== Ollama Batch Embedding Request ===");
log::debug!("Model: {}", embedding_model);
log::debug!("Batch size: {} texts", texts.len());
@@ -1003,7 +1177,7 @@ Analyze the image and use specific details from both the visual content and the
embedding_model
);
let primary_result = self
.try_generate_embeddings(&self.primary_url, embedding_model, texts)
.try_generate_embeddings(Some(&mut span), &self.primary_url, embedding_model, texts)
.await;
let embeddings = match primary_result {
@@ -1026,7 +1200,12 @@ Analyze the image and use specific details from both the visual content and the
embedding_model
);
match self
.try_generate_embeddings(fallback_url, embedding_model, texts)
.try_generate_embeddings(
Some(&mut span),
fallback_url,
embedding_model,
texts,
)
.await
{
Ok(embeddings) => {
@@ -1073,6 +1252,7 @@ Analyze the image and use specific details from both the visual content and the
/// Internal helper to try generating embeddings for multiple texts from a specific server
async fn try_generate_embeddings(
&self,
mut span: Option<&mut opentelemetry::global::BoxedSpan>,
url: &str,
model: &str,
texts: &[&str],
@@ -1082,15 +1262,33 @@ Analyze the image and use specific details from both the visual content and the
input: texts.iter().map(|s| s.to_string()).collect(),
};
let response = self
.client
.post(format!("{}/api/embed", url))
.json(&request)
.send()
.await?;
let response = {
let mut rb = self
.client
.post(format!("{}/api/embed", url))
.json(&request);
if let Some(s) = span.as_mut() {
rb = self.inject_traceparent(rb, s);
}
match rb.send().await {
Ok(r) => r,
Err(e) => {
if let Some(s) = span.as_mut() {
s.set_status(Status::error("Ollama embedding request failed"));
}
return Err(e.into());
}
}
};
if !response.status().is_success() {
let status = response.status();
if let Some(s) = span.as_mut() {
s.set_attribute(KeyValue::new("http.status_code", status.as_u16() as i64));
s.set_status(Status::error(format!(
"Ollama batch embedding request failed: {status}"
)));
}
let error_body = response.text().await.unwrap_or_default();
return Err(anyhow::anyhow!(
"Ollama batch embedding request failed: {} - {}",
@@ -1098,8 +1296,25 @@ Analyze the image and use specific details from both the visual content and the
error_body
));
}
if let Some(s) = span.as_mut() {
s.set_attribute(KeyValue::new(
"http.status_code",
response.status().as_u16() as i64,
));
}
let result: OllamaEmbedResponse = response.json().await?;
let result: OllamaEmbedResponse = match response.json().await {
Ok(r) => r,
Err(e) => {
if let Some(s) = span.as_mut() {
s.set_status(Status::error("parsing Ollama embed response failed"));
}
return Err(e.into());
}
};
if let Some(s) = span.as_mut() {
s.set_status(Status::Ok);
}
Ok(result.embeddings)
}
}
@@ -1108,35 +1323,38 @@ Analyze the image and use specific details from both the visual content and the
impl LlmClient for OllamaClient {
async fn generate(
&self,
cx: &Context,
prompt: &str,
system: Option<&str>,
images: Option<Vec<String>>,
) -> Result<String> {
self.generate_with_images(prompt, system, images).await
self.generate_with_images(cx, prompt, system, images).await
}
async fn chat_with_tools(
&self,
cx: &Context,
messages: Vec<ChatMessage>,
tools: Vec<Tool>,
) -> Result<(ChatMessage, Option<i32>, Option<i32>)> {
OllamaClient::chat_with_tools(self, messages, tools).await
OllamaClient::chat_with_tools(self, cx, messages, tools).await
}
async fn chat_with_tools_stream(
&self,
cx: &Context,
messages: Vec<ChatMessage>,
tools: Vec<Tool>,
) -> Result<BoxStream<'static, Result<LlmStreamEvent>>> {
OllamaClient::chat_with_tools_stream(self, messages, tools).await
OllamaClient::chat_with_tools_stream(self, cx, messages, tools).await
}
async fn generate_embeddings(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
OllamaClient::generate_embeddings(self, texts).await
async fn generate_embeddings(&self, cx: &Context, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
OllamaClient::generate_embeddings(self, cx, texts).await
}
async fn describe_image(&self, image_base64: &str) -> Result<String> {
self.generate_photo_description(image_base64).await
async fn describe_image(&self, cx: &Context, image_base64: &str) -> Result<String> {
self.generate_photo_description(cx, image_base64).await
}
async fn list_models(&self) -> Result<Vec<ModelCapabilities>> {