Cameron CordesandClaude Opus 5.5
09b9e87396
README, CLAUDE.md and .env.example now point at llm.toml for LLM settings (OPENROUTER_API_KEY stays in env) and describe the legacy env override window, the discovered picker and the startup cross-check. ADR-0003 records the move and amends ADR-0002's deployment-config home; CONTEXT.md's Deployment config / Hard default entries follow. Source comments and the "set LLAMA_SWAP_URL" error strings name the llm.toml keys. Also fixes README's AGENTIC_MAX_ITERATIONS default (10 -> 12, what the code uses). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1104 lines
41 KiB
Rust
1104 lines
41 KiB
Rust
// First consumer lands in a later PR (hybrid backend routing). Tests exercise
|
|
// the translation helpers directly.
|
|
#![allow(dead_code)]
|
|
|
|
use anyhow::Context as _;
|
|
use anyhow::{Result, anyhow, bail};
|
|
use async_trait::async_trait;
|
|
use opentelemetry::trace::Status;
|
|
use reqwest::Client;
|
|
use serde::Deserialize;
|
|
use serde_json::{Value, json};
|
|
use std::collections::HashMap;
|
|
use std::sync::{Arc, Mutex};
|
|
use std::time::{Duration, Instant};
|
|
use tracing_opentelemetry::OpenTelemetrySpanExt as _;
|
|
|
|
use crate::ai::llm_client::{
|
|
ChatMessage, LlmClient, LlmStreamEvent, ModelCapabilities, Tool, ToolCall, ToolCallFunction,
|
|
};
|
|
|
|
use futures::stream::{BoxStream, StreamExt};
|
|
|
|
const DEFAULT_BASE_URL: &str = "https://openrouter.ai/api/v1";
|
|
const DEFAULT_EMBEDDING_MODEL: &str = "openai/text-embedding-3-small";
|
|
const CACHE_DURATION_SECS: u64 = 15 * 60;
|
|
/// Default completion cap sent as `max_tokens`. OpenRouter pre-authorizes
|
|
/// credit against the requested max (defaulting to the model's full output
|
|
/// window when omitted), so expensive models fail with credit-limit errors
|
|
/// even when the actual reply is short. Override via `openrouter.max_tokens`.
|
|
pub const DEFAULT_MAX_TOKENS: u32 = 16384;
|
|
|
|
#[derive(Clone)]
|
|
struct CachedEntry<T> {
|
|
data: T,
|
|
cached_at: Instant,
|
|
}
|
|
|
|
impl<T> CachedEntry<T> {
|
|
fn new(data: T) -> Self {
|
|
Self {
|
|
data,
|
|
cached_at: Instant::now(),
|
|
}
|
|
}
|
|
|
|
fn is_expired(&self) -> bool {
|
|
self.cached_at.elapsed().as_secs() > CACHE_DURATION_SECS
|
|
}
|
|
}
|
|
|
|
/// The error every image-carrying OpenRouter request fails with.
|
|
pub const IMAGES_STAY_LOCAL: &str = "refusing to send image data to OpenRouter: photos stay on \
|
|
the local network (describe them with the local vision model and send the text)";
|
|
|
|
lazy_static::lazy_static! {
|
|
static ref MODEL_CAPABILITIES_CACHE: Arc<Mutex<HashMap<String, CachedEntry<Vec<ModelCapabilities>>>>> =
|
|
Arc::new(Mutex::new(HashMap::new()));
|
|
}
|
|
|
|
/// OpenAI-compatible client for OpenRouter (https://openrouter.ai).
|
|
///
|
|
/// Translates canonical `ChatMessage` / `Tool` shapes to OpenAI wire format:
|
|
/// - Tool-call `arguments` serialized as JSON-encoded strings (vs Ollama's
|
|
/// native JSON).
|
|
/// - **Never sends images.** Photos stay on the local network: hybrid mode
|
|
/// describes them with the local vision model and sends only the text.
|
|
/// A message that still carries image data is refused (see
|
|
/// [`IMAGES_STAY_LOCAL`]) rather than uploaded, so a caller bug fails a
|
|
/// turn instead of leaking a photo.
|
|
/// - `role=tool` messages attach a `tool_call_id` inferred from the preceding
|
|
/// assistant turn's tool call.
|
|
#[derive(Clone)]
|
|
pub struct OpenRouterClient {
|
|
client: Client,
|
|
pub api_key: String,
|
|
pub base_url: String,
|
|
pub primary_model: String,
|
|
pub embedding_model: String,
|
|
num_ctx: Option<i32>,
|
|
temperature: Option<f32>,
|
|
top_p: Option<f32>,
|
|
top_k: Option<i32>,
|
|
min_p: Option<f32>,
|
|
/// Upper bound on generated tokens per call, sent as `max_tokens`.
|
|
/// `None` omits the field (provider default).
|
|
max_tokens: Option<u32>,
|
|
/// Optional `HTTP-Referer` header OpenRouter uses for attribution.
|
|
pub referer: Option<String>,
|
|
/// Optional `X-Title` header OpenRouter uses for attribution.
|
|
pub app_title: Option<String>,
|
|
}
|
|
|
|
impl OpenRouterClient {
|
|
pub fn new(api_key: String, base_url: Option<String>, primary_model: String) -> Self {
|
|
Self {
|
|
client: Client::builder()
|
|
.connect_timeout(Duration::from_secs(10))
|
|
.timeout(Duration::from_secs(180))
|
|
.build()
|
|
.unwrap_or_else(|_| Client::new()),
|
|
api_key,
|
|
base_url: base_url.unwrap_or_else(|| DEFAULT_BASE_URL.to_string()),
|
|
primary_model,
|
|
embedding_model: DEFAULT_EMBEDDING_MODEL.to_string(),
|
|
num_ctx: None,
|
|
temperature: None,
|
|
top_p: None,
|
|
top_k: None,
|
|
min_p: None,
|
|
max_tokens: Some(DEFAULT_MAX_TOKENS),
|
|
referer: None,
|
|
app_title: None,
|
|
}
|
|
}
|
|
|
|
pub fn set_embedding_model(&mut self, model: String) {
|
|
self.embedding_model = model;
|
|
}
|
|
|
|
#[allow(dead_code)]
|
|
pub fn set_num_ctx(&mut self, num_ctx: Option<i32>) {
|
|
self.num_ctx = num_ctx;
|
|
}
|
|
|
|
#[allow(dead_code)]
|
|
pub fn set_sampling_params(
|
|
&mut self,
|
|
temperature: Option<f32>,
|
|
top_p: Option<f32>,
|
|
top_k: Option<i32>,
|
|
min_p: Option<f32>,
|
|
) {
|
|
self.temperature = temperature;
|
|
self.top_p = top_p;
|
|
self.top_k = top_k;
|
|
self.min_p = min_p;
|
|
}
|
|
|
|
pub fn set_max_tokens(&mut self, max_tokens: Option<u32>) {
|
|
self.max_tokens = max_tokens;
|
|
}
|
|
|
|
pub fn set_attribution(&mut self, referer: Option<String>, app_title: Option<String>) {
|
|
self.referer = referer;
|
|
self.app_title = app_title;
|
|
}
|
|
|
|
fn authed(&self, builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
|
|
let mut b = builder.bearer_auth(&self.api_key);
|
|
if let Some(r) = &self.referer {
|
|
b = b.header("HTTP-Referer", r);
|
|
}
|
|
if let Some(t) = &self.app_title {
|
|
b = b.header("X-Title", t);
|
|
}
|
|
b
|
|
}
|
|
|
|
/// Start the `llm.request` span for an outgoing call. `operation` is
|
|
/// `chat`, `embeddings`, or `describe` — the call shape, not the entry
|
|
/// point (a `describe_image` funneled through `chat_with_tools` is
|
|
/// still a `describe`).
|
|
fn start_request_span(&self, operation: &str, model: &str) -> tracing::Span {
|
|
crate::otel::start_llm_request_span("openrouter", operation, model)
|
|
}
|
|
|
|
/// Translate canonical messages to the OpenAI-compatible wire shape.
|
|
///
|
|
/// Walks in order so it can attach `tool_call_id` to `role=tool` messages
|
|
/// based on the most recent assistant turn's tool call.
|
|
///
|
|
/// Errors if any message carries image data: this is the one choke point
|
|
/// every OpenRouter request passes through, so the "photos never leave
|
|
/// the network" rule is enforced here rather than trusted to callers.
|
|
fn messages_to_openai(messages: &[ChatMessage]) -> Result<Vec<Value>> {
|
|
if messages
|
|
.iter()
|
|
.any(|m| m.images.as_ref().is_some_and(|imgs| !imgs.is_empty()))
|
|
{
|
|
log::error!("{IMAGES_STAY_LOCAL}");
|
|
bail!(IMAGES_STAY_LOCAL);
|
|
}
|
|
let mut out = Vec::with_capacity(messages.len());
|
|
let mut last_tool_call_ids: Vec<String> = Vec::new();
|
|
let mut next_tool_result_idx: usize = 0;
|
|
|
|
for msg in messages {
|
|
let mut obj = serde_json::Map::new();
|
|
obj.insert("role".into(), Value::String(msg.role.clone()));
|
|
|
|
// Text only — images were refused above.
|
|
obj.insert("content".into(), Value::String(msg.content.clone()));
|
|
|
|
// Assistant message with tool_calls: stringify arguments, remember
|
|
// the ids so the subsequent tool messages can reference them.
|
|
if let Some(tcs) = &msg.tool_calls
|
|
&& msg.role == "assistant"
|
|
{
|
|
let converted: Vec<Value> = tcs
|
|
.iter()
|
|
.enumerate()
|
|
.map(|(i, call)| {
|
|
let id = call.id.clone().unwrap_or_else(|| format!("call_{}", i));
|
|
let args_str = serde_json::to_string(&call.function.arguments)
|
|
.unwrap_or_else(|_| "{}".to_string());
|
|
json!({
|
|
"id": id,
|
|
"type": "function",
|
|
"function": {
|
|
"name": call.function.name,
|
|
"arguments": args_str,
|
|
}
|
|
})
|
|
})
|
|
.collect();
|
|
last_tool_call_ids = converted
|
|
.iter()
|
|
.filter_map(|v| v.get("id").and_then(|x| x.as_str()).map(String::from))
|
|
.collect();
|
|
next_tool_result_idx = 0;
|
|
obj.insert("tool_calls".into(), Value::Array(converted));
|
|
}
|
|
|
|
// Tool result messages: attach tool_call_id from the last assistant turn.
|
|
if msg.role == "tool" {
|
|
let id = last_tool_call_ids
|
|
.get(next_tool_result_idx)
|
|
.cloned()
|
|
.unwrap_or_else(|| "call_0".to_string());
|
|
obj.insert("tool_call_id".into(), Value::String(id));
|
|
next_tool_result_idx += 1;
|
|
}
|
|
|
|
out.push(Value::Object(obj));
|
|
}
|
|
|
|
Ok(out)
|
|
}
|
|
|
|
/// Parse an OpenAI-compatible assistant message back into canonical shape.
|
|
fn openai_message_to_chat(msg: &Value) -> Result<ChatMessage> {
|
|
let obj = msg
|
|
.as_object()
|
|
.ok_or_else(|| anyhow!("response message is not an object"))?;
|
|
let role = obj
|
|
.get("role")
|
|
.and_then(|v| v.as_str())
|
|
.unwrap_or("assistant")
|
|
.to_string();
|
|
let content = obj
|
|
.get("content")
|
|
.and_then(|v| v.as_str())
|
|
.unwrap_or("")
|
|
.to_string();
|
|
|
|
let tool_calls = if let Some(tcs) = obj.get("tool_calls").and_then(|v| v.as_array()) {
|
|
let mut parsed = Vec::with_capacity(tcs.len());
|
|
for tc in tcs {
|
|
let id = tc.get("id").and_then(|v| v.as_str()).map(String::from);
|
|
let function = tc
|
|
.get("function")
|
|
.ok_or_else(|| anyhow!("tool_call missing function field"))?;
|
|
let name = function
|
|
.get("name")
|
|
.and_then(|v| v.as_str())
|
|
.unwrap_or_default()
|
|
.to_string();
|
|
let args_value = match function.get("arguments") {
|
|
// OpenAI-compat: stringified JSON.
|
|
Some(Value::String(s)) => {
|
|
serde_json::from_str::<Value>(s).unwrap_or_else(|_| json!({}))
|
|
}
|
|
// Some providers emit arguments as an object directly — accept both.
|
|
Some(v @ Value::Object(_)) => v.clone(),
|
|
_ => json!({}),
|
|
};
|
|
parsed.push(ToolCall {
|
|
id,
|
|
function: ToolCallFunction {
|
|
name,
|
|
arguments: args_value,
|
|
},
|
|
});
|
|
}
|
|
Some(parsed)
|
|
} else {
|
|
None
|
|
};
|
|
|
|
Ok(ChatMessage {
|
|
role,
|
|
content,
|
|
tool_calls,
|
|
images: None,
|
|
})
|
|
}
|
|
|
|
fn build_options(&self) -> Vec<(&'static str, Value)> {
|
|
let mut v = Vec::new();
|
|
if let Some(t) = self.temperature {
|
|
v.push(("temperature", json!(t)));
|
|
}
|
|
if let Some(p) = self.top_p {
|
|
v.push(("top_p", json!(p)));
|
|
}
|
|
if let Some(k) = self.top_k {
|
|
v.push(("top_k", json!(k)));
|
|
}
|
|
if let Some(m) = self.min_p {
|
|
v.push(("min_p", json!(m)));
|
|
}
|
|
if let Some(c) = self.num_ctx {
|
|
// OpenAI uses max_tokens for generation bound; num_ctx isn't
|
|
// directly transferable. Skip rather than silently mis-map.
|
|
let _ = c;
|
|
}
|
|
if let Some(m) = self.max_tokens {
|
|
v.push(("max_tokens", json!(m)));
|
|
}
|
|
v
|
|
}
|
|
}
|
|
|
|
#[async_trait]
|
|
impl LlmClient for OpenRouterClient {
|
|
async fn generate(
|
|
&self,
|
|
prompt: &str,
|
|
system: Option<&str>,
|
|
images: Option<Vec<String>>,
|
|
) -> Result<String> {
|
|
let mut messages: Vec<ChatMessage> = Vec::new();
|
|
if let Some(sys) = system {
|
|
messages.push(ChatMessage::system(sys));
|
|
}
|
|
let mut user = ChatMessage::user(prompt);
|
|
user.images = images;
|
|
messages.push(user);
|
|
|
|
// Delegates to chat_with_tools, which starts the llm.request span
|
|
// (operation=chat) — one span per HTTP request.
|
|
let (reply, _, _) = self.chat_with_tools(messages, Vec::new()).await?;
|
|
Ok(reply.content)
|
|
}
|
|
|
|
async fn chat_with_tools(
|
|
&self,
|
|
messages: Vec<ChatMessage>,
|
|
tools: Vec<Tool>,
|
|
) -> Result<(ChatMessage, Option<i32>, Option<i32>)> {
|
|
let span = self.start_request_span("chat", &self.primary_model);
|
|
self.chat_with_tools_impl(Some(&span), messages, tools)
|
|
.await
|
|
}
|
|
|
|
async fn chat_with_tools_stream(
|
|
&self,
|
|
messages: Vec<ChatMessage>,
|
|
tools: Vec<Tool>,
|
|
) -> Result<BoxStream<'static, Result<LlmStreamEvent>>> {
|
|
// The span outlives this method — the stream yields until the
|
|
// response fully drains, so the owned span moves into the
|
|
// generator and is finalized there.
|
|
let span = self.start_request_span("chat_stream", &self.primary_model);
|
|
|
|
let url = format!("{}/chat/completions", self.base_url);
|
|
let mut body = serde_json::Map::new();
|
|
body.insert("model".into(), Value::String(self.primary_model.clone()));
|
|
body.insert(
|
|
"messages".into(),
|
|
Value::Array(Self::messages_to_openai(&messages)?),
|
|
);
|
|
body.insert("stream".into(), Value::Bool(true));
|
|
// Ask for usage data in the final chunk (OpenAI + OpenRouter
|
|
// both honor this options bag).
|
|
body.insert(
|
|
"stream_options".into(),
|
|
serde_json::json!({ "include_usage": true }),
|
|
);
|
|
if !tools.is_empty() {
|
|
body.insert(
|
|
"tools".into(),
|
|
serde_json::to_value(&tools).context("serializing tools")?,
|
|
);
|
|
}
|
|
for (k, v) in self.build_options() {
|
|
body.insert(k.into(), v);
|
|
}
|
|
|
|
let mut request = self
|
|
.authed(self.client.post(&url))
|
|
.json(&Value::Object(body));
|
|
request = crate::otel::with_traceparent(request, &span);
|
|
let resp = request
|
|
.send()
|
|
.await
|
|
.with_context(|| format!("POST {} failed", url))?;
|
|
|
|
if !resp.status().is_success() {
|
|
let status = resp.status();
|
|
crate::otel::record_http_failure(
|
|
Some(&span),
|
|
status.as_u16(),
|
|
&format!("OpenRouter stream request failed: {status}"),
|
|
);
|
|
let body = resp.text().await.unwrap_or_default();
|
|
bail!("OpenRouter stream request failed: {} — {}", status, body);
|
|
}
|
|
crate::otel::set_http_status(Some(&span), resp.status().as_u16());
|
|
|
|
// OpenAI-compat SSE stream. Each event is `data: <json>\n\n`, with
|
|
// `data: [DONE]` signalling completion. Tool calls arrive as
|
|
// `delta.tool_calls[i]` chunks that must be concatenated by index.
|
|
let byte_stream = resp.bytes_stream();
|
|
let stream = async_stream::stream! {
|
|
let mut byte_stream = byte_stream;
|
|
let mut buf: Vec<u8> = Vec::new();
|
|
let mut accumulated_content = String::new();
|
|
// tool call state: index -> (id, name, args_string)
|
|
let mut tool_state: std::collections::BTreeMap<
|
|
usize,
|
|
(Option<String>, Option<String>, String),
|
|
> = std::collections::BTreeMap::new();
|
|
let mut role = "assistant".to_string();
|
|
let mut prompt_tokens: Option<i32> = None;
|
|
let mut completion_tokens: Option<i32> = None;
|
|
let mut done_seen = false;
|
|
|
|
while let Some(chunk) = byte_stream.next().await {
|
|
let chunk = match chunk {
|
|
Ok(b) => b,
|
|
Err(e) => {
|
|
span.set_status(Status::error(format!("stream read failed: {e}")));
|
|
yield Err(anyhow!("stream read failed: {}", e));
|
|
return;
|
|
}
|
|
};
|
|
buf.extend_from_slice(&chunk);
|
|
|
|
// SSE frames are delimited by a blank line. Walk the buffer
|
|
// for "\n\n" markers; anything before them is a complete
|
|
// frame (possibly multi-line).
|
|
while let Some(sep) = find_double_newline(&buf) {
|
|
let frame = buf.drain(..sep + 2).collect::<Vec<_>>();
|
|
let frame_str = match std::str::from_utf8(&frame) {
|
|
Ok(s) => s,
|
|
Err(_) => continue,
|
|
};
|
|
// A frame is one or more lines; the payload is on data:
|
|
// lines. Ignore comments and other fields.
|
|
for line in frame_str.lines() {
|
|
let line = line.trim_end_matches('\r');
|
|
let payload = match line.strip_prefix("data: ") {
|
|
Some(p) => p,
|
|
None => continue,
|
|
};
|
|
if payload == "[DONE]" {
|
|
done_seen = true;
|
|
break;
|
|
}
|
|
let v: Value = match serde_json::from_str(payload) {
|
|
Ok(v) => v,
|
|
Err(e) => {
|
|
log::warn!(
|
|
"malformed OpenRouter SSE frame: {} ({})",
|
|
payload,
|
|
e
|
|
);
|
|
continue;
|
|
}
|
|
};
|
|
|
|
// Usage can arrive in a dedicated final frame with
|
|
// empty choices.
|
|
if let Some(usage) = v.get("usage") {
|
|
prompt_tokens = usage
|
|
.get("prompt_tokens")
|
|
.and_then(|n| n.as_i64())
|
|
.map(|n| n as i32);
|
|
completion_tokens = usage
|
|
.get("completion_tokens")
|
|
.and_then(|n| n.as_i64())
|
|
.map(|n| n as i32);
|
|
if let Some(cost) = usage.get("cost").and_then(|c| c.as_f64()) {
|
|
span.set_attribute("llm.cost_usd", cost);
|
|
}
|
|
}
|
|
|
|
let Some(choices) = v.get("choices").and_then(|c| c.as_array())
|
|
else {
|
|
continue;
|
|
};
|
|
let Some(choice) = choices.first() else { continue };
|
|
let delta = match choice.get("delta") {
|
|
Some(d) => d,
|
|
None => continue,
|
|
};
|
|
if let Some(r) = delta.get("role").and_then(|v| v.as_str()) {
|
|
role = r.to_string();
|
|
}
|
|
if let Some(content) =
|
|
delta.get("content").and_then(|v| v.as_str())
|
|
&& !content.is_empty()
|
|
{
|
|
accumulated_content.push_str(content);
|
|
yield Ok(LlmStreamEvent::TextDelta(content.to_string()));
|
|
}
|
|
if let Some(tcs) = delta.get("tool_calls").and_then(|v| v.as_array()) {
|
|
for tc_delta in tcs {
|
|
let idx = tc_delta
|
|
.get("index")
|
|
.and_then(|n| n.as_u64())
|
|
.unwrap_or(0) as usize;
|
|
let entry = tool_state
|
|
.entry(idx)
|
|
.or_insert((None, None, String::new()));
|
|
if let Some(id) =
|
|
tc_delta.get("id").and_then(|v| v.as_str())
|
|
{
|
|
entry.0 = Some(id.to_string());
|
|
}
|
|
if let Some(func) = tc_delta.get("function") {
|
|
if let Some(name) =
|
|
func.get("name").and_then(|v| v.as_str())
|
|
{
|
|
entry.1 = Some(name.to_string());
|
|
}
|
|
if let Some(args) =
|
|
func.get("arguments").and_then(|v| v.as_str())
|
|
{
|
|
entry.2.push_str(args);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if done_seen {
|
|
break;
|
|
}
|
|
}
|
|
if done_seen {
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Finalize tool calls: parse accumulated argument strings.
|
|
let tool_calls: Option<Vec<ToolCall>> = if tool_state.is_empty() {
|
|
None
|
|
} else {
|
|
let mut v = Vec::with_capacity(tool_state.len());
|
|
for (_idx, (id, name, args)) in tool_state {
|
|
let arguments: Value = if args.trim().is_empty() {
|
|
Value::Object(Default::default())
|
|
} else {
|
|
serde_json::from_str(&args).unwrap_or_else(|_| {
|
|
Value::Object(Default::default())
|
|
})
|
|
};
|
|
v.push(ToolCall {
|
|
id,
|
|
function: ToolCallFunction {
|
|
name: name.unwrap_or_default(),
|
|
arguments,
|
|
},
|
|
});
|
|
}
|
|
Some(v)
|
|
};
|
|
|
|
let message = ChatMessage {
|
|
role,
|
|
content: accumulated_content,
|
|
tool_calls,
|
|
images: None,
|
|
};
|
|
if let Some(pt) = prompt_tokens {
|
|
span.set_attribute("llm.tokens.input", pt as i64);
|
|
}
|
|
if let Some(ct) = completion_tokens {
|
|
span.set_attribute("llm.tokens.output", ct as i64);
|
|
}
|
|
span.set_status(Status::Ok);
|
|
yield Ok(LlmStreamEvent::Done {
|
|
message,
|
|
prompt_eval_count: prompt_tokens,
|
|
eval_count: completion_tokens,
|
|
});
|
|
};
|
|
|
|
Ok(Box::pin(stream))
|
|
}
|
|
|
|
async fn generate_embeddings(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
|
|
let span = self.start_request_span("embeddings", &self.embedding_model);
|
|
let url = format!("{}/embeddings", self.base_url);
|
|
let body = json!({
|
|
"model": self.embedding_model,
|
|
"input": texts,
|
|
});
|
|
|
|
let mut request = self.authed(self.client.post(&url)).json(&body);
|
|
request = crate::otel::with_traceparent(request, &span);
|
|
let resp = request
|
|
.send()
|
|
.await
|
|
.with_context(|| format!("POST {} failed", url))
|
|
.inspect_err(|_| {
|
|
span.set_status(Status::error("OpenRouter embedding request failed"));
|
|
})?;
|
|
|
|
if !resp.status().is_success() {
|
|
let status = resp.status();
|
|
crate::otel::record_http_failure(
|
|
Some(&span),
|
|
status.as_u16(),
|
|
&format!("OpenRouter embedding request failed: {status}"),
|
|
);
|
|
let body = resp.text().await.unwrap_or_default();
|
|
bail!("OpenRouter embedding request failed: {} — {}", status, body);
|
|
}
|
|
crate::otel::set_http_status(Some(&span), resp.status().as_u16());
|
|
|
|
#[derive(Deserialize)]
|
|
struct EmbedResponse {
|
|
data: Vec<EmbedItem>,
|
|
}
|
|
#[derive(Deserialize)]
|
|
struct EmbedItem {
|
|
embedding: Vec<f32>,
|
|
}
|
|
|
|
let parsed: EmbedResponse = resp.json().await.context("parsing embed response")?;
|
|
span.set_status(Status::Ok);
|
|
Ok(parsed.data.into_iter().map(|i| i.embedding).collect())
|
|
}
|
|
|
|
async fn describe_image(&self, _image_base64: &str) -> Result<String> {
|
|
// Describing an image means uploading it. Hybrid mode always
|
|
// describes on the local backend (`ResolvedBackend::local`), so this
|
|
// is never the intended path — refuse loudly if something reaches it.
|
|
log::error!("{IMAGES_STAY_LOCAL}");
|
|
bail!(IMAGES_STAY_LOCAL)
|
|
}
|
|
|
|
async fn list_models(&self) -> Result<Vec<ModelCapabilities>> {
|
|
{
|
|
let cache = MODEL_CAPABILITIES_CACHE.lock().unwrap();
|
|
if let Some(entry) = cache.get(&self.base_url)
|
|
&& !entry.is_expired()
|
|
{
|
|
return Ok(entry.data.clone());
|
|
}
|
|
}
|
|
|
|
let url = format!("{}/models", self.base_url);
|
|
let resp = self
|
|
.authed(self.client.get(&url))
|
|
.send()
|
|
.await
|
|
.with_context(|| format!("GET {} failed", url))?;
|
|
|
|
if !resp.status().is_success() {
|
|
let status = resp.status();
|
|
let body = resp.text().await.unwrap_or_default();
|
|
bail!("OpenRouter list_models failed: {} — {}", status, body);
|
|
}
|
|
|
|
let parsed: Value = resp.json().await.context("parsing models response")?;
|
|
let data = parsed
|
|
.get("data")
|
|
.and_then(|v| v.as_array())
|
|
.ok_or_else(|| anyhow!("models response missing data[]"))?;
|
|
|
|
let caps: Vec<ModelCapabilities> = data.iter().map(parse_model_capabilities).collect();
|
|
|
|
{
|
|
let mut cache = MODEL_CAPABILITIES_CACHE.lock().unwrap();
|
|
cache.insert(self.base_url.clone(), CachedEntry::new(caps.clone()));
|
|
}
|
|
|
|
Ok(caps)
|
|
}
|
|
|
|
async fn model_capabilities(&self, model: &str) -> Result<ModelCapabilities> {
|
|
let all = self.list_models().await?;
|
|
all.into_iter()
|
|
.find(|m| m.name == model)
|
|
.ok_or_else(|| anyhow!("model '{}' not found on OpenRouter", model))
|
|
}
|
|
|
|
fn primary_model(&self) -> &str {
|
|
&self.primary_model
|
|
}
|
|
}
|
|
|
|
impl OpenRouterClient {
|
|
/// HTTP core of `chat_with_tools`. `span` is the `llm.request` span
|
|
/// started by the caller (or a funnelling entry point like
|
|
/// `describe_image`) so attributes from the response land on the right
|
|
/// operation label.
|
|
async fn chat_with_tools_impl(
|
|
&self,
|
|
span: Option<&tracing::Span>,
|
|
messages: Vec<ChatMessage>,
|
|
tools: Vec<Tool>,
|
|
) -> Result<(ChatMessage, Option<i32>, Option<i32>)> {
|
|
let url = format!("{}/chat/completions", self.base_url);
|
|
let mut body = serde_json::Map::new();
|
|
body.insert("model".into(), Value::String(self.primary_model.clone()));
|
|
body.insert(
|
|
"messages".into(),
|
|
Value::Array(Self::messages_to_openai(&messages)?),
|
|
);
|
|
body.insert("stream".into(), Value::Bool(false));
|
|
if !tools.is_empty() {
|
|
body.insert(
|
|
"tools".into(),
|
|
serde_json::to_value(&tools).context("serializing tools")?,
|
|
);
|
|
}
|
|
for (k, v) in self.build_options() {
|
|
body.insert(k.into(), v);
|
|
}
|
|
|
|
log::info!(
|
|
"OpenRouter chat_with_tools: model={} messages={} tools={}",
|
|
self.primary_model,
|
|
messages.len(),
|
|
tools.len()
|
|
);
|
|
|
|
let mut request = self
|
|
.authed(self.client.post(&url))
|
|
.json(&Value::Object(body));
|
|
if let Some(s) = span {
|
|
request = crate::otel::with_traceparent(request, s);
|
|
}
|
|
let resp = match request.send().await {
|
|
Ok(r) => r,
|
|
Err(e) => {
|
|
if let Some(s) = span {
|
|
s.set_status(Status::error("OpenRouter request failed"));
|
|
}
|
|
return Err(e).with_context(|| format!("POST {} failed", url));
|
|
}
|
|
};
|
|
|
|
if !resp.status().is_success() {
|
|
let status = resp.status();
|
|
crate::otel::record_http_failure(
|
|
span,
|
|
status.as_u16(),
|
|
&format!("OpenRouter chat request failed: {status}"),
|
|
);
|
|
let body = resp.text().await.unwrap_or_default();
|
|
bail!("OpenRouter chat request failed: {} — {}", status, body);
|
|
}
|
|
crate::otel::set_http_status(span, resp.status().as_u16());
|
|
|
|
let parsed: Value = match resp.json().await {
|
|
Ok(v) => v,
|
|
Err(e) => {
|
|
if let Some(s) = span {
|
|
s.set_status(Status::error("parsing chat response failed"));
|
|
}
|
|
return Err(e).context("parsing chat response");
|
|
}
|
|
};
|
|
let choice = parsed
|
|
.get("choices")
|
|
.and_then(|v| v.as_array())
|
|
.and_then(|a| a.first())
|
|
.ok_or_else(|| {
|
|
anyhow!(
|
|
"response missing choices[0]: {}",
|
|
extract_openrouter_error_detail(&parsed)
|
|
)
|
|
})?;
|
|
let msg = choice.get("message").ok_or_else(|| {
|
|
anyhow!(
|
|
"choices[0] missing message: {}",
|
|
extract_openrouter_error_detail(&parsed)
|
|
)
|
|
})?;
|
|
let chat_msg = Self::openai_message_to_chat(msg)?;
|
|
|
|
let usage = parsed.get("usage");
|
|
let prompt_tokens = usage
|
|
.and_then(|u| u.get("prompt_tokens"))
|
|
.and_then(|v| v.as_i64())
|
|
.map(|n| n as i32);
|
|
let completion_tokens = usage
|
|
.and_then(|u| u.get("completion_tokens"))
|
|
.and_then(|v| v.as_i64())
|
|
.map(|n| n as i32);
|
|
// OpenRouter reports per-request USD cost in `usage.cost`.
|
|
if let Some(s) = span {
|
|
if let Some(pt) = prompt_tokens {
|
|
s.set_attribute("llm.tokens.input", pt as i64);
|
|
}
|
|
if let Some(ct) = completion_tokens {
|
|
s.set_attribute("llm.tokens.output", ct as i64);
|
|
}
|
|
if let Some(cost) = usage.and_then(|u| u.get("cost")).and_then(|c| c.as_f64()) {
|
|
s.set_attribute("llm.cost_usd", cost);
|
|
}
|
|
s.set_status(Status::Ok);
|
|
}
|
|
|
|
Ok((chat_msg, prompt_tokens, completion_tokens))
|
|
}
|
|
}
|
|
|
|
/// Extract a diagnostic fragment from an OpenRouter response body that
|
|
/// doesn't match the expected `{choices: [...]}` shape. OpenRouter will
|
|
/// sometimes return 200 OK with `{"error": {"message": "...", "code": ...}}`
|
|
/// when the upstream provider (Anthropic/OpenAI/Google/etc) errored out
|
|
/// — rate limits, content moderation, model overload, provider timeout.
|
|
/// Surface the structured error if present; otherwise fall back to a
|
|
/// truncated raw-JSON view so the log line is actionable.
|
|
fn extract_openrouter_error_detail(parsed: &Value) -> String {
|
|
if let Some(err) = parsed.get("error") {
|
|
let message = err
|
|
.get("message")
|
|
.and_then(|v| v.as_str())
|
|
.unwrap_or("(no message)");
|
|
let code = err
|
|
.get("code")
|
|
.map(|v| match v {
|
|
Value::String(s) => s.clone(),
|
|
other => other.to_string(),
|
|
})
|
|
.unwrap_or_else(|| "?".to_string());
|
|
let short_message: String = message.chars().take(240).collect();
|
|
return format!("error code={} message=\"{}\"", code, short_message);
|
|
}
|
|
let raw = parsed.to_string();
|
|
raw.chars().take(300).collect()
|
|
}
|
|
|
|
/// Find the byte offset of the first `\n\n` (end of an SSE frame) in `buf`.
|
|
/// Returns the index of the first `\n` of the pair, so the full separator is
|
|
/// `buf[idx..=idx+1]`. Also handles `\r\n\r\n` since some servers emit it.
|
|
fn find_double_newline(buf: &[u8]) -> Option<usize> {
|
|
for i in 0..buf.len().saturating_sub(1) {
|
|
if buf[i] == b'\n' && buf[i + 1] == b'\n' {
|
|
return Some(i);
|
|
}
|
|
// \r\n\r\n: the second \n of this pattern is at i+2; flag at i so the
|
|
// drain call (which consumes ..sep+2) takes exactly the frame.
|
|
if i + 3 < buf.len()
|
|
&& buf[i] == b'\r'
|
|
&& buf[i + 1] == b'\n'
|
|
&& buf[i + 2] == b'\r'
|
|
&& buf[i + 3] == b'\n'
|
|
{
|
|
return Some(i + 1);
|
|
}
|
|
}
|
|
None
|
|
}
|
|
|
|
fn parse_model_capabilities(m: &Value) -> ModelCapabilities {
|
|
let name = m
|
|
.get("id")
|
|
.and_then(|v| v.as_str())
|
|
.unwrap_or_default()
|
|
.to_string();
|
|
let has_tool_calling = m
|
|
.get("supported_parameters")
|
|
.and_then(|v| v.as_array())
|
|
.map(|arr| arr.iter().any(|x| x.as_str() == Some("tools")))
|
|
.unwrap_or(false);
|
|
let has_vision = m
|
|
.get("architecture")
|
|
.and_then(|v| v.get("input_modalities"))
|
|
.and_then(|v| v.as_array())
|
|
.map(|arr| arr.iter().any(|x| x.as_str() == Some("image")))
|
|
.unwrap_or(false);
|
|
ModelCapabilities {
|
|
name,
|
|
has_vision,
|
|
has_tool_calling,
|
|
context_window: None,
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn tool_call_arguments_stringified_on_send() {
|
|
let mut msg = ChatMessage {
|
|
role: "assistant".into(),
|
|
content: String::new(),
|
|
tool_calls: Some(vec![ToolCall {
|
|
id: Some("call_abc".into()),
|
|
function: ToolCallFunction {
|
|
name: "search_sms".into(),
|
|
arguments: json!({"query": "hello", "limit": 5}),
|
|
},
|
|
}]),
|
|
images: None,
|
|
};
|
|
msg.tool_calls.as_mut().unwrap()[0].function.arguments =
|
|
json!({"query": "hello", "limit": 5});
|
|
|
|
let wire = OpenRouterClient::messages_to_openai(&[msg]).unwrap();
|
|
let tcs = wire[0]
|
|
.get("tool_calls")
|
|
.and_then(|v| v.as_array())
|
|
.expect("tool_calls present");
|
|
let args = tcs[0]
|
|
.get("function")
|
|
.and_then(|f| f.get("arguments"))
|
|
.and_then(|a| a.as_str())
|
|
.expect("arguments stringified");
|
|
let parsed: Value = serde_json::from_str(args).unwrap();
|
|
assert_eq!(parsed["query"], "hello");
|
|
assert_eq!(parsed["limit"], 5);
|
|
}
|
|
|
|
#[test]
|
|
fn tool_call_arguments_parsed_on_receive() {
|
|
let response_msg = json!({
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [{
|
|
"id": "call_xyz",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "get_weather",
|
|
"arguments": "{\"city\":\"Boston\",\"units\":\"celsius\"}"
|
|
}
|
|
}]
|
|
});
|
|
|
|
let parsed = OpenRouterClient::openai_message_to_chat(&response_msg).unwrap();
|
|
let tcs = parsed.tool_calls.unwrap();
|
|
assert_eq!(tcs.len(), 1);
|
|
assert_eq!(tcs[0].function.name, "get_weather");
|
|
assert_eq!(tcs[0].function.arguments["city"], "Boston");
|
|
assert_eq!(tcs[0].function.arguments["units"], "celsius");
|
|
assert_eq!(tcs[0].id.as_deref(), Some("call_xyz"));
|
|
}
|
|
|
|
#[test]
|
|
fn tool_call_arguments_accept_native_json_on_receive() {
|
|
// Some providers return arguments as an object directly; accept both.
|
|
let response_msg = json!({
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [{
|
|
"id": "call_1",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "foo",
|
|
"arguments": {"nested": {"k": 1}}
|
|
}
|
|
}]
|
|
});
|
|
let parsed = OpenRouterClient::openai_message_to_chat(&response_msg).unwrap();
|
|
let tc = &parsed.tool_calls.unwrap()[0];
|
|
assert_eq!(tc.function.arguments["nested"]["k"], 1);
|
|
}
|
|
|
|
#[test]
|
|
fn any_image_anywhere_in_the_transcript_is_refused() {
|
|
// Not just the latest message: a replayed transcript whose first
|
|
// user turn still holds the photo must be refused too.
|
|
let mut first = ChatMessage::user("What is in this photo?");
|
|
first.images = Some(vec!["BASE64DATA".into()]);
|
|
let later = ChatMessage::user("and then?");
|
|
let err = OpenRouterClient::messages_to_openai(&[first, later]).unwrap_err();
|
|
assert!(err.to_string().contains("photos stay on the local network"));
|
|
}
|
|
|
|
#[test]
|
|
fn an_empty_image_list_is_not_an_image() {
|
|
let mut msg = ChatMessage::user("hi");
|
|
msg.images = Some(Vec::new());
|
|
assert!(OpenRouterClient::messages_to_openai(&[msg]).is_ok());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn describe_image_refuses_without_a_network_call() {
|
|
// Points at a closed port: if the guard ever stopped refusing, this
|
|
// would fail with a connection error instead of the guard's.
|
|
let client =
|
|
OpenRouterClient::new("key".into(), Some("http://127.0.0.1:9".into()), "m".into());
|
|
let err = client.describe_image("BASE64DATA").await.unwrap_err();
|
|
assert!(err.to_string().contains("photos stay on the local network"));
|
|
}
|
|
|
|
#[test]
|
|
fn text_only_message_stays_string() {
|
|
let msg = ChatMessage::user("hello");
|
|
let wire = OpenRouterClient::messages_to_openai(&[msg]).unwrap();
|
|
assert_eq!(wire[0]["content"], "hello");
|
|
assert!(wire[0]["content"].as_str().is_some());
|
|
}
|
|
|
|
#[test]
|
|
fn tool_result_inherits_tool_call_id_from_prior_assistant() {
|
|
let assistant = ChatMessage {
|
|
role: "assistant".into(),
|
|
content: String::new(),
|
|
tool_calls: Some(vec![ToolCall {
|
|
id: Some("call_42".into()),
|
|
function: ToolCallFunction {
|
|
name: "lookup".into(),
|
|
arguments: json!({}),
|
|
},
|
|
}]),
|
|
images: None,
|
|
};
|
|
let tool_result = ChatMessage::tool_result("found it");
|
|
|
|
let wire = OpenRouterClient::messages_to_openai(&[assistant, tool_result]).unwrap();
|
|
assert_eq!(wire[1]["role"], "tool");
|
|
assert_eq!(wire[1]["tool_call_id"], "call_42");
|
|
}
|
|
|
|
#[test]
|
|
fn multiple_tool_results_map_to_sequential_call_ids() {
|
|
let assistant = ChatMessage {
|
|
role: "assistant".into(),
|
|
content: String::new(),
|
|
tool_calls: Some(vec![
|
|
ToolCall {
|
|
id: Some("call_A".into()),
|
|
function: ToolCallFunction {
|
|
name: "a".into(),
|
|
arguments: json!({}),
|
|
},
|
|
},
|
|
ToolCall {
|
|
id: Some("call_B".into()),
|
|
function: ToolCallFunction {
|
|
name: "b".into(),
|
|
arguments: json!({}),
|
|
},
|
|
},
|
|
]),
|
|
images: None,
|
|
};
|
|
let r1 = ChatMessage::tool_result("a result");
|
|
let r2 = ChatMessage::tool_result("b result");
|
|
|
|
let wire = OpenRouterClient::messages_to_openai(&[assistant, r1, r2]).unwrap();
|
|
assert_eq!(wire[1]["tool_call_id"], "call_A");
|
|
assert_eq!(wire[2]["tool_call_id"], "call_B");
|
|
}
|
|
|
|
#[test]
|
|
fn missing_tool_call_id_gets_synthetic_fallback() {
|
|
let assistant = ChatMessage {
|
|
role: "assistant".into(),
|
|
content: String::new(),
|
|
tool_calls: Some(vec![ToolCall {
|
|
id: None,
|
|
function: ToolCallFunction {
|
|
name: "noid".into(),
|
|
arguments: json!({}),
|
|
},
|
|
}]),
|
|
images: None,
|
|
};
|
|
let wire = OpenRouterClient::messages_to_openai(&[assistant]).unwrap();
|
|
let tcs = wire[0]
|
|
.get("tool_calls")
|
|
.and_then(|v| v.as_array())
|
|
.unwrap();
|
|
assert_eq!(tcs[0]["id"], "call_0");
|
|
}
|
|
|
|
#[test]
|
|
fn parse_model_capabilities_extracts_tools_and_vision() {
|
|
let m = json!({
|
|
"id": "anthropic/claude-sonnet-4",
|
|
"supported_parameters": ["temperature", "top_p", "tools", "max_tokens"],
|
|
"architecture": {
|
|
"input_modalities": ["text", "image"]
|
|
}
|
|
});
|
|
let caps = parse_model_capabilities(&m);
|
|
assert_eq!(caps.name, "anthropic/claude-sonnet-4");
|
|
assert!(caps.has_tool_calling);
|
|
assert!(caps.has_vision);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_model_capabilities_handles_missing_fields() {
|
|
let m = json!({
|
|
"id": "some/text-only-model"
|
|
});
|
|
let caps = parse_model_capabilities(&m);
|
|
assert_eq!(caps.name, "some/text-only-model");
|
|
assert!(!caps.has_tool_calling);
|
|
assert!(!caps.has_vision);
|
|
}
|
|
}
|