fix(openrouter): never send image data to OpenRouter

Nothing in the client stopped it: a message carrying images became image_url
parts and went out. Photos stayed local only because callers routed around it.
messages_to_openai, which every OpenRouter request passes through, now refuses
any image (IMAGES_STAY_LOCAL), and describe_image always errors; hybrid mode
describes photos with the local model. A caller bug now fails the turn instead
of uploading a photo.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Cameron CordesandClaude Opus 5.5 committed 2026-10-18 21:00:05 -04:00
1 parent edfa48d116
commit cc11884272
1 file changed
+59 -82
+59 -82
View File
@@ -48,6 +48,10 @@ impl<T> CachedEntry<T> {
} }
} }
/// The error every image-carrying OpenRouter request fails with.
pub const IMAGES_STAY_LOCAL: &str = "refusing to send image data to OpenRouter: photos stay on \
the local network (describe them with the local vision model and send the text)";
lazy_static::lazy_static! { lazy_static::lazy_static! {
static ref MODEL_CAPABILITIES_CACHE: Arc<Mutex<HashMap<String, CachedEntry<Vec<ModelCapabilities>>>>> = static ref MODEL_CAPABILITIES_CACHE: Arc<Mutex<HashMap<String, CachedEntry<Vec<ModelCapabilities>>>>> =
Arc::new(Mutex::new(HashMap::new())); Arc::new(Mutex::new(HashMap::new()));
@@ -58,7 +62,11 @@ lazy_static::lazy_static! {
/// Translates canonical `ChatMessage` / `Tool` shapes to OpenAI wire format: /// Translates canonical `ChatMessage` / `Tool` shapes to OpenAI wire format:
/// - Tool-call `arguments` serialized as JSON-encoded strings (vs Ollama's /// - Tool-call `arguments` serialized as JSON-encoded strings (vs Ollama's
/// native JSON). /// native JSON).
/// - Image content rewritten into content-parts array with `image_url` entries. /// - **Never sends images.** Photos stay on the local network: hybrid mode
/// describes them with the local vision model and sends only the text.
/// A message that still carries image data is refused (see
/// [`IMAGES_STAY_LOCAL`]) rather than uploaded, so a caller bug fails a
/// turn instead of leaking a photo.
/// - `role=tool` messages attach a `tool_call_id` inferred from the preceding /// - `role=tool` messages attach a `tool_call_id` inferred from the preceding
/// assistant turn's tool call. /// assistant turn's tool call.
#[derive(Clone)] #[derive(Clone)]
@@ -160,7 +168,18 @@ impl OpenRouterClient {
/// ///
/// Walks in order so it can attach `tool_call_id` to `role=tool` messages /// Walks in order so it can attach `tool_call_id` to `role=tool` messages
/// based on the most recent assistant turn's tool call. /// based on the most recent assistant turn's tool call.
fn messages_to_openai(messages: &[ChatMessage]) -> Vec<Value> { ///
/// Errors if any message carries image data: this is the one choke point
/// every OpenRouter request passes through, so the "photos never leave
/// the network" rule is enforced here rather than trusted to callers.
fn messages_to_openai(messages: &[ChatMessage]) -> Result<Vec<Value>> {
if messages
.iter()
.any(|m| m.images.as_ref().is_some_and(|imgs| !imgs.is_empty()))
{
log::error!("{IMAGES_STAY_LOCAL}");
bail!(IMAGES_STAY_LOCAL);
}
let mut out = Vec::with_capacity(messages.len()); let mut out = Vec::with_capacity(messages.len());
let mut last_tool_call_ids: Vec<String> = Vec::new(); let mut last_tool_call_ids: Vec<String> = Vec::new();
let mut next_tool_result_idx: usize = 0; let mut next_tool_result_idx: usize = 0;
@@ -169,26 +188,8 @@ impl OpenRouterClient {
let mut obj = serde_json::Map::new(); let mut obj = serde_json::Map::new();
obj.insert("role".into(), Value::String(msg.role.clone())); obj.insert("role".into(), Value::String(msg.role.clone()));
// Content: string OR content-parts array (when images present). // Text only — images were refused above.
match &msg.images { obj.insert("content".into(), Value::String(msg.content.clone()));
Some(images) if !images.is_empty() => {
let mut parts: Vec<Value> = Vec::new();
if !msg.content.is_empty() {
parts.push(json!({"type": "text", "text": msg.content}));
}
for img in images {
let url = image_to_data_url(img);
parts.push(json!({
"type": "image_url",
"image_url": { "url": url }
}));
}
obj.insert("content".into(), Value::Array(parts));
}
_ => {
obj.insert("content".into(), Value::String(msg.content.clone()));
}
}
// Assistant message with tool_calls: stringify arguments, remember // Assistant message with tool_calls: stringify arguments, remember
// the ids so the subsequent tool messages can reference them. // the ids so the subsequent tool messages can reference them.
@@ -233,7 +234,7 @@ impl OpenRouterClient {
out.push(Value::Object(obj)); out.push(Value::Object(obj));
} }
out Ok(out)
} }
/// Parse an OpenAI-compatible assistant message back into canonical shape. /// Parse an OpenAI-compatible assistant message back into canonical shape.
@@ -367,7 +368,7 @@ impl LlmClient for OpenRouterClient {
body.insert("model".into(), Value::String(self.primary_model.clone())); body.insert("model".into(), Value::String(self.primary_model.clone()));
body.insert( body.insert(
"messages".into(), "messages".into(),
Value::Array(Self::messages_to_openai(&messages)), Value::Array(Self::messages_to_openai(&messages)?),
); );
body.insert("stream".into(), Value::Bool(true)); body.insert("stream".into(), Value::Bool(true));
// Ask for usage data in the final chunk (OpenAI + OpenRouter // Ask for usage data in the final chunk (OpenAI + OpenRouter
@@ -633,24 +634,12 @@ impl LlmClient for OpenRouterClient {
Ok(parsed.data.into_iter().map(|i| i.embedding).collect()) Ok(parsed.data.into_iter().map(|i| i.embedding).collect())
} }
async fn describe_image(&self, image_base64: &str) -> Result<String> { async fn describe_image(&self, _image_base64: &str) -> Result<String> {
// Own span with operation=describe_image — the funnel into // Describing an image means uploading it. Hybrid mode always
// chat_with_tools_impl keeps the request on a single span. // describes on the local backend (`ResolvedBackend::local`), so this
let span = self.start_request_span("describe_image", &self.primary_model); // is never the intended path — refuse loudly if something reaches it.
let prompt = "Briefly describe what you see in this image in 1-2 sentences. \ log::error!("{IMAGES_STAY_LOCAL}");
Focus on the people, location, and activity."; bail!(IMAGES_STAY_LOCAL)
let mut messages: Vec<ChatMessage> = Vec::new();
messages.push(ChatMessage::system(
"You are a scene description assistant. Be concise and factual.",
));
let mut user = ChatMessage::user(prompt);
user.images = Some(vec![image_base64.to_string()]);
messages.push(user);
let (reply, _, _) = self
.chat_with_tools_impl(Some(&span), messages, Vec::new())
.await?;
Ok(reply.content)
} }
async fn list_models(&self) -> Result<Vec<ModelCapabilities>> { async fn list_models(&self) -> Result<Vec<ModelCapabilities>> {
@@ -720,7 +709,7 @@ impl OpenRouterClient {
body.insert("model".into(), Value::String(self.primary_model.clone())); body.insert("model".into(), Value::String(self.primary_model.clone()));
body.insert( body.insert(
"messages".into(), "messages".into(),
Value::Array(Self::messages_to_openai(&messages)), Value::Array(Self::messages_to_openai(&messages)?),
); );
body.insert("stream".into(), Value::Bool(false)); body.insert("stream".into(), Value::Bool(false));
if !tools.is_empty() { if !tools.is_empty() {
@@ -871,15 +860,6 @@ fn find_double_newline(buf: &[u8]) -> Option<usize> {
None None
} }
/// Build a `data:` URL if the provided string is raw base64, otherwise pass it through.
fn image_to_data_url(img: &str) -> String {
if img.starts_with("data:") {
img.to_string()
} else {
format!("data:image/jpeg;base64,{}", img)
}
}
fn parse_model_capabilities(m: &Value) -> ModelCapabilities { fn parse_model_capabilities(m: &Value) -> ModelCapabilities {
let name = m let name = m
.get("id") .get("id")
@@ -926,7 +906,7 @@ mod tests {
msg.tool_calls.as_mut().unwrap()[0].function.arguments = msg.tool_calls.as_mut().unwrap()[0].function.arguments =
json!({"query": "hello", "limit": 5}); json!({"query": "hello", "limit": 5});
let wire = OpenRouterClient::messages_to_openai(&[msg]); let wire = OpenRouterClient::messages_to_openai(&[msg]).unwrap();
let tcs = wire[0] let tcs = wire[0]
.get("tool_calls") .get("tool_calls")
.and_then(|v| v.as_array()) .and_then(|v| v.as_array())
@@ -986,40 +966,37 @@ mod tests {
} }
#[test] #[test]
fn images_become_content_parts() { fn any_image_anywhere_in_the_transcript_is_refused() {
let mut msg = ChatMessage::user("What is in this photo?"); // Not just the latest message: a replayed transcript whose first
msg.images = Some(vec!["BASE64DATA".into()]); // user turn still holds the photo must be refused too.
let mut first = ChatMessage::user("What is in this photo?");
let wire = OpenRouterClient::messages_to_openai(&[msg]); first.images = Some(vec!["BASE64DATA".into()]);
let content = wire[0].get("content").and_then(|v| v.as_array()).unwrap(); let later = ChatMessage::user("and then?");
assert_eq!(content.len(), 2); let err = OpenRouterClient::messages_to_openai(&[first, later]).unwrap_err();
assert_eq!(content[0]["type"], "text"); assert!(err.to_string().contains("photos stay on the local network"));
assert_eq!(content[0]["text"], "What is in this photo?");
assert_eq!(content[1]["type"], "image_url");
assert_eq!(
content[1]["image_url"]["url"],
"data:image/jpeg;base64,BASE64DATA"
);
} }
#[test] #[test]
fn data_url_images_pass_through_unchanged() { fn an_empty_image_list_is_not_an_image() {
let mut msg = ChatMessage::user(""); let mut msg = ChatMessage::user("hi");
msg.images = Some(vec!["data:image/png;base64,ABCDEF".into()]); msg.images = Some(Vec::new());
let wire = OpenRouterClient::messages_to_openai(&[msg]); assert!(OpenRouterClient::messages_to_openai(&[msg]).is_ok());
let content = wire[0].get("content").and_then(|v| v.as_array()).unwrap(); }
// No text part when content is empty.
assert_eq!(content.len(), 1); #[tokio::test]
assert_eq!( async fn describe_image_refuses_without_a_network_call() {
content[0]["image_url"]["url"], // Points at a closed port: if the guard ever stopped refusing, this
"data:image/png;base64,ABCDEF" // would fail with a connection error instead of the guard's.
); let client =
OpenRouterClient::new("key".into(), Some("http://127.0.0.1:9".into()), "m".into());
let err = client.describe_image("BASE64DATA").await.unwrap_err();
assert!(err.to_string().contains("photos stay on the local network"));
} }
#[test] #[test]
fn text_only_message_stays_string() { fn text_only_message_stays_string() {
let msg = ChatMessage::user("hello"); let msg = ChatMessage::user("hello");
let wire = OpenRouterClient::messages_to_openai(&[msg]); let wire = OpenRouterClient::messages_to_openai(&[msg]).unwrap();
assert_eq!(wire[0]["content"], "hello"); assert_eq!(wire[0]["content"], "hello");
assert!(wire[0]["content"].as_str().is_some()); assert!(wire[0]["content"].as_str().is_some());
} }
@@ -1040,7 +1017,7 @@ mod tests {
}; };
let tool_result = ChatMessage::tool_result("found it"); let tool_result = ChatMessage::tool_result("found it");
let wire = OpenRouterClient::messages_to_openai(&[assistant, tool_result]); let wire = OpenRouterClient::messages_to_openai(&[assistant, tool_result]).unwrap();
assert_eq!(wire[1]["role"], "tool"); assert_eq!(wire[1]["role"], "tool");
assert_eq!(wire[1]["tool_call_id"], "call_42"); assert_eq!(wire[1]["tool_call_id"], "call_42");
} }
@@ -1071,7 +1048,7 @@ mod tests {
let r1 = ChatMessage::tool_result("a result"); let r1 = ChatMessage::tool_result("a result");
let r2 = ChatMessage::tool_result("b result"); let r2 = ChatMessage::tool_result("b result");
let wire = OpenRouterClient::messages_to_openai(&[assistant, r1, r2]); let wire = OpenRouterClient::messages_to_openai(&[assistant, r1, r2]).unwrap();
assert_eq!(wire[1]["tool_call_id"], "call_A"); assert_eq!(wire[1]["tool_call_id"], "call_A");
assert_eq!(wire[2]["tool_call_id"], "call_B"); assert_eq!(wire[2]["tool_call_id"], "call_B");
} }
@@ -1090,7 +1067,7 @@ mod tests {
}]), }]),
images: None, images: None,
}; };
let wire = OpenRouterClient::messages_to_openai(&[assistant]); let wire = OpenRouterClient::messages_to_openai(&[assistant]).unwrap();
let tcs = wire[0] let tcs = wire[0]
.get("tool_calls") .get("tool_calls")
.and_then(|v| v.as_array()) .and_then(|v| v.as_array())