Apps/ImageApi
1
0 Fork 0
Code Issues 8 Pull Requests Releases Wiki Activity

fix(openrouter): never send image data to OpenRouter

Browse Source 
Nothing in the client stopped it: a message carrying images became image_url
parts and went out. Photos stayed local only because callers routed around it.
messages_to_openai, which every OpenRouter request passes through, now refuses
any image (IMAGES_STAY_LOCAL), and describe_image always errors; hybrid mode
describes photos with the local model. A caller bug now fails the turn instead
of uploading a photo.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Cameron CordesandClaude Opus 5.5 committed 2026-10-18 21:00:05 -04:00
1 parent edfa48d116
commit cc11884272
1 file changed
+59 -82
Download Patch File Download Diff File Expand all files Collapse all files
src/ai/openrouter.rs
+59 -82
View File
@@ -48,6 +48,10 @@ impl<T> CachedEntry<T> {
}
}
/// The error every image-carrying OpenRouter request fails with.
pub const IMAGES_STAY_LOCAL: &str = "refusing to send image data to OpenRouter: photos stay on \
the local network (describe them with the local vision model and send the text)";
lazy_static::lazy_static! {
static ref MODEL_CAPABILITIES_CACHE: Arc<Mutex<HashMap<String, CachedEntry<Vec<ModelCapabilities>>>>> =
Arc::new(Mutex::new(HashMap::new()));
@@ -58,7 +62,11 @@ lazy_static::lazy_static! {
/// Translates canonical `ChatMessage` / `Tool` shapes to OpenAI wire format:
/// - Tool-call `arguments` serialized as JSON-encoded strings (vs Ollama's
/// native JSON).
/// - Image content rewritten into content-parts array with `image_url` entries.
/// - **Never sends images.** Photos stay on the local network: hybrid mode
/// describes them with the local vision model and sends only the text.
/// A message that still carries image data is refused (see
/// [`IMAGES_STAY_LOCAL`]) rather than uploaded, so a caller bug fails a
/// turn instead of leaking a photo.
/// - `role=tool` messages attach a `tool_call_id` inferred from the preceding
/// assistant turn's tool call.
#[derive(Clone)]
@@ -160,7 +168,18 @@ impl OpenRouterClient {
///
/// Walks in order so it can attach `tool_call_id` to `role=tool` messages
/// based on the most recent assistant turn's tool call.
fn messages_to_openai(messages: &[ChatMessage]) -> Vec<Value> {
///
/// Errors if any message carries image data: this is the one choke point
/// every OpenRouter request passes through, so the "photos never leave
/// the network" rule is enforced here rather than trusted to callers.
fn messages_to_openai(messages: &[ChatMessage]) -> Result<Vec<Value>> {
if messages
.iter()
.any(|m| m.images.as_ref().is_some_and(|imgs| !imgs.is_empty()))
{
log::error!("{IMAGES_STAY_LOCAL}");
bail!(IMAGES_STAY_LOCAL);
}
let mut out = Vec::with_capacity(messages.len());
let mut last_tool_call_ids: Vec<String> = Vec::new();
let mut next_tool_result_idx: usize = 0;
@@ -169,26 +188,8 @@ impl OpenRouterClient {
let mut obj = serde_json::Map::new();
obj.insert("role".into(), Value::String(msg.role.clone()));
// Content: string OR content-parts array (when images present).
match &msg.images {
Some(images) if !images.is_empty() => {
let mut parts: Vec<Value> = Vec::new();
if !msg.content.is_empty() {
parts.push(json!({"type": "text", "text": msg.content}));
}
for img in images {
let url = image_to_data_url(img);
parts.push(json!({
"type": "image_url",
"image_url": { "url": url }
}));
}
obj.insert("content".into(), Value::Array(parts));
}
_ => {
obj.insert("content".into(), Value::String(msg.content.clone()));
}
}
// Text only — images were refused above.
obj.insert("content".into(), Value::String(msg.content.clone()));
// Assistant message with tool_calls: stringify arguments, remember
// the ids so the subsequent tool messages can reference them.
@@ -233,7 +234,7 @@ impl OpenRouterClient {
out.push(Value::Object(obj));
}
out
Ok(out)
}
/// Parse an OpenAI-compatible assistant message back into canonical shape.
@@ -367,7 +368,7 @@ impl LlmClient for OpenRouterClient {
body.insert("model".into(), Value::String(self.primary_model.clone()));
body.insert(
"messages".into(),
Value::Array(Self::messages_to_openai(&messages)),
Value::Array(Self::messages_to_openai(&messages)?),
);
body.insert("stream".into(), Value::Bool(true));
// Ask for usage data in the final chunk (OpenAI + OpenRouter
@@ -633,24 +634,12 @@ impl LlmClient for OpenRouterClient {
Ok(parsed.data.into_iter().map(|i| i.embedding).collect())
}
async fn describe_image(&self, image_base64: &str) -> Result<String> {
// Own span with operation=describe_image — the funnel into
// chat_with_tools_impl keeps the request on a single span.
let span = self.start_request_span("describe_image", &self.primary_model);
let prompt = "Briefly describe what you see in this image in 1-2 sentences. \
Focus on the people, location, and activity.";
let mut messages: Vec<ChatMessage> = Vec::new();
messages.push(ChatMessage::system(
"You are a scene description assistant. Be concise and factual.",
));
let mut user = ChatMessage::user(prompt);
user.images = Some(vec![image_base64.to_string()]);
messages.push(user);
let (reply, _, _) = self
.chat_with_tools_impl(Some(&span), messages, Vec::new())
.await?;
Ok(reply.content)
async fn describe_image(&self, _image_base64: &str) -> Result<String> {
// Describing an image means uploading it. Hybrid mode always
// describes on the local backend (`ResolvedBackend::local`), so this
// is never the intended path — refuse loudly if something reaches it.
log::error!("{IMAGES_STAY_LOCAL}");
bail!(IMAGES_STAY_LOCAL)
}
async fn list_models(&self) -> Result<Vec<ModelCapabilities>> {
@@ -720,7 +709,7 @@ impl OpenRouterClient {
body.insert("model".into(), Value::String(self.primary_model.clone()));
body.insert(
"messages".into(),
Value::Array(Self::messages_to_openai(&messages)),
Value::Array(Self::messages_to_openai(&messages)?),
);
body.insert("stream".into(), Value::Bool(false));
if !tools.is_empty() {
@@ -871,15 +860,6 @@ fn find_double_newline(buf: &[u8]) -> Option<usize> {
None
}
/// Build a `data:` URL if the provided string is raw base64, otherwise pass it through.
fn image_to_data_url(img: &str) -> String {
if img.starts_with("data:") {
img.to_string()
} else {
format!("data:image/jpeg;base64,{}", img)
}
}
fn parse_model_capabilities(m: &Value) -> ModelCapabilities {
let name = m
.get("id")
@@ -926,7 +906,7 @@ mod tests {
msg.tool_calls.as_mut().unwrap()[0].function.arguments =
json!({"query": "hello", "limit": 5});
let wire = OpenRouterClient::messages_to_openai(&[msg]);
let wire = OpenRouterClient::messages_to_openai(&[msg]).unwrap();
let tcs = wire[0]
.get("tool_calls")
.and_then(|v| v.as_array())
@@ -986,40 +966,37 @@ mod tests {
}
#[test]
fn images_become_content_parts() {
let mut msg = ChatMessage::user("What is in this photo?");
msg.images = Some(vec!["BASE64DATA".into()]);
let wire = OpenRouterClient::messages_to_openai(&[msg]);
let content = wire[0].get("content").and_then(|v| v.as_array()).unwrap();
assert_eq!(content.len(), 2);
assert_eq!(content[0]["type"], "text");
assert_eq!(content[0]["text"], "What is in this photo?");
assert_eq!(content[1]["type"], "image_url");
assert_eq!(
content[1]["image_url"]["url"],
"data:image/jpeg;base64,BASE64DATA"
);
fn any_image_anywhere_in_the_transcript_is_refused() {
// Not just the latest message: a replayed transcript whose first
// user turn still holds the photo must be refused too.
let mut first = ChatMessage::user("What is in this photo?");
first.images = Some(vec!["BASE64DATA".into()]);
let later = ChatMessage::user("and then?");
let err = OpenRouterClient::messages_to_openai(&[first, later]).unwrap_err();
assert!(err.to_string().contains("photos stay on the local network"));
}
#[test]
fn data_url_images_pass_through_unchanged() {
let mut msg = ChatMessage::user("");
msg.images = Some(vec!["data:image/png;base64,ABCDEF".into()]);
let wire = OpenRouterClient::messages_to_openai(&[msg]);
let content = wire[0].get("content").and_then(|v| v.as_array()).unwrap();
// No text part when content is empty.
assert_eq!(content.len(), 1);
assert_eq!(
content[0]["image_url"]["url"],
"data:image/png;base64,ABCDEF"
);
fn an_empty_image_list_is_not_an_image() {
let mut msg = ChatMessage::user("hi");
msg.images = Some(Vec::new());
assert!(OpenRouterClient::messages_to_openai(&[msg]).is_ok());
}
#[tokio::test]
async fn describe_image_refuses_without_a_network_call() {
// Points at a closed port: if the guard ever stopped refusing, this
// would fail with a connection error instead of the guard's.
let client =
OpenRouterClient::new("key".into(), Some("http://127.0.0.1:9".into()), "m".into());
let err = client.describe_image("BASE64DATA").await.unwrap_err();
assert!(err.to_string().contains("photos stay on the local network"));
}
#[test]
fn text_only_message_stays_string() {
let msg = ChatMessage::user("hello");
let wire = OpenRouterClient::messages_to_openai(&[msg]);
let wire = OpenRouterClient::messages_to_openai(&[msg]).unwrap();
assert_eq!(wire[0]["content"], "hello");
assert!(wire[0]["content"].as_str().is_some());
}
@@ -1040,7 +1017,7 @@ mod tests {
};
let tool_result = ChatMessage::tool_result("found it");
let wire = OpenRouterClient::messages_to_openai(&[assistant, tool_result]);
let wire = OpenRouterClient::messages_to_openai(&[assistant, tool_result]).unwrap();
assert_eq!(wire[1]["role"], "tool");
assert_eq!(wire[1]["tool_call_id"], "call_42");
}
@@ -1071,7 +1048,7 @@ mod tests {
let r1 = ChatMessage::tool_result("a result");
let r2 = ChatMessage::tool_result("b result");
let wire = OpenRouterClient::messages_to_openai(&[assistant, r1, r2]);
let wire = OpenRouterClient::messages_to_openai(&[assistant, r1, r2]).unwrap();
assert_eq!(wire[1]["tool_call_id"], "call_A");
assert_eq!(wire[2]["tool_call_id"], "call_B");
}
@@ -1090,7 +1067,7 @@ mod tests {
}]),
images: None,
};
let wire = OpenRouterClient::messages_to_openai(&[assistant]);
let wire = OpenRouterClient::messages_to_openai(&[assistant]).unwrap();
let tcs = wire[0]
.get("tool_calls")
.and_then(|v| v.as_array())