fix(openrouter): never send image data to OpenRouter
Nothing in the client stopped it: a message carrying images became image_url parts and went out. Photos stayed local only because callers routed around it. messages_to_openai, which every OpenRouter request passes through, now refuses any image (IMAGES_STAY_LOCAL), and describe_image always errors; hybrid mode describes photos with the local model. A caller bug now fails the turn instead of uploading a photo. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
edfa48d116
commit
cc11884272
1 file changed
+59
-82
+59
-82
@@ -48,6 +48,10 @@ impl<T> CachedEntry<T> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The error every image-carrying OpenRouter request fails with.
|
||||||
|
pub const IMAGES_STAY_LOCAL: &str = "refusing to send image data to OpenRouter: photos stay on \
|
||||||
|
the local network (describe them with the local vision model and send the text)";
|
||||||
|
|
||||||
lazy_static::lazy_static! {
|
lazy_static::lazy_static! {
|
||||||
static ref MODEL_CAPABILITIES_CACHE: Arc<Mutex<HashMap<String, CachedEntry<Vec<ModelCapabilities>>>>> =
|
static ref MODEL_CAPABILITIES_CACHE: Arc<Mutex<HashMap<String, CachedEntry<Vec<ModelCapabilities>>>>> =
|
||||||
Arc::new(Mutex::new(HashMap::new()));
|
Arc::new(Mutex::new(HashMap::new()));
|
||||||
@@ -58,7 +62,11 @@ lazy_static::lazy_static! {
|
|||||||
/// Translates canonical `ChatMessage` / `Tool` shapes to OpenAI wire format:
|
/// Translates canonical `ChatMessage` / `Tool` shapes to OpenAI wire format:
|
||||||
/// - Tool-call `arguments` serialized as JSON-encoded strings (vs Ollama's
|
/// - Tool-call `arguments` serialized as JSON-encoded strings (vs Ollama's
|
||||||
/// native JSON).
|
/// native JSON).
|
||||||
/// - Image content rewritten into content-parts array with `image_url` entries.
|
/// - **Never sends images.** Photos stay on the local network: hybrid mode
|
||||||
|
/// describes them with the local vision model and sends only the text.
|
||||||
|
/// A message that still carries image data is refused (see
|
||||||
|
/// [`IMAGES_STAY_LOCAL`]) rather than uploaded, so a caller bug fails a
|
||||||
|
/// turn instead of leaking a photo.
|
||||||
/// - `role=tool` messages attach a `tool_call_id` inferred from the preceding
|
/// - `role=tool` messages attach a `tool_call_id` inferred from the preceding
|
||||||
/// assistant turn's tool call.
|
/// assistant turn's tool call.
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
@@ -160,7 +168,18 @@ impl OpenRouterClient {
|
|||||||
///
|
///
|
||||||
/// Walks in order so it can attach `tool_call_id` to `role=tool` messages
|
/// Walks in order so it can attach `tool_call_id` to `role=tool` messages
|
||||||
/// based on the most recent assistant turn's tool call.
|
/// based on the most recent assistant turn's tool call.
|
||||||
fn messages_to_openai(messages: &[ChatMessage]) -> Vec<Value> {
|
///
|
||||||
|
/// Errors if any message carries image data: this is the one choke point
|
||||||
|
/// every OpenRouter request passes through, so the "photos never leave
|
||||||
|
/// the network" rule is enforced here rather than trusted to callers.
|
||||||
|
fn messages_to_openai(messages: &[ChatMessage]) -> Result<Vec<Value>> {
|
||||||
|
if messages
|
||||||
|
.iter()
|
||||||
|
.any(|m| m.images.as_ref().is_some_and(|imgs| !imgs.is_empty()))
|
||||||
|
{
|
||||||
|
log::error!("{IMAGES_STAY_LOCAL}");
|
||||||
|
bail!(IMAGES_STAY_LOCAL);
|
||||||
|
}
|
||||||
let mut out = Vec::with_capacity(messages.len());
|
let mut out = Vec::with_capacity(messages.len());
|
||||||
let mut last_tool_call_ids: Vec<String> = Vec::new();
|
let mut last_tool_call_ids: Vec<String> = Vec::new();
|
||||||
let mut next_tool_result_idx: usize = 0;
|
let mut next_tool_result_idx: usize = 0;
|
||||||
@@ -169,26 +188,8 @@ impl OpenRouterClient {
|
|||||||
let mut obj = serde_json::Map::new();
|
let mut obj = serde_json::Map::new();
|
||||||
obj.insert("role".into(), Value::String(msg.role.clone()));
|
obj.insert("role".into(), Value::String(msg.role.clone()));
|
||||||
|
|
||||||
// Content: string OR content-parts array (when images present).
|
// Text only — images were refused above.
|
||||||
match &msg.images {
|
obj.insert("content".into(), Value::String(msg.content.clone()));
|
||||||
Some(images) if !images.is_empty() => {
|
|
||||||
let mut parts: Vec<Value> = Vec::new();
|
|
||||||
if !msg.content.is_empty() {
|
|
||||||
parts.push(json!({"type": "text", "text": msg.content}));
|
|
||||||
}
|
|
||||||
for img in images {
|
|
||||||
let url = image_to_data_url(img);
|
|
||||||
parts.push(json!({
|
|
||||||
"type": "image_url",
|
|
||||||
"image_url": { "url": url }
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
obj.insert("content".into(), Value::Array(parts));
|
|
||||||
}
|
|
||||||
_ => {
|
|
||||||
obj.insert("content".into(), Value::String(msg.content.clone()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Assistant message with tool_calls: stringify arguments, remember
|
// Assistant message with tool_calls: stringify arguments, remember
|
||||||
// the ids so the subsequent tool messages can reference them.
|
// the ids so the subsequent tool messages can reference them.
|
||||||
@@ -233,7 +234,7 @@ impl OpenRouterClient {
|
|||||||
out.push(Value::Object(obj));
|
out.push(Value::Object(obj));
|
||||||
}
|
}
|
||||||
|
|
||||||
out
|
Ok(out)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Parse an OpenAI-compatible assistant message back into canonical shape.
|
/// Parse an OpenAI-compatible assistant message back into canonical shape.
|
||||||
@@ -367,7 +368,7 @@ impl LlmClient for OpenRouterClient {
|
|||||||
body.insert("model".into(), Value::String(self.primary_model.clone()));
|
body.insert("model".into(), Value::String(self.primary_model.clone()));
|
||||||
body.insert(
|
body.insert(
|
||||||
"messages".into(),
|
"messages".into(),
|
||||||
Value::Array(Self::messages_to_openai(&messages)),
|
Value::Array(Self::messages_to_openai(&messages)?),
|
||||||
);
|
);
|
||||||
body.insert("stream".into(), Value::Bool(true));
|
body.insert("stream".into(), Value::Bool(true));
|
||||||
// Ask for usage data in the final chunk (OpenAI + OpenRouter
|
// Ask for usage data in the final chunk (OpenAI + OpenRouter
|
||||||
@@ -633,24 +634,12 @@ impl LlmClient for OpenRouterClient {
|
|||||||
Ok(parsed.data.into_iter().map(|i| i.embedding).collect())
|
Ok(parsed.data.into_iter().map(|i| i.embedding).collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn describe_image(&self, image_base64: &str) -> Result<String> {
|
async fn describe_image(&self, _image_base64: &str) -> Result<String> {
|
||||||
// Own span with operation=describe_image — the funnel into
|
// Describing an image means uploading it. Hybrid mode always
|
||||||
// chat_with_tools_impl keeps the request on a single span.
|
// describes on the local backend (`ResolvedBackend::local`), so this
|
||||||
let span = self.start_request_span("describe_image", &self.primary_model);
|
// is never the intended path — refuse loudly if something reaches it.
|
||||||
let prompt = "Briefly describe what you see in this image in 1-2 sentences. \
|
log::error!("{IMAGES_STAY_LOCAL}");
|
||||||
Focus on the people, location, and activity.";
|
bail!(IMAGES_STAY_LOCAL)
|
||||||
let mut messages: Vec<ChatMessage> = Vec::new();
|
|
||||||
messages.push(ChatMessage::system(
|
|
||||||
"You are a scene description assistant. Be concise and factual.",
|
|
||||||
));
|
|
||||||
let mut user = ChatMessage::user(prompt);
|
|
||||||
user.images = Some(vec![image_base64.to_string()]);
|
|
||||||
messages.push(user);
|
|
||||||
|
|
||||||
let (reply, _, _) = self
|
|
||||||
.chat_with_tools_impl(Some(&span), messages, Vec::new())
|
|
||||||
.await?;
|
|
||||||
Ok(reply.content)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn list_models(&self) -> Result<Vec<ModelCapabilities>> {
|
async fn list_models(&self) -> Result<Vec<ModelCapabilities>> {
|
||||||
@@ -720,7 +709,7 @@ impl OpenRouterClient {
|
|||||||
body.insert("model".into(), Value::String(self.primary_model.clone()));
|
body.insert("model".into(), Value::String(self.primary_model.clone()));
|
||||||
body.insert(
|
body.insert(
|
||||||
"messages".into(),
|
"messages".into(),
|
||||||
Value::Array(Self::messages_to_openai(&messages)),
|
Value::Array(Self::messages_to_openai(&messages)?),
|
||||||
);
|
);
|
||||||
body.insert("stream".into(), Value::Bool(false));
|
body.insert("stream".into(), Value::Bool(false));
|
||||||
if !tools.is_empty() {
|
if !tools.is_empty() {
|
||||||
@@ -871,15 +860,6 @@ fn find_double_newline(buf: &[u8]) -> Option<usize> {
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build a `data:` URL if the provided string is raw base64, otherwise pass it through.
|
|
||||||
fn image_to_data_url(img: &str) -> String {
|
|
||||||
if img.starts_with("data:") {
|
|
||||||
img.to_string()
|
|
||||||
} else {
|
|
||||||
format!("data:image/jpeg;base64,{}", img)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn parse_model_capabilities(m: &Value) -> ModelCapabilities {
|
fn parse_model_capabilities(m: &Value) -> ModelCapabilities {
|
||||||
let name = m
|
let name = m
|
||||||
.get("id")
|
.get("id")
|
||||||
@@ -926,7 +906,7 @@ mod tests {
|
|||||||
msg.tool_calls.as_mut().unwrap()[0].function.arguments =
|
msg.tool_calls.as_mut().unwrap()[0].function.arguments =
|
||||||
json!({"query": "hello", "limit": 5});
|
json!({"query": "hello", "limit": 5});
|
||||||
|
|
||||||
let wire = OpenRouterClient::messages_to_openai(&[msg]);
|
let wire = OpenRouterClient::messages_to_openai(&[msg]).unwrap();
|
||||||
let tcs = wire[0]
|
let tcs = wire[0]
|
||||||
.get("tool_calls")
|
.get("tool_calls")
|
||||||
.and_then(|v| v.as_array())
|
.and_then(|v| v.as_array())
|
||||||
@@ -986,40 +966,37 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn images_become_content_parts() {
|
fn any_image_anywhere_in_the_transcript_is_refused() {
|
||||||
let mut msg = ChatMessage::user("What is in this photo?");
|
// Not just the latest message: a replayed transcript whose first
|
||||||
msg.images = Some(vec!["BASE64DATA".into()]);
|
// user turn still holds the photo must be refused too.
|
||||||
|
let mut first = ChatMessage::user("What is in this photo?");
|
||||||
let wire = OpenRouterClient::messages_to_openai(&[msg]);
|
first.images = Some(vec!["BASE64DATA".into()]);
|
||||||
let content = wire[0].get("content").and_then(|v| v.as_array()).unwrap();
|
let later = ChatMessage::user("and then?");
|
||||||
assert_eq!(content.len(), 2);
|
let err = OpenRouterClient::messages_to_openai(&[first, later]).unwrap_err();
|
||||||
assert_eq!(content[0]["type"], "text");
|
assert!(err.to_string().contains("photos stay on the local network"));
|
||||||
assert_eq!(content[0]["text"], "What is in this photo?");
|
|
||||||
assert_eq!(content[1]["type"], "image_url");
|
|
||||||
assert_eq!(
|
|
||||||
content[1]["image_url"]["url"],
|
|
||||||
"data:image/jpeg;base64,BASE64DATA"
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn data_url_images_pass_through_unchanged() {
|
fn an_empty_image_list_is_not_an_image() {
|
||||||
let mut msg = ChatMessage::user("");
|
let mut msg = ChatMessage::user("hi");
|
||||||
msg.images = Some(vec!["data:image/png;base64,ABCDEF".into()]);
|
msg.images = Some(Vec::new());
|
||||||
let wire = OpenRouterClient::messages_to_openai(&[msg]);
|
assert!(OpenRouterClient::messages_to_openai(&[msg]).is_ok());
|
||||||
let content = wire[0].get("content").and_then(|v| v.as_array()).unwrap();
|
}
|
||||||
// No text part when content is empty.
|
|
||||||
assert_eq!(content.len(), 1);
|
#[tokio::test]
|
||||||
assert_eq!(
|
async fn describe_image_refuses_without_a_network_call() {
|
||||||
content[0]["image_url"]["url"],
|
// Points at a closed port: if the guard ever stopped refusing, this
|
||||||
"data:image/png;base64,ABCDEF"
|
// would fail with a connection error instead of the guard's.
|
||||||
);
|
let client =
|
||||||
|
OpenRouterClient::new("key".into(), Some("http://127.0.0.1:9".into()), "m".into());
|
||||||
|
let err = client.describe_image("BASE64DATA").await.unwrap_err();
|
||||||
|
assert!(err.to_string().contains("photos stay on the local network"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn text_only_message_stays_string() {
|
fn text_only_message_stays_string() {
|
||||||
let msg = ChatMessage::user("hello");
|
let msg = ChatMessage::user("hello");
|
||||||
let wire = OpenRouterClient::messages_to_openai(&[msg]);
|
let wire = OpenRouterClient::messages_to_openai(&[msg]).unwrap();
|
||||||
assert_eq!(wire[0]["content"], "hello");
|
assert_eq!(wire[0]["content"], "hello");
|
||||||
assert!(wire[0]["content"].as_str().is_some());
|
assert!(wire[0]["content"].as_str().is_some());
|
||||||
}
|
}
|
||||||
@@ -1040,7 +1017,7 @@ mod tests {
|
|||||||
};
|
};
|
||||||
let tool_result = ChatMessage::tool_result("found it");
|
let tool_result = ChatMessage::tool_result("found it");
|
||||||
|
|
||||||
let wire = OpenRouterClient::messages_to_openai(&[assistant, tool_result]);
|
let wire = OpenRouterClient::messages_to_openai(&[assistant, tool_result]).unwrap();
|
||||||
assert_eq!(wire[1]["role"], "tool");
|
assert_eq!(wire[1]["role"], "tool");
|
||||||
assert_eq!(wire[1]["tool_call_id"], "call_42");
|
assert_eq!(wire[1]["tool_call_id"], "call_42");
|
||||||
}
|
}
|
||||||
@@ -1071,7 +1048,7 @@ mod tests {
|
|||||||
let r1 = ChatMessage::tool_result("a result");
|
let r1 = ChatMessage::tool_result("a result");
|
||||||
let r2 = ChatMessage::tool_result("b result");
|
let r2 = ChatMessage::tool_result("b result");
|
||||||
|
|
||||||
let wire = OpenRouterClient::messages_to_openai(&[assistant, r1, r2]);
|
let wire = OpenRouterClient::messages_to_openai(&[assistant, r1, r2]).unwrap();
|
||||||
assert_eq!(wire[1]["tool_call_id"], "call_A");
|
assert_eq!(wire[1]["tool_call_id"], "call_A");
|
||||||
assert_eq!(wire[2]["tool_call_id"], "call_B");
|
assert_eq!(wire[2]["tool_call_id"], "call_B");
|
||||||
}
|
}
|
||||||
@@ -1090,7 +1067,7 @@ mod tests {
|
|||||||
}]),
|
}]),
|
||||||
images: None,
|
images: None,
|
||||||
};
|
};
|
||||||
let wire = OpenRouterClient::messages_to_openai(&[assistant]);
|
let wire = OpenRouterClient::messages_to_openai(&[assistant]).unwrap();
|
||||||
let tcs = wire[0]
|
let tcs = wire[0]
|
||||||
.get("tool_calls")
|
.get("tool_calls")
|
||||||
.and_then(|v| v.as_array())
|
.and_then(|v| v.as_array())
|
||||||
|
|||||||
Reference in new issue
Block a user