diff --git a/src/ai/openrouter.rs b/src/ai/openrouter.rs index 3c450ee..ae0c544 100644 --- a/src/ai/openrouter.rs +++ b/src/ai/openrouter.rs @@ -48,6 +48,10 @@ impl CachedEntry { } } +/// The error every image-carrying OpenRouter request fails with. +pub const IMAGES_STAY_LOCAL: &str = "refusing to send image data to OpenRouter: photos stay on \ + the local network (describe them with the local vision model and send the text)"; + lazy_static::lazy_static! { static ref MODEL_CAPABILITIES_CACHE: Arc>>>> = Arc::new(Mutex::new(HashMap::new())); @@ -58,7 +62,11 @@ lazy_static::lazy_static! { /// Translates canonical `ChatMessage` / `Tool` shapes to OpenAI wire format: /// - Tool-call `arguments` serialized as JSON-encoded strings (vs Ollama's /// native JSON). -/// - Image content rewritten into content-parts array with `image_url` entries. +/// - **Never sends images.** Photos stay on the local network: hybrid mode +/// describes them with the local vision model and sends only the text. +/// A message that still carries image data is refused (see +/// [`IMAGES_STAY_LOCAL`]) rather than uploaded, so a caller bug fails a +/// turn instead of leaking a photo. /// - `role=tool` messages attach a `tool_call_id` inferred from the preceding /// assistant turn's tool call. #[derive(Clone)] @@ -160,7 +168,18 @@ impl OpenRouterClient { /// /// Walks in order so it can attach `tool_call_id` to `role=tool` messages /// based on the most recent assistant turn's tool call. - fn messages_to_openai(messages: &[ChatMessage]) -> Vec { + /// + /// Errors if any message carries image data: this is the one choke point + /// every OpenRouter request passes through, so the "photos never leave + /// the network" rule is enforced here rather than trusted to callers. + fn messages_to_openai(messages: &[ChatMessage]) -> Result> { + if messages + .iter() + .any(|m| m.images.as_ref().is_some_and(|imgs| !imgs.is_empty())) + { + log::error!("{IMAGES_STAY_LOCAL}"); + bail!(IMAGES_STAY_LOCAL); + } let mut out = Vec::with_capacity(messages.len()); let mut last_tool_call_ids: Vec = Vec::new(); let mut next_tool_result_idx: usize = 0; @@ -169,26 +188,8 @@ impl OpenRouterClient { let mut obj = serde_json::Map::new(); obj.insert("role".into(), Value::String(msg.role.clone())); - // Content: string OR content-parts array (when images present). - match &msg.images { - Some(images) if !images.is_empty() => { - let mut parts: Vec = Vec::new(); - if !msg.content.is_empty() { - parts.push(json!({"type": "text", "text": msg.content})); - } - for img in images { - let url = image_to_data_url(img); - parts.push(json!({ - "type": "image_url", - "image_url": { "url": url } - })); - } - obj.insert("content".into(), Value::Array(parts)); - } - _ => { - obj.insert("content".into(), Value::String(msg.content.clone())); - } - } + // Text only — images were refused above. + obj.insert("content".into(), Value::String(msg.content.clone())); // Assistant message with tool_calls: stringify arguments, remember // the ids so the subsequent tool messages can reference them. @@ -233,7 +234,7 @@ impl OpenRouterClient { out.push(Value::Object(obj)); } - out + Ok(out) } /// Parse an OpenAI-compatible assistant message back into canonical shape. @@ -367,7 +368,7 @@ impl LlmClient for OpenRouterClient { body.insert("model".into(), Value::String(self.primary_model.clone())); body.insert( "messages".into(), - Value::Array(Self::messages_to_openai(&messages)), + Value::Array(Self::messages_to_openai(&messages)?), ); body.insert("stream".into(), Value::Bool(true)); // Ask for usage data in the final chunk (OpenAI + OpenRouter @@ -633,24 +634,12 @@ impl LlmClient for OpenRouterClient { Ok(parsed.data.into_iter().map(|i| i.embedding).collect()) } - async fn describe_image(&self, image_base64: &str) -> Result { - // Own span with operation=describe_image — the funnel into - // chat_with_tools_impl keeps the request on a single span. - let span = self.start_request_span("describe_image", &self.primary_model); - let prompt = "Briefly describe what you see in this image in 1-2 sentences. \ - Focus on the people, location, and activity."; - let mut messages: Vec = Vec::new(); - messages.push(ChatMessage::system( - "You are a scene description assistant. Be concise and factual.", - )); - let mut user = ChatMessage::user(prompt); - user.images = Some(vec![image_base64.to_string()]); - messages.push(user); - - let (reply, _, _) = self - .chat_with_tools_impl(Some(&span), messages, Vec::new()) - .await?; - Ok(reply.content) + async fn describe_image(&self, _image_base64: &str) -> Result { + // Describing an image means uploading it. Hybrid mode always + // describes on the local backend (`ResolvedBackend::local`), so this + // is never the intended path — refuse loudly if something reaches it. + log::error!("{IMAGES_STAY_LOCAL}"); + bail!(IMAGES_STAY_LOCAL) } async fn list_models(&self) -> Result> { @@ -720,7 +709,7 @@ impl OpenRouterClient { body.insert("model".into(), Value::String(self.primary_model.clone())); body.insert( "messages".into(), - Value::Array(Self::messages_to_openai(&messages)), + Value::Array(Self::messages_to_openai(&messages)?), ); body.insert("stream".into(), Value::Bool(false)); if !tools.is_empty() { @@ -871,15 +860,6 @@ fn find_double_newline(buf: &[u8]) -> Option { None } -/// Build a `data:` URL if the provided string is raw base64, otherwise pass it through. -fn image_to_data_url(img: &str) -> String { - if img.starts_with("data:") { - img.to_string() - } else { - format!("data:image/jpeg;base64,{}", img) - } -} - fn parse_model_capabilities(m: &Value) -> ModelCapabilities { let name = m .get("id") @@ -926,7 +906,7 @@ mod tests { msg.tool_calls.as_mut().unwrap()[0].function.arguments = json!({"query": "hello", "limit": 5}); - let wire = OpenRouterClient::messages_to_openai(&[msg]); + let wire = OpenRouterClient::messages_to_openai(&[msg]).unwrap(); let tcs = wire[0] .get("tool_calls") .and_then(|v| v.as_array()) @@ -986,40 +966,37 @@ mod tests { } #[test] - fn images_become_content_parts() { - let mut msg = ChatMessage::user("What is in this photo?"); - msg.images = Some(vec!["BASE64DATA".into()]); - - let wire = OpenRouterClient::messages_to_openai(&[msg]); - let content = wire[0].get("content").and_then(|v| v.as_array()).unwrap(); - assert_eq!(content.len(), 2); - assert_eq!(content[0]["type"], "text"); - assert_eq!(content[0]["text"], "What is in this photo?"); - assert_eq!(content[1]["type"], "image_url"); - assert_eq!( - content[1]["image_url"]["url"], - "data:image/jpeg;base64,BASE64DATA" - ); + fn any_image_anywhere_in_the_transcript_is_refused() { + // Not just the latest message: a replayed transcript whose first + // user turn still holds the photo must be refused too. + let mut first = ChatMessage::user("What is in this photo?"); + first.images = Some(vec!["BASE64DATA".into()]); + let later = ChatMessage::user("and then?"); + let err = OpenRouterClient::messages_to_openai(&[first, later]).unwrap_err(); + assert!(err.to_string().contains("photos stay on the local network")); } #[test] - fn data_url_images_pass_through_unchanged() { - let mut msg = ChatMessage::user(""); - msg.images = Some(vec!["data:image/png;base64,ABCDEF".into()]); - let wire = OpenRouterClient::messages_to_openai(&[msg]); - let content = wire[0].get("content").and_then(|v| v.as_array()).unwrap(); - // No text part when content is empty. - assert_eq!(content.len(), 1); - assert_eq!( - content[0]["image_url"]["url"], - "data:image/png;base64,ABCDEF" - ); + fn an_empty_image_list_is_not_an_image() { + let mut msg = ChatMessage::user("hi"); + msg.images = Some(Vec::new()); + assert!(OpenRouterClient::messages_to_openai(&[msg]).is_ok()); + } + + #[tokio::test] + async fn describe_image_refuses_without_a_network_call() { + // Points at a closed port: if the guard ever stopped refusing, this + // would fail with a connection error instead of the guard's. + let client = + OpenRouterClient::new("key".into(), Some("http://127.0.0.1:9".into()), "m".into()); + let err = client.describe_image("BASE64DATA").await.unwrap_err(); + assert!(err.to_string().contains("photos stay on the local network")); } #[test] fn text_only_message_stays_string() { let msg = ChatMessage::user("hello"); - let wire = OpenRouterClient::messages_to_openai(&[msg]); + let wire = OpenRouterClient::messages_to_openai(&[msg]).unwrap(); assert_eq!(wire[0]["content"], "hello"); assert!(wire[0]["content"].as_str().is_some()); } @@ -1040,7 +1017,7 @@ mod tests { }; let tool_result = ChatMessage::tool_result("found it"); - let wire = OpenRouterClient::messages_to_openai(&[assistant, tool_result]); + let wire = OpenRouterClient::messages_to_openai(&[assistant, tool_result]).unwrap(); assert_eq!(wire[1]["role"], "tool"); assert_eq!(wire[1]["tool_call_id"], "call_42"); } @@ -1071,7 +1048,7 @@ mod tests { let r1 = ChatMessage::tool_result("a result"); let r2 = ChatMessage::tool_result("b result"); - let wire = OpenRouterClient::messages_to_openai(&[assistant, r1, r2]); + let wire = OpenRouterClient::messages_to_openai(&[assistant, r1, r2]).unwrap(); assert_eq!(wire[1]["tool_call_id"], "call_A"); assert_eq!(wire[2]["tool_call_id"], "call_B"); } @@ -1090,7 +1067,7 @@ mod tests { }]), images: None, }; - let wire = OpenRouterClient::messages_to_openai(&[assistant]); + let wire = OpenRouterClient::messages_to_openai(&[assistant]).unwrap(); let tcs = wire[0] .get("tool_calls") .and_then(|v| v.as_array())