feat(ai): hybrid backend mode for agentic insights
Adds a `backend` column to photo_insights (default 'local', migration 2026-04-20-000000) and a corresponding optional `backend` field on the agentic request. When a request sets backend=hybrid: - The local Ollama vision model is called once via describe_image to produce a text description. - The description is inlined into the first user message as text — no base64 image is ever sent to the chat model. - The agentic tool-calling loop and title generation route through an OpenRouterClient (dispatched via &dyn LlmClient), letting the user pick any tool-capable model from OpenRouter per request. - describe_photo is removed from the offered tools since the description is already present. Embeddings and vision stay on local Ollama regardless of backend. Hybrid mode requires OPENROUTER_API_KEY; handlers return a clear error when hybrid is requested without it, and also when the selected OpenRouter model lacks tool-calling support. AppState gains an optional openrouter client built from OPENROUTER_API_KEY / OPENROUTER_BASE_URL / OPENROUTER_DEFAULT_MODEL / OPENROUTER_EMBEDDING_MODEL / attribution headers. Default model is anthropic/claude-sonnet-4. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -28,6 +28,10 @@ pub struct GeneratePhotoInsightRequest {
|
||||
pub top_k: Option<i32>,
|
||||
#[serde(default)]
|
||||
pub min_p: Option<f32>,
|
||||
/// `"local"` (default, Ollama with images) | `"hybrid"` (local vision +
|
||||
/// OpenRouter chat). Only respected by the agentic endpoint.
|
||||
#[serde(default)]
|
||||
pub backend: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
@@ -65,6 +69,7 @@ pub struct PhotoInsightResponse {
|
||||
pub eval_count: Option<i32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub approved: Option<bool>,
|
||||
pub backend: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
@@ -187,6 +192,7 @@ pub async fn get_insight_handler(
|
||||
prompt_eval_count: None,
|
||||
eval_count: None,
|
||||
approved: insight.approved,
|
||||
backend: insight.backend,
|
||||
};
|
||||
HttpResponse::Ok().json(response)
|
||||
}
|
||||
@@ -254,6 +260,7 @@ pub async fn get_all_insights_handler(
|
||||
prompt_eval_count: None,
|
||||
eval_count: None,
|
||||
approved: insight.approved,
|
||||
backend: insight.backend,
|
||||
})
|
||||
.collect();
|
||||
|
||||
@@ -309,6 +316,10 @@ pub async fn generate_agentic_insight_handler(
|
||||
max_iterations
|
||||
);
|
||||
|
||||
if let Some(ref b) = request.backend {
|
||||
span.set_attribute(KeyValue::new("backend", b.clone()));
|
||||
}
|
||||
|
||||
let result = insight_generator
|
||||
.generate_agentic_insight_for_photo(
|
||||
&normalized_path,
|
||||
@@ -320,6 +331,7 @@ pub async fn generate_agentic_insight_handler(
|
||||
request.top_k,
|
||||
request.min_p,
|
||||
max_iterations,
|
||||
request.backend.clone(),
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -341,6 +353,7 @@ pub async fn generate_agentic_insight_handler(
|
||||
prompt_eval_count,
|
||||
eval_count,
|
||||
approved: insight.approved,
|
||||
backend: insight.backend,
|
||||
};
|
||||
HttpResponse::Ok().json(response)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user