feat(ai): hybrid backend mode for agentic insights
Adds a `backend` column to photo_insights (default 'local', migration 2026-04-20-000000) and a corresponding optional `backend` field on the agentic request. When a request sets backend=hybrid: - The local Ollama vision model is called once via describe_image to produce a text description. - The description is inlined into the first user message as text — no base64 image is ever sent to the chat model. - The agentic tool-calling loop and title generation route through an OpenRouterClient (dispatched via &dyn LlmClient), letting the user pick any tool-capable model from OpenRouter per request. - describe_photo is removed from the offered tools since the description is already present. Embeddings and vision stay on local Ollama regardless of backend. Hybrid mode requires OPENROUTER_API_KEY; handlers return a clear error when hybrid is requested without it, and also when the selected OpenRouter model lacks tool-calling support. AppState gains an optional openrouter client built from OPENROUTER_API_KEY / OPENROUTER_BASE_URL / OPENROUTER_DEFAULT_MODEL / OPENROUTER_EMBEDDING_MODEL / attribution headers. Default model is anthropic/claude-sonnet-4. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
35
src/state.rs
35
src/state.rs
@@ -1,3 +1,4 @@
|
||||
use crate::ai::openrouter::OpenRouterClient;
|
||||
use crate::ai::{InsightGenerator, OllamaClient, SmsApiClient};
|
||||
use crate::database::{
|
||||
CalendarEventDao, DailySummaryDao, ExifDao, InsightDao, KnowledgeDao, LocationHistoryDao,
|
||||
@@ -31,6 +32,13 @@ pub struct AppState {
|
||||
pub preview_clips_path: String,
|
||||
pub excluded_dirs: Vec<String>,
|
||||
pub ollama: OllamaClient,
|
||||
/// `None` when `OPENROUTER_API_KEY` is not configured. Consulted only
|
||||
/// when a request explicitly opts into `backend=hybrid`. Currently
|
||||
/// reached via `insight_generator`; kept here so future handlers
|
||||
/// (insight_chat) can route to it without threading it through the
|
||||
/// generator.
|
||||
#[allow(dead_code)]
|
||||
pub openrouter: Option<Arc<OpenRouterClient>>,
|
||||
pub sms_client: SmsApiClient,
|
||||
pub insight_generator: InsightGenerator,
|
||||
}
|
||||
@@ -61,6 +69,7 @@ impl AppState {
|
||||
preview_clips_path: String,
|
||||
excluded_dirs: Vec<String>,
|
||||
ollama: OllamaClient,
|
||||
openrouter: Option<Arc<OpenRouterClient>>,
|
||||
sms_client: SmsApiClient,
|
||||
insight_generator: InsightGenerator,
|
||||
preview_dao: Arc<Mutex<Box<dyn PreviewDao>>>,
|
||||
@@ -92,6 +101,7 @@ impl AppState {
|
||||
preview_clips_path,
|
||||
excluded_dirs,
|
||||
ollama,
|
||||
openrouter,
|
||||
sms_client,
|
||||
insight_generator,
|
||||
}
|
||||
@@ -127,6 +137,8 @@ impl Default for AppState {
|
||||
ollama_fallback_model,
|
||||
);
|
||||
|
||||
let openrouter = build_openrouter_from_env();
|
||||
|
||||
let sms_api_url =
|
||||
env::var("SMS_API_URL").unwrap_or_else(|_| "http://localhost:8000".to_string());
|
||||
let sms_api_token = env::var("SMS_API_TOKEN").ok();
|
||||
@@ -168,6 +180,7 @@ impl Default for AppState {
|
||||
// Initialize InsightGenerator with all data sources
|
||||
let insight_generator = InsightGenerator::new(
|
||||
ollama.clone(),
|
||||
openrouter.clone(),
|
||||
sms_client.clone(),
|
||||
insight_dao.clone(),
|
||||
exif_dao.clone(),
|
||||
@@ -195,6 +208,7 @@ impl Default for AppState {
|
||||
preview_clips_path,
|
||||
Self::parse_excluded_dirs(),
|
||||
ollama,
|
||||
openrouter,
|
||||
sms_client,
|
||||
insight_generator,
|
||||
preview_dao,
|
||||
@@ -202,6 +216,25 @@ impl Default for AppState {
|
||||
}
|
||||
}
|
||||
|
||||
/// Build an `OpenRouterClient` from environment variables. Returns `None`
|
||||
/// when `OPENROUTER_API_KEY` is unset (the hybrid backend is then
|
||||
/// unavailable and requests for it return a clear error).
|
||||
fn build_openrouter_from_env() -> Option<Arc<OpenRouterClient>> {
|
||||
let api_key = env::var("OPENROUTER_API_KEY").ok()?;
|
||||
let base_url = env::var("OPENROUTER_BASE_URL").ok();
|
||||
let default_model = env::var("OPENROUTER_DEFAULT_MODEL")
|
||||
.unwrap_or_else(|_| "anthropic/claude-sonnet-4".to_string());
|
||||
let mut client = OpenRouterClient::new(api_key, base_url, default_model);
|
||||
client.set_attribution(
|
||||
env::var("OPENROUTER_HTTP_REFERER").ok(),
|
||||
env::var("OPENROUTER_APP_TITLE").ok(),
|
||||
);
|
||||
if let Ok(model) = env::var("OPENROUTER_EMBEDDING_MODEL") {
|
||||
client.set_embedding_model(model);
|
||||
}
|
||||
Some(Arc::new(client))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl AppState {
|
||||
/// Creates an AppState instance for testing with temporary directories
|
||||
@@ -255,6 +288,7 @@ impl AppState {
|
||||
};
|
||||
let insight_generator = InsightGenerator::new(
|
||||
ollama.clone(),
|
||||
None,
|
||||
sms_client.clone(),
|
||||
insight_dao.clone(),
|
||||
exif_dao.clone(),
|
||||
@@ -286,6 +320,7 @@ impl AppState {
|
||||
preview_clips_path.to_string_lossy().to_string(),
|
||||
Vec::new(), // No excluded directories for test state
|
||||
ollama,
|
||||
None,
|
||||
sms_client,
|
||||
insight_generator,
|
||||
preview_dao,
|
||||
|
||||
Reference in New Issue
Block a user