feat(ai): hybrid backend mode for agentic insights

Adds a `backend` column to photo_insights (default 'local', migration
2026-04-20-000000) and a corresponding optional `backend` field on the
agentic request. When a request sets backend=hybrid:

- The local Ollama vision model is called once via describe_image to
  produce a text description.
- The description is inlined into the first user message as text —
  no base64 image is ever sent to the chat model.
- The agentic tool-calling loop and title generation route through an
  OpenRouterClient (dispatched via &dyn LlmClient), letting the user
  pick any tool-capable model from OpenRouter per request.
- describe_photo is removed from the offered tools since the description
  is already present.

Embeddings and vision stay on local Ollama regardless of backend.
Hybrid mode requires OPENROUTER_API_KEY; handlers return a clear error
when hybrid is requested without it, and also when the selected
OpenRouter model lacks tool-calling support.

AppState gains an optional openrouter client built from
OPENROUTER_API_KEY / OPENROUTER_BASE_URL / OPENROUTER_DEFAULT_MODEL /
OPENROUTER_EMBEDDING_MODEL / attribution headers. Default model is
anthropic/claude-sonnet-4.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Cameron
2026-04-20 22:30:40 -04:00
parent e799ba716c
commit 3ac0cd62eb
8 changed files with 342 additions and 100 deletions

View File

@@ -1,3 +1,4 @@
use crate::ai::openrouter::OpenRouterClient;
use crate::ai::{InsightGenerator, OllamaClient, SmsApiClient};
use crate::database::{
CalendarEventDao, DailySummaryDao, ExifDao, InsightDao, KnowledgeDao, LocationHistoryDao,
@@ -31,6 +32,13 @@ pub struct AppState {
pub preview_clips_path: String,
pub excluded_dirs: Vec<String>,
pub ollama: OllamaClient,
/// `None` when `OPENROUTER_API_KEY` is not configured. Consulted only
/// when a request explicitly opts into `backend=hybrid`. Currently
/// reached via `insight_generator`; kept here so future handlers
/// (insight_chat) can route to it without threading it through the
/// generator.
#[allow(dead_code)]
pub openrouter: Option<Arc<OpenRouterClient>>,
pub sms_client: SmsApiClient,
pub insight_generator: InsightGenerator,
}
@@ -61,6 +69,7 @@ impl AppState {
preview_clips_path: String,
excluded_dirs: Vec<String>,
ollama: OllamaClient,
openrouter: Option<Arc<OpenRouterClient>>,
sms_client: SmsApiClient,
insight_generator: InsightGenerator,
preview_dao: Arc<Mutex<Box<dyn PreviewDao>>>,
@@ -92,6 +101,7 @@ impl AppState {
preview_clips_path,
excluded_dirs,
ollama,
openrouter,
sms_client,
insight_generator,
}
@@ -127,6 +137,8 @@ impl Default for AppState {
ollama_fallback_model,
);
let openrouter = build_openrouter_from_env();
let sms_api_url =
env::var("SMS_API_URL").unwrap_or_else(|_| "http://localhost:8000".to_string());
let sms_api_token = env::var("SMS_API_TOKEN").ok();
@@ -168,6 +180,7 @@ impl Default for AppState {
// Initialize InsightGenerator with all data sources
let insight_generator = InsightGenerator::new(
ollama.clone(),
openrouter.clone(),
sms_client.clone(),
insight_dao.clone(),
exif_dao.clone(),
@@ -195,6 +208,7 @@ impl Default for AppState {
preview_clips_path,
Self::parse_excluded_dirs(),
ollama,
openrouter,
sms_client,
insight_generator,
preview_dao,
@@ -202,6 +216,25 @@ impl Default for AppState {
}
}
/// Build an `OpenRouterClient` from environment variables. Returns `None`
/// when `OPENROUTER_API_KEY` is unset (the hybrid backend is then
/// unavailable and requests for it return a clear error).
fn build_openrouter_from_env() -> Option<Arc<OpenRouterClient>> {
let api_key = env::var("OPENROUTER_API_KEY").ok()?;
let base_url = env::var("OPENROUTER_BASE_URL").ok();
let default_model = env::var("OPENROUTER_DEFAULT_MODEL")
.unwrap_or_else(|_| "anthropic/claude-sonnet-4".to_string());
let mut client = OpenRouterClient::new(api_key, base_url, default_model);
client.set_attribution(
env::var("OPENROUTER_HTTP_REFERER").ok(),
env::var("OPENROUTER_APP_TITLE").ok(),
);
if let Ok(model) = env::var("OPENROUTER_EMBEDDING_MODEL") {
client.set_embedding_model(model);
}
Some(Arc::new(client))
}
#[cfg(test)]
impl AppState {
/// Creates an AppState instance for testing with temporary directories
@@ -255,6 +288,7 @@ impl AppState {
};
let insight_generator = InsightGenerator::new(
ollama.clone(),
None,
sms_client.clone(),
insight_dao.clone(),
exif_dao.clone(),
@@ -286,6 +320,7 @@ impl AppState {
preview_clips_path.to_string_lossy().to_string(),
Vec::new(), // No excluded directories for test state
ollama,
None,
sms_client,
insight_generator,
preview_dao,