Files
ImageApi/src/ai/handlers.rs
Cameron 2d942a9926 feat: content-hash-aware tag/insight sharing + library scoping
Tags and insights now follow content across libraries via content_hash
lookups on the read path, so the same file indexed at different rel_paths
in multiple libraries shares its annotations. Recursive tag search scopes
hits to the selected library by checking each tagged rel_path against
the library's disk (with a content-hash sibling fallback so tags attached
under one library's rel_path still match a content-equivalent file in
another). The /image and /image/metadata handlers fall back across
libraries when the file isn't under the resolved one, so union-mode
search results (which carry no library attribution in the response)
still serve correctly.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-04-21 01:55:07 +00:00

517 lines
18 KiB
Rust

use actix_web::{HttpRequest, HttpResponse, Responder, delete, get, post, web};
use opentelemetry::KeyValue;
use opentelemetry::trace::{Span, Status, Tracer};
use serde::{Deserialize, Serialize};
use crate::ai::{InsightGenerator, ModelCapabilities, OllamaClient};
use crate::data::Claims;
use crate::database::{ExifDao, InsightDao};
use crate::libraries;
use crate::otel::{extract_context_from_request, global_tracer};
use crate::state::AppState;
use crate::utils::normalize_path;
#[derive(Debug, Deserialize)]
pub struct GeneratePhotoInsightRequest {
pub file_path: String,
#[serde(default)]
pub model: Option<String>,
#[serde(default)]
pub system_prompt: Option<String>,
#[serde(default)]
pub num_ctx: Option<i32>,
#[serde(default)]
pub temperature: Option<f32>,
#[serde(default)]
pub top_p: Option<f32>,
#[serde(default)]
pub top_k: Option<i32>,
#[serde(default)]
pub min_p: Option<f32>,
}
#[derive(Debug, Deserialize)]
pub struct GetPhotoInsightQuery {
pub path: String,
/// Library context for this lookup. Used to pick the right content
/// hash when the same rel_path exists under multiple roots.
#[serde(default)]
pub library: Option<String>,
}
#[derive(Debug, Deserialize)]
pub struct RateInsightRequest {
pub file_path: String,
pub approved: bool,
}
#[derive(Debug, Deserialize)]
pub struct ExportTrainingDataQuery {
#[serde(default)]
pub approved_only: Option<bool>,
}
#[derive(Debug, Serialize)]
pub struct PhotoInsightResponse {
pub id: i32,
pub file_path: String,
pub title: String,
pub summary: String,
pub generated_at: i64,
pub model_version: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub prompt_eval_count: Option<i32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub eval_count: Option<i32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub approved: Option<bool>,
}
#[derive(Debug, Serialize)]
pub struct AvailableModelsResponse {
pub primary: ServerModels,
#[serde(skip_serializing_if = "Option::is_none")]
pub fallback: Option<ServerModels>,
}
#[derive(Debug, Serialize)]
pub struct ServerModels {
pub url: String,
pub models: Vec<ModelCapabilities>,
pub default_model: String,
}
/// POST /insights/generate - Generate insight for a specific photo
#[post("/insights/generate")]
pub async fn generate_insight_handler(
http_request: HttpRequest,
_claims: Claims,
request: web::Json<GeneratePhotoInsightRequest>,
insight_generator: web::Data<InsightGenerator>,
) -> impl Responder {
let parent_context = extract_context_from_request(&http_request);
let tracer = global_tracer();
let mut span = tracer.start_with_context("http.insights.generate", &parent_context);
let normalized_path = normalize_path(&request.file_path);
span.set_attribute(KeyValue::new("file_path", normalized_path.clone()));
if let Some(ref model) = request.model {
span.set_attribute(KeyValue::new("model", model.clone()));
}
if let Some(ref prompt) = request.system_prompt {
span.set_attribute(KeyValue::new("has_custom_prompt", true));
span.set_attribute(KeyValue::new("prompt_length", prompt.len() as i64));
}
if let Some(ctx) = request.num_ctx {
span.set_attribute(KeyValue::new("num_ctx", ctx as i64));
}
log::info!(
"Manual insight generation triggered for photo: {} with model: {:?}, custom_prompt: {}, num_ctx: {:?}",
normalized_path,
request.model,
request.system_prompt.is_some(),
request.num_ctx
);
// Generate insight with optional custom model, system prompt, and context size
let result = insight_generator
.generate_insight_for_photo_with_config(
&normalized_path,
request.model.clone(),
request.system_prompt.clone(),
request.num_ctx,
request.temperature,
request.top_p,
request.top_k,
request.min_p,
)
.await;
match result {
Ok(()) => {
span.set_status(Status::Ok);
HttpResponse::Ok().json(serde_json::json!({
"success": true,
"message": "Insight generated successfully"
}))
}
Err(e) => {
log::error!("Failed to generate insight: {:?}", e);
span.set_status(Status::error(e.to_string()));
HttpResponse::InternalServerError().json(serde_json::json!({
"error": format!("Failed to generate insight: {:?}", e)
}))
}
}
}
/// GET /insights?path=/path/to/photo.jpg - Fetch insight for specific photo
#[get("/insights")]
pub async fn get_insight_handler(
_claims: Claims,
query: web::Query<GetPhotoInsightQuery>,
app_state: web::Data<AppState>,
insight_dao: web::Data<std::sync::Mutex<Box<dyn InsightDao>>>,
exif_dao: web::Data<std::sync::Mutex<Box<dyn ExifDao>>>,
) -> impl Responder {
let normalized_path = normalize_path(&query.path);
log::debug!("Fetching insight for {}", normalized_path);
let otel_context = opentelemetry::Context::new();
// Expand to rel_paths sharing content so an insight generated under
// library 1 still shows when the same photo is viewed from library 2.
let library = libraries::resolve_library_param(&app_state, query.library.as_deref())
.ok()
.flatten()
.unwrap_or_else(|| app_state.primary_library());
let sibling_paths = {
let mut exif = exif_dao.lock().expect("Unable to lock ExifDao");
exif.get_rel_paths_sharing_content(&otel_context, library.id, &normalized_path)
.unwrap_or_else(|_| vec![normalized_path.clone()])
};
let mut dao = insight_dao.lock().expect("Unable to lock InsightDao");
match dao.get_insight_for_paths(&otel_context, &sibling_paths) {
Ok(Some(insight)) => {
let response = PhotoInsightResponse {
id: insight.id,
file_path: insight.file_path,
title: insight.title,
summary: insight.summary,
generated_at: insight.generated_at,
model_version: insight.model_version,
prompt_eval_count: None,
eval_count: None,
approved: insight.approved,
};
HttpResponse::Ok().json(response)
}
Ok(None) => HttpResponse::NotFound().json(serde_json::json!({
"error": "Insight not found"
})),
Err(e) => {
log::error!("Failed to fetch insight ({}): {:?}", &query.path, e);
HttpResponse::InternalServerError().json(serde_json::json!({
"error": format!("Failed to fetch insight: {:?}", e)
}))
}
}
}
/// DELETE /insights?path=/path/to/photo.jpg - Remove insight (will regenerate on next request)
#[delete("/insights")]
pub async fn delete_insight_handler(
_claims: Claims,
query: web::Query<GetPhotoInsightQuery>,
insight_dao: web::Data<std::sync::Mutex<Box<dyn InsightDao>>>,
) -> impl Responder {
let normalized_path = normalize_path(&query.path);
log::info!("Deleting insight for {}", normalized_path);
let otel_context = opentelemetry::Context::new();
let mut dao = insight_dao.lock().expect("Unable to lock InsightDao");
match dao.delete_insight(&otel_context, &normalized_path) {
Ok(()) => HttpResponse::Ok().json(serde_json::json!({
"success": true,
"message": "Insight deleted successfully"
})),
Err(e) => {
log::error!("Failed to delete insight: {:?}", e);
HttpResponse::InternalServerError().json(serde_json::json!({
"error": format!("Failed to delete insight: {:?}", e)
}))
}
}
}
/// GET /insights/all - Get all insights
#[get("/insights/all")]
pub async fn get_all_insights_handler(
_claims: Claims,
insight_dao: web::Data<std::sync::Mutex<Box<dyn InsightDao>>>,
) -> impl Responder {
log::debug!("Fetching all insights");
let otel_context = opentelemetry::Context::new();
let mut dao = insight_dao.lock().expect("Unable to lock InsightDao");
match dao.get_all_insights(&otel_context) {
Ok(insights) => {
let responses: Vec<PhotoInsightResponse> = insights
.into_iter()
.map(|insight| PhotoInsightResponse {
id: insight.id,
file_path: insight.file_path,
title: insight.title,
summary: insight.summary,
generated_at: insight.generated_at,
model_version: insight.model_version,
prompt_eval_count: None,
eval_count: None,
approved: insight.approved,
})
.collect();
HttpResponse::Ok().json(responses)
}
Err(e) => {
log::error!("Failed to fetch all insights: {:?}", e);
HttpResponse::InternalServerError().json(serde_json::json!({
"error": format!("Failed to fetch insights: {:?}", e)
}))
}
}
}
/// POST /insights/generate/agentic - Generate insight using agentic tool-calling loop
#[post("/insights/generate/agentic")]
pub async fn generate_agentic_insight_handler(
http_request: HttpRequest,
_claims: Claims,
request: web::Json<GeneratePhotoInsightRequest>,
insight_generator: web::Data<InsightGenerator>,
insight_dao: web::Data<std::sync::Mutex<Box<dyn InsightDao>>>,
) -> impl Responder {
let parent_context = extract_context_from_request(&http_request);
let tracer = global_tracer();
let mut span = tracer.start_with_context("http.insights.generate_agentic", &parent_context);
let normalized_path = normalize_path(&request.file_path);
span.set_attribute(KeyValue::new("file_path", normalized_path.clone()));
if let Some(ref model) = request.model {
span.set_attribute(KeyValue::new("model", model.clone()));
}
if let Some(ref prompt) = request.system_prompt {
span.set_attribute(KeyValue::new("has_custom_prompt", true));
span.set_attribute(KeyValue::new("prompt_length", prompt.len() as i64));
}
if let Some(ctx) = request.num_ctx {
span.set_attribute(KeyValue::new("num_ctx", ctx as i64));
}
let max_iterations: usize = std::env::var("AGENTIC_MAX_ITERATIONS")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(12);
span.set_attribute(KeyValue::new("max_iterations", max_iterations as i64));
log::info!(
"Agentic insight generation triggered for photo: {} with model: {:?}, max_iterations: {}",
normalized_path,
request.model,
max_iterations
);
let result = insight_generator
.generate_agentic_insight_for_photo(
&normalized_path,
request.model.clone(),
request.system_prompt.clone(),
request.num_ctx,
request.temperature,
request.top_p,
request.top_k,
request.min_p,
max_iterations,
)
.await;
match result {
Ok((prompt_eval_count, eval_count)) => {
span.set_status(Status::Ok);
// Fetch the stored insight to return it
let otel_context = opentelemetry::Context::new();
let mut dao = insight_dao.lock().expect("Unable to lock InsightDao");
match dao.get_insight(&otel_context, &normalized_path) {
Ok(Some(insight)) => {
let response = PhotoInsightResponse {
id: insight.id,
file_path: insight.file_path,
title: insight.title,
summary: insight.summary,
generated_at: insight.generated_at,
model_version: insight.model_version,
prompt_eval_count,
eval_count,
approved: insight.approved,
};
HttpResponse::Ok().json(response)
}
Ok(None) => HttpResponse::Ok().json(serde_json::json!({
"success": true,
"message": "Agentic insight generated successfully"
})),
Err(e) => {
log::warn!("Insight stored but failed to retrieve: {:?}", e);
HttpResponse::Ok().json(serde_json::json!({
"success": true,
"message": "Agentic insight generated successfully"
}))
}
}
}
Err(e) => {
let error_msg = format!("{:?}", e);
log::error!("Failed to generate agentic insight: {}", error_msg);
span.set_status(Status::error(error_msg.clone()));
if error_msg.contains("tool calling not supported")
|| error_msg.contains("model not available")
{
HttpResponse::BadRequest().json(serde_json::json!({
"error": format!("Failed to generate agentic insight: {}", error_msg)
}))
} else if error_msg.contains("error parsing tool call") {
HttpResponse::BadRequest().json(serde_json::json!({
"error": "Model is not compatible with Ollama's tool calling protocol. Try a model known to support native tool calling (e.g. llama3.1, llama3.2, qwen2.5, mistral-nemo)."
}))
} else {
HttpResponse::InternalServerError().json(serde_json::json!({
"error": format!("Failed to generate agentic insight: {}", error_msg)
}))
}
}
}
}
/// GET /insights/models - List available models from both servers with capabilities
#[get("/insights/models")]
pub async fn get_available_models_handler(
_claims: Claims,
app_state: web::Data<crate::state::AppState>,
) -> impl Responder {
log::debug!("Fetching available models with capabilities");
let ollama_client = &app_state.ollama;
// Fetch models with capabilities from primary server
let primary_models =
match OllamaClient::list_models_with_capabilities(&ollama_client.primary_url).await {
Ok(models) => models,
Err(e) => {
log::warn!("Failed to fetch models from primary server: {:?}", e);
vec![]
}
};
let primary = ServerModels {
url: ollama_client.primary_url.clone(),
models: primary_models,
default_model: ollama_client.primary_model.clone(),
};
// Fetch models with capabilities from fallback server if configured
let fallback = if let Some(fallback_url) = &ollama_client.fallback_url {
match OllamaClient::list_models_with_capabilities(fallback_url).await {
Ok(models) => Some(ServerModels {
url: fallback_url.clone(),
models,
default_model: ollama_client
.fallback_model
.clone()
.unwrap_or_else(|| ollama_client.primary_model.clone()),
}),
Err(e) => {
log::warn!("Failed to fetch models from fallback server: {:?}", e);
None
}
}
} else {
None
};
let response = AvailableModelsResponse { primary, fallback };
HttpResponse::Ok().json(response)
}
/// POST /insights/rate - Rate an insight (thumbs up/down for training data)
#[post("/insights/rate")]
pub async fn rate_insight_handler(
_claims: Claims,
request: web::Json<RateInsightRequest>,
insight_dao: web::Data<std::sync::Mutex<Box<dyn InsightDao>>>,
) -> impl Responder {
let normalized_path = normalize_path(&request.file_path);
log::info!(
"Rating insight for {}: approved={}",
normalized_path,
request.approved
);
let otel_context = opentelemetry::Context::new();
let mut dao = insight_dao.lock().expect("Unable to lock InsightDao");
match dao.rate_insight(&otel_context, &normalized_path, request.approved) {
Ok(()) => HttpResponse::Ok().json(serde_json::json!({
"success": true,
"message": "Insight rated successfully"
})),
Err(e) => {
log::error!("Failed to rate insight: {:?}", e);
HttpResponse::InternalServerError().json(serde_json::json!({
"error": format!("Failed to rate insight: {:?}", e)
}))
}
}
}
/// GET /insights/training-data - Export approved training data as JSONL
#[get("/insights/training-data")]
pub async fn export_training_data_handler(
_claims: Claims,
query: web::Query<ExportTrainingDataQuery>,
insight_dao: web::Data<std::sync::Mutex<Box<dyn InsightDao>>>,
) -> impl Responder {
let approved_only = query.approved_only.unwrap_or(true);
log::info!("Exporting training data (approved_only={})", approved_only);
let otel_context = opentelemetry::Context::new();
let mut dao = insight_dao.lock().expect("Unable to lock InsightDao");
let insights = if approved_only {
dao.get_approved_insights(&otel_context)
} else {
dao.get_all_insights(&otel_context)
};
match insights {
Ok(insights) => {
let mut jsonl = String::new();
for insight in &insights {
if let Some(ref messages) = insight.training_messages {
let entry = serde_json::json!({
"file_path": insight.file_path,
"model_version": insight.model_version,
"generated_at": insight.generated_at,
"title": insight.title,
"summary": insight.summary,
"messages": serde_json::from_str::<serde_json::Value>(messages)
.unwrap_or(serde_json::Value::Null),
});
jsonl.push_str(&entry.to_string());
jsonl.push('\n');
}
}
HttpResponse::Ok()
.content_type("application/jsonl")
.insert_header(("Content-Disposition", "attachment; filename=\"training_data.jsonl\""))
.body(jsonl)
}
Err(e) => {
log::error!("Failed to export training data: {:?}", e);
HttpResponse::InternalServerError().json(serde_json::json!({
"error": format!("Failed to export training data: {:?}", e)
}))
}
}
}