Cameron CordesandClaude Opus 5.5
bfc3c1c21a
/photos, /photos/exif and unified search take date_from_day / date_to_day (YYYY-MM-DD): the server turns them into the first day's start and the last day's end in the owner's display zone, the same days /memories groups by. Each replaces its instant counterpart. Clients stop computing midnights in the device's zone, which drifted by the travel offset whenever the phone's zone differed from display_tz. e2e check: every fixture is included by its display-zone day and excluded by the days either side. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
865 lines
32 KiB
Rust
865 lines
32 KiB
Rust
//! `/photos/search/unified?q=<natural language>` — unified NL photo search.
|
|
//!
|
|
//! One free-text box that composes the two existing engines instead of making
|
|
//! the user pick between them:
|
|
//! 1. A grounded local-LLM call ([`crate::ai::nl_query`]) translates the
|
|
//! query into a structured filter + a semantic term.
|
|
//! 2. Structured filters (tags / EXIF / geo / date / media-type) define the
|
|
//! candidate set; the semantic term ranks within it via CLIP.
|
|
//!
|
|
//! Path A (orchestration): we reuse `clip_search`'s scoring core and the
|
|
//! existing `ExifDao` / `TagDao` queries, joining on `content_hash`. EXIF rows
|
|
//! are the universal candidate carrier — each has `(library_id, file_path,
|
|
//! content_hash, date_taken)` — so the structured filter is just a predicate
|
|
//! over them, and the CLIP hits (which key on `content_hash`) intersect by
|
|
//! hash. No new schema, no surgery on `list_photos`.
|
|
//!
|
|
//! Degenerate cases collapse to the existing behavior: semantic-only → plain
|
|
//! CLIP search; filters-only → a date-sorted filtered listing.
|
|
//!
|
|
//! Person filtering is intentionally deferred (no person→photos resolver yet).
|
|
|
|
use crate::AppState;
|
|
use crate::ai::backend::{BackendKind, SamplingOverrides};
|
|
use crate::ai::nl_query::{StructuredQuery, translate_nl_query};
|
|
use crate::clip_search::{
|
|
SearchHit, parse_library_scope, resolve_hits, score_error_response, score_photos,
|
|
};
|
|
use crate::data::{Claims, FilterMode, MediaType};
|
|
use crate::database::ExifDao;
|
|
use crate::file_types::{is_image_file, is_video_file};
|
|
use crate::geo::{forward_geocode, gps_bounding_box, haversine_distance};
|
|
use crate::tags::TagDao;
|
|
use actix_web::HttpResponse;
|
|
use actix_web::web::{Data, Query};
|
|
use serde::{Deserialize, Serialize};
|
|
use std::collections::HashSet;
|
|
use std::path::Path;
|
|
use std::sync::Mutex;
|
|
|
|
#[derive(Debug, Deserialize)]
|
|
pub struct UnifiedQuery {
|
|
/// Natural-language query. Required; empty triggers 400.
|
|
pub q: String,
|
|
#[serde(default = "default_limit")]
|
|
pub limit: usize,
|
|
#[serde(default)]
|
|
pub offset: usize,
|
|
/// CLIP cosine floor for the semantic ranking stage. Same default as the
|
|
/// plain search endpoint.
|
|
#[serde(default = "default_threshold")]
|
|
pub threshold: f32,
|
|
/// Legacy single-library scope (see clip_search).
|
|
pub library: Option<i32>,
|
|
/// Multi-library scope, comma-separated ids.
|
|
pub library_ids: Option<String>,
|
|
/// Optional model override. The client passes the user's currently-selected
|
|
/// local model so the translation step reuses a model that's already loaded
|
|
/// (avoids a llama-swap eviction / cold start). Falls back to the configured
|
|
/// default local model when absent. Local only — no hybrid here.
|
|
pub model: Option<String>,
|
|
/// When false, skip the LLM translation and use `q` verbatim as the CLIP
|
|
/// term ("visual match only"). Explicit filters below still apply.
|
|
#[serde(default = "default_interpret")]
|
|
pub interpret: bool,
|
|
|
|
// ── Explicit filters ──
|
|
// Set by the client's filter chips. They're AND-ed with whatever the
|
|
// model extracted from `q`; for single-valued filters (camera, date,
|
|
// media type, place) the explicit value replaces the model's guess.
|
|
// Same encodings as `/photos`.
|
|
/// Comma-separated tag ids the results must carry (see `tag_filter_mode`).
|
|
pub tag_ids: Option<String>,
|
|
/// Comma-separated tag ids the results must not carry.
|
|
pub exclude_tag_ids: Option<String>,
|
|
/// `All` (default) or `Any` for `tag_ids`.
|
|
pub tag_filter_mode: Option<FilterMode>,
|
|
pub media_type: Option<MediaType>,
|
|
pub date_from: Option<i64>,
|
|
pub date_to: Option<i64>,
|
|
/// Calendar days (`YYYY-MM-DD`) in the owner's display zone; each
|
|
/// replaces its instant counterpart (see `date_view::apply_day_bounds`).
|
|
pub date_from_day: Option<String>,
|
|
pub date_to_day: Option<String>,
|
|
pub camera_make: Option<String>,
|
|
pub camera_model: Option<String>,
|
|
pub gps_lat: Option<f64>,
|
|
pub gps_lon: Option<f64>,
|
|
pub gps_radius_km: Option<f64>,
|
|
/// Folder (relative path) to search under, recursively.
|
|
pub path: Option<String>,
|
|
}
|
|
|
|
fn default_limit() -> usize {
|
|
20
|
|
}
|
|
fn default_threshold() -> f32 {
|
|
0.20
|
|
}
|
|
fn default_interpret() -> bool {
|
|
true
|
|
}
|
|
|
|
/// The client's explicit tag filter, applied as its own predicate on top of
|
|
/// the model's (ANY-mode) tags.
|
|
#[derive(Debug, Clone, PartialEq)]
|
|
struct ExplicitTags {
|
|
include: Vec<i32>,
|
|
exclude: Vec<i32>,
|
|
mode: FilterMode,
|
|
}
|
|
|
|
/// Parse a comma-separated id list, ignoring blanks.
|
|
fn parse_ids(raw: Option<&str>, name: &str) -> Result<Vec<i32>, String> {
|
|
raw.unwrap_or("")
|
|
.split(',')
|
|
.map(str::trim)
|
|
.filter(|s| !s.is_empty())
|
|
.map(|s| {
|
|
s.parse::<i32>()
|
|
.map_err(|_| format!("invalid id in `{name}`: {s}"))
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// Merge the explicit filters in `query` into the model's interpretation.
|
|
///
|
|
/// Returns `(effective, echo, tags, gps)`:
|
|
/// - `effective` is what the search runs on: explicit single-valued filters
|
|
/// replace the model's, and the model's tags lose any the client already
|
|
/// included or excluded.
|
|
/// - `echo` is the model's interpretation minus everything the explicit
|
|
/// filters overrode, so the client's "interpreted" chips only show what
|
|
/// the model added.
|
|
/// - `tags` is the explicit tag predicate, if any.
|
|
/// - `gps` is the explicit place circle, if any (it replaces `place`).
|
|
#[allow(clippy::type_complexity)]
|
|
fn merge_explicit(
|
|
sq: StructuredQuery,
|
|
query: &UnifiedQuery,
|
|
) -> Result<
|
|
(
|
|
StructuredQuery,
|
|
StructuredQuery,
|
|
Option<ExplicitTags>,
|
|
Option<(f64, f64, f64)>,
|
|
),
|
|
String,
|
|
> {
|
|
let include = parse_ids(query.tag_ids.as_deref(), "tag_ids")?;
|
|
let exclude = parse_ids(query.exclude_tag_ids.as_deref(), "exclude_tag_ids")?;
|
|
let tags = (!include.is_empty() || !exclude.is_empty()).then(|| ExplicitTags {
|
|
include: include.clone(),
|
|
exclude: exclude.clone(),
|
|
mode: query.tag_filter_mode.unwrap_or(FilterMode::All),
|
|
});
|
|
|
|
let gps = match (query.gps_lat, query.gps_lon, query.gps_radius_km) {
|
|
(Some(lat), Some(lon), Some(r)) => Some((lat, lon, r)),
|
|
(None, None, None) => None,
|
|
_ => return Err("GPS search requires gps_lat, gps_lon and gps_radius_km".into()),
|
|
};
|
|
|
|
let media = match query.media_type {
|
|
Some(MediaType::Photo) => Some("photo".to_string()),
|
|
Some(MediaType::Video) => Some("video".to_string()),
|
|
Some(MediaType::All) | None => None,
|
|
};
|
|
let explicit_date = query.date_from.is_some() || query.date_to.is_some();
|
|
let explicit_make = query
|
|
.camera_make
|
|
.as_deref()
|
|
.filter(|s| !s.trim().is_empty());
|
|
let explicit_model = query
|
|
.camera_model
|
|
.as_deref()
|
|
.filter(|s| !s.trim().is_empty());
|
|
|
|
let mut echo = sq;
|
|
echo.tag_ids
|
|
.retain(|id| !include.contains(id) && !exclude.contains(id));
|
|
echo.exclude_tag_ids
|
|
.retain(|id| !include.contains(id) && !exclude.contains(id));
|
|
let mut effective = echo.clone();
|
|
|
|
if let Some(m) = media {
|
|
echo.media_type = None;
|
|
effective.media_type = Some(m);
|
|
}
|
|
if explicit_date {
|
|
// A date range is one filter: an explicit "after X" replaces the
|
|
// model's whole range, not just its lower bound.
|
|
echo.date_from = None;
|
|
echo.date_to = None;
|
|
effective.date_from = query.date_from;
|
|
effective.date_to = query.date_to;
|
|
}
|
|
if let Some(make) = explicit_make {
|
|
echo.camera_make = None;
|
|
effective.camera_make = Some(make.to_string());
|
|
}
|
|
if let Some(model) = explicit_model {
|
|
echo.camera_model = None;
|
|
effective.camera_model = Some(model.to_string());
|
|
}
|
|
if gps.is_some() {
|
|
echo.place = None;
|
|
effective.place = None;
|
|
}
|
|
Ok((effective, echo, tags, gps))
|
|
}
|
|
|
|
/// Normalized folder prefix for the `path` filter (`None` = whole library).
|
|
fn folder_prefix(path: Option<&str>) -> Option<String> {
|
|
let p = path?.replace('\\', "/");
|
|
let p = p.trim_matches('/');
|
|
(!p.is_empty()).then(|| format!("{p}/"))
|
|
}
|
|
|
|
fn in_folder(file_path: &str, prefix: &str) -> bool {
|
|
file_path.replace('\\', "/").starts_with(prefix)
|
|
}
|
|
|
|
/// A geocoded place echoed back so the client can show / edit the location
|
|
/// filter it actually searched.
|
|
#[derive(Debug, Serialize)]
|
|
struct ResolvedPlace {
|
|
display_name: String,
|
|
lat: f64,
|
|
lon: f64,
|
|
radius_km: f64,
|
|
}
|
|
|
|
/// How the server interpreted the NL query — echoed to the client to render
|
|
/// editable filter chips. tag ids map to the client's existing tag list.
|
|
#[derive(Debug, Serialize)]
|
|
struct Interpreted {
|
|
semantic: Option<String>,
|
|
tag_ids: Vec<i32>,
|
|
exclude_tag_ids: Vec<i32>,
|
|
/// Words the model treated as tags that don't exist in the vocab; folded
|
|
/// into the semantic term and surfaced here so the UI can explain it.
|
|
unmatched_tags: Vec<String>,
|
|
camera_make: Option<String>,
|
|
camera_model: Option<String>,
|
|
lens_model: Option<String>,
|
|
date_from: Option<i64>,
|
|
date_to: Option<i64>,
|
|
media_type: Option<String>,
|
|
place: Option<ResolvedPlace>,
|
|
}
|
|
|
|
#[derive(Debug, Serialize)]
|
|
struct UnifiedResponse {
|
|
query: String,
|
|
interpreted: Interpreted,
|
|
/// CLIP model version used for ranking; `None` when the query had no
|
|
/// semantic term (filters-only).
|
|
model_version: Option<String>,
|
|
/// Embeddings scored by CLIP (0 when filters-only).
|
|
considered: usize,
|
|
/// Matches before pagination.
|
|
total_matching: usize,
|
|
offset: usize,
|
|
results: Vec<SearchHit>,
|
|
}
|
|
|
|
#[derive(Debug, Serialize)]
|
|
struct ErrorBody {
|
|
error: String,
|
|
}
|
|
|
|
fn bad_request(msg: impl Into<String>) -> HttpResponse {
|
|
HttpResponse::BadRequest().json(ErrorBody { error: msg.into() })
|
|
}
|
|
|
|
/// Combine the model's semantic term with any tag words that didn't match the
|
|
/// vocab, so a hallucinated/non-vocab tag becomes a soft semantic signal
|
|
/// rather than being dropped.
|
|
fn effective_semantic(sq: &StructuredQuery) -> Option<String> {
|
|
let mut parts: Vec<String> = Vec::new();
|
|
if let Some(s) = sq.semantic.as_deref() {
|
|
parts.push(s.to_string());
|
|
}
|
|
parts.extend(sq.unmatched_tags.iter().cloned());
|
|
if parts.is_empty() {
|
|
None
|
|
} else {
|
|
Some(parts.join(" "))
|
|
}
|
|
}
|
|
|
|
pub async fn unified_search<TagD: TagDao>(
|
|
_: Claims,
|
|
state: Data<AppState>,
|
|
exif_dao: Data<Mutex<Box<dyn ExifDao>>>,
|
|
tag_dao: Data<Mutex<TagD>>,
|
|
query: Query<UnifiedQuery>,
|
|
) -> HttpResponse {
|
|
let mut query = query.into_inner();
|
|
crate::date_view::apply_day_bounds(
|
|
&mut query.date_from,
|
|
&mut query.date_to,
|
|
query.date_from_day.as_deref(),
|
|
query.date_to_day.as_deref(),
|
|
|| crate::date_view::owner_display_zone(&state.user_ai_prefs_dao),
|
|
);
|
|
let nl = query.q.trim().to_string();
|
|
if nl.is_empty() {
|
|
return bad_request("query parameter `q` is required");
|
|
}
|
|
|
|
let limit = query.limit.clamp(1, 200);
|
|
let offset = query.offset;
|
|
let threshold = query.threshold.clamp(-1.0, 1.0);
|
|
|
|
let library_ids = match parse_library_scope(query.library_ids.as_deref(), query.library) {
|
|
Ok(ids) => ids,
|
|
Err(msg) => return bad_request(msg),
|
|
};
|
|
|
|
// ── 1. Translate the NL query, grounded on the real tag vocabulary ──
|
|
// (or take it verbatim as the CLIP term when interpretation is off).
|
|
let sq = if query.interpret {
|
|
match translate(&state, &tag_dao, &query, &nl).await {
|
|
Ok(sq) => sq,
|
|
Err(resp) => return resp,
|
|
}
|
|
} else {
|
|
StructuredQuery {
|
|
semantic: Some(nl.clone()),
|
|
..Default::default()
|
|
}
|
|
};
|
|
let (sq, echo, explicit_tags, explicit_gps) = match merge_explicit(sq, &query) {
|
|
Ok(merged) => merged,
|
|
Err(msg) => return bad_request(msg),
|
|
};
|
|
let folder = folder_prefix(query.path.as_deref());
|
|
|
|
// ── 2. Forward-geocode the place name into a gps circle ──
|
|
// (an explicit place from the client wins; merge_explicit dropped the
|
|
// model's place in that case, so no geocode call is spent on it).
|
|
let resolved_place = match sq.place.as_deref() {
|
|
Some(p) => forward_geocode(p).await.map(|g| ResolvedPlace {
|
|
display_name: g.display_name,
|
|
lat: g.lat,
|
|
lon: g.lon,
|
|
radius_km: g.radius_km,
|
|
}),
|
|
None => None,
|
|
};
|
|
let gps = explicit_gps.or(resolved_place.as_ref().map(|p| (p.lat, p.lon, p.radius_km)));
|
|
|
|
let semantic = effective_semantic(&sq);
|
|
|
|
let has_exif_filter = sq.camera_make.is_some()
|
|
|| sq.camera_model.is_some()
|
|
|| sq.lens_model.is_some()
|
|
|| sq.date_from.is_some()
|
|
|| sq.date_to.is_some();
|
|
let has_struct = has_exif_filter
|
|
|| gps.is_some()
|
|
|| !sq.tag_ids.is_empty()
|
|
|| sq.media_type.is_some()
|
|
|| explicit_tags.is_some()
|
|
|| folder.is_some();
|
|
let display_zone = crate::date_view::owner_display_zone(&state.user_ai_prefs_dao);
|
|
let server_zone = crate::camera_tz::server_zone();
|
|
|
|
// Stage trace: what the model extracted + whether a structured filter is
|
|
// active. The chips show this to the user too, but logging it makes the
|
|
// "why no results" path debuggable from the server side.
|
|
log::info!(
|
|
"unified_search: q={nl:?} semantic={:?} tag_ids={:?} exclude={:?} place={:?} gps={:?} date=({:?},{:?}) media={:?} unmatched={:?} has_struct={has_struct}",
|
|
sq.semantic,
|
|
sq.tag_ids,
|
|
sq.exclude_tag_ids,
|
|
resolved_place.as_ref().map(|p| p.display_name.as_str()),
|
|
gps,
|
|
sq.date_from,
|
|
sq.date_to,
|
|
sq.media_type,
|
|
sq.unmatched_tags,
|
|
);
|
|
|
|
// ── 3. Build the structured candidate set (EXIF rows passing every
|
|
// filter). Skipped entirely for a pure-semantic query. ──
|
|
let mut candidate: Vec<crate::database::models::ImageExif> = Vec::new();
|
|
let mut allowed_hashes: HashSet<String> = HashSet::new();
|
|
if has_struct {
|
|
// Tag membership set (rel_path only — same cross-library imprecision
|
|
// as the existing /photos tag listing). ANY-mode: a photo matches if
|
|
// it carries any of the named tags. ALL-mode over-constrains NL
|
|
// queries (the model maps several words to tags and few photos carry
|
|
// them all); the semantic term does the precision work instead.
|
|
let tag_set: Option<HashSet<String>> = if sq.tag_ids.is_empty() {
|
|
None
|
|
} else {
|
|
let mut dao = tag_dao.lock().expect("tag dao");
|
|
match dao.get_files_with_any_tag_ids(sq.tag_ids.clone(), sq.exclude_tag_ids.clone()) {
|
|
Ok(files) => Some(files.into_iter().map(|f| f.file_name).collect()),
|
|
Err(e) => {
|
|
log::warn!("unified_search: tag filter failed: {e:?}");
|
|
Some(HashSet::new())
|
|
}
|
|
}
|
|
};
|
|
// The client's explicit tags: their own predicate with the user's
|
|
// ALL/ANY choice. Exclude-only becomes a "must not carry" set.
|
|
let explicit_tag_filter: Option<(bool, HashSet<String>)> =
|
|
explicit_tags.as_ref().map(|t| {
|
|
let mut dao = tag_dao.lock().expect("tag dao");
|
|
let (must_contain, files) = if t.include.is_empty() {
|
|
(
|
|
false,
|
|
dao.get_files_with_any_tag_ids(t.exclude.clone(), Vec::new()),
|
|
)
|
|
} else if t.mode == FilterMode::Any {
|
|
(
|
|
true,
|
|
dao.get_files_with_any_tag_ids(t.include.clone(), t.exclude.clone()),
|
|
)
|
|
} else {
|
|
(
|
|
true,
|
|
dao.get_files_with_all_tag_ids(t.include.clone(), t.exclude.clone()),
|
|
)
|
|
};
|
|
let files = files.unwrap_or_else(|e| {
|
|
log::warn!("unified_search: explicit tag filter failed: {e:?}");
|
|
Vec::new()
|
|
});
|
|
(
|
|
must_contain,
|
|
files.into_iter().map(|f| f.file_name).collect(),
|
|
)
|
|
});
|
|
log::info!(
|
|
"unified_search: tag_ids={:?} -> tag_set_files={:?}",
|
|
sq.tag_ids,
|
|
tag_set.as_ref().map(|s| s.len())
|
|
);
|
|
|
|
// EXIF query handles camera/lens/gps-box/date. With no EXIF filters
|
|
// it returns the whole table, which we then narrow by the predicates
|
|
// below (tags / media / scope). Fine at personal-library scale.
|
|
let gps_bounds = gps.map(|(lat, lon, r)| gps_bounding_box(lat, lon, r));
|
|
let rows = {
|
|
let mut dao = exif_dao.lock().expect("exif dao");
|
|
dao.query_by_exif(
|
|
None, // scope filtered in-Rust to support multi-library
|
|
sq.camera_make.as_deref(),
|
|
sq.camera_model.as_deref(),
|
|
sq.lens_model.as_deref(),
|
|
gps_bounds,
|
|
crate::date_view::InstantRange::new(
|
|
sq.date_from,
|
|
sq.date_to,
|
|
display_zone,
|
|
server_zone,
|
|
),
|
|
)
|
|
.unwrap_or_else(|e| {
|
|
log::warn!("unified_search: query_by_exif failed: {e:?}");
|
|
Vec::new()
|
|
})
|
|
};
|
|
|
|
candidate = rows
|
|
.into_iter()
|
|
.filter(|row| {
|
|
// Library scope.
|
|
if !library_ids.is_empty() && !library_ids.contains(&row.library_id) {
|
|
return false;
|
|
}
|
|
// Precise GPS distance (the EXIF query only did a coarse box).
|
|
if let Some((lat, lon, radius_km)) = gps {
|
|
match (row.gps_latitude, row.gps_longitude) {
|
|
(Some(plat), Some(plon)) => {
|
|
if haversine_distance(lat, lon, plat as f64, plon as f64) > radius_km {
|
|
return false;
|
|
}
|
|
}
|
|
_ => return false,
|
|
}
|
|
}
|
|
// Media type.
|
|
if let Some(mt) = sq.media_type.as_deref() {
|
|
let p = Path::new(&row.file_path);
|
|
let ok = if mt == "video" {
|
|
is_video_file(p)
|
|
} else {
|
|
is_image_file(p)
|
|
};
|
|
if !ok {
|
|
return false;
|
|
}
|
|
}
|
|
// Tag membership.
|
|
if let Some(ts) = &tag_set
|
|
&& !ts.contains(&row.file_path)
|
|
{
|
|
return false;
|
|
}
|
|
if let Some((must_contain, ts)) = &explicit_tag_filter
|
|
&& ts.contains(&row.file_path) != *must_contain
|
|
{
|
|
return false;
|
|
}
|
|
// Folder scope.
|
|
if let Some(prefix) = &folder
|
|
&& !in_folder(&row.file_path, prefix)
|
|
{
|
|
return false;
|
|
}
|
|
true
|
|
})
|
|
.collect();
|
|
|
|
allowed_hashes = candidate
|
|
.iter()
|
|
.filter_map(|r| r.content_hash.clone())
|
|
.collect();
|
|
log::info!(
|
|
"unified_search: candidate_rows={} allowed_hashes={}",
|
|
candidate.len(),
|
|
allowed_hashes.len()
|
|
);
|
|
}
|
|
|
|
// ── 4. Rank ──
|
|
match semantic {
|
|
Some(ref sem) => {
|
|
// When structured filters are present they ARE the constraint —
|
|
// CLIP only ranks within the candidate set. So drop the global
|
|
// similarity threshold (it's tuned for whole-library search and
|
|
// would pre-discard filter-matching photos that scored just under
|
|
// it — e.g. a 2022 beach photo at 0.18 — before the intersection
|
|
// ever runs). With no filters, keep the user's threshold for the
|
|
// plain semantic case.
|
|
let clip_threshold = if has_struct { -1.0 } else { threshold };
|
|
let scored = match score_photos(
|
|
&state,
|
|
&exif_dao,
|
|
sem,
|
|
&library_ids,
|
|
clip_threshold,
|
|
None,
|
|
)
|
|
.await
|
|
{
|
|
Ok(s) => s,
|
|
Err(e) => return score_error_response(e),
|
|
};
|
|
let considered = scored.considered;
|
|
let clip_hits = scored.hits.len();
|
|
let hits: Vec<(f32, String)> = if has_struct {
|
|
scored
|
|
.hits
|
|
.into_iter()
|
|
.filter(|(_, h)| allowed_hashes.contains(h))
|
|
.collect()
|
|
} else {
|
|
scored.hits
|
|
};
|
|
log::info!(
|
|
"unified_search: clip considered={considered} hits={clip_hits} after_struct_filter={}",
|
|
hits.len()
|
|
);
|
|
let total_matching = hits.len();
|
|
let page = paginate(&hits, offset, limit);
|
|
let results = resolve_hits(&exif_dao, &page);
|
|
HttpResponse::Ok().json(UnifiedResponse {
|
|
query: nl,
|
|
interpreted: interpreted(&echo, resolved_place),
|
|
model_version: Some(scored.model_version),
|
|
considered: scored.considered,
|
|
total_matching,
|
|
offset,
|
|
results,
|
|
})
|
|
}
|
|
None => {
|
|
// Filters-only: no semantic term. Require at least one filter,
|
|
// then return the candidate set newest-first.
|
|
if !has_struct {
|
|
return bad_request("query had no searchable terms");
|
|
}
|
|
candidate.sort_by(|a, b| b.date_taken.cmp(&a.date_taken));
|
|
let total_matching = candidate.len();
|
|
log::info!("unified_search: filters-only matches={total_matching}");
|
|
let end = (offset + limit).min(total_matching);
|
|
let results: Vec<SearchHit> = if offset >= total_matching {
|
|
Vec::new()
|
|
} else {
|
|
candidate[offset..end]
|
|
.iter()
|
|
.map(|r| SearchHit {
|
|
library_id: r.library_id,
|
|
rel_path: r.file_path.clone(),
|
|
content_hash: r.content_hash.clone().unwrap_or_default(),
|
|
score: 0.0,
|
|
})
|
|
.collect()
|
|
};
|
|
HttpResponse::Ok().json(UnifiedResponse {
|
|
query: nl,
|
|
interpreted: interpreted(&echo, resolved_place),
|
|
model_version: None,
|
|
considered: 0,
|
|
total_matching,
|
|
offset,
|
|
results,
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Translate `nl` into a structured query with the local LLM, grounded on the
|
|
/// real tag vocabulary. Errors are ready-to-send responses.
|
|
async fn translate<TagD: TagDao>(
|
|
state: &AppState,
|
|
tag_dao: &Mutex<TagD>,
|
|
query: &UnifiedQuery,
|
|
nl: &str,
|
|
) -> Result<StructuredQuery, HttpResponse> {
|
|
let tag_vocab: Vec<(i32, String)> = {
|
|
let mut dao = tag_dao.lock().expect("tag dao");
|
|
match dao.get_all_tags(None) {
|
|
Ok(tags) => tags.into_iter().map(|(_, t)| (t.id, t.name)).collect(),
|
|
Err(e) => {
|
|
log::warn!("unified_search: get_all_tags failed: {e:?}");
|
|
Vec::new()
|
|
}
|
|
}
|
|
};
|
|
|
|
// Respect env/config for the LLM backend (LLM_BACKEND → ollama or
|
|
// llama-swap); local only, no hybrid, per the feature's design.
|
|
//
|
|
// Translation-model precedence:
|
|
// 1. UNIFIED_SEARCH_MODEL env — pin a small, fast model that can stay
|
|
// co-resident with CLIP (and the chat model) so translation never
|
|
// evicts them. This is the recommended setup on a tight VRAM budget.
|
|
// 2. the client-selected model — routes translation to whatever the user
|
|
// already has loaded (no swap) when no dedicated model is pinned.
|
|
// 3. None → resolve_backend uses the configured default local model.
|
|
let translation_model = std::env::var("UNIFIED_SEARCH_MODEL")
|
|
.ok()
|
|
.filter(|m| !m.trim().is_empty())
|
|
.or_else(|| query.model.clone())
|
|
.filter(|m| !m.trim().is_empty());
|
|
let overrides = SamplingOverrides {
|
|
model: translation_model,
|
|
num_ctx: None,
|
|
temperature: None,
|
|
top_p: None,
|
|
top_k: None,
|
|
min_p: None,
|
|
enable_thinking: None,
|
|
};
|
|
let backend = match state
|
|
.insight_generator
|
|
.resolve_backend(BackendKind::Local, &overrides)
|
|
.await
|
|
{
|
|
Ok(b) => b,
|
|
Err(e) => {
|
|
log::warn!("unified_search: resolve_backend failed: {e:?}");
|
|
return Err(HttpResponse::ServiceUnavailable().json(ErrorBody {
|
|
error: "LLM backend unavailable".into(),
|
|
}));
|
|
}
|
|
};
|
|
log::info!("unified_search: translating with model={}", backend.model());
|
|
|
|
// The owner's day and zone, not UTC's: relative dates ("last week")
|
|
// anchor on the owner's today, and ISO dates become display-zone day
|
|
// bounds that the true-instant filters compare against (#152).
|
|
let display_zone = crate::date_view::owner_display_zone(&state.user_ai_prefs_dao);
|
|
let today = chrono::Utc::now().with_timezone(&display_zone).date_naive();
|
|
translate_nl_query(backend.chat(), nl, &tag_vocab, today, display_zone)
|
|
.await
|
|
.map_err(|e| {
|
|
log::warn!("unified_search: translate_nl_query failed: {e:?}");
|
|
HttpResponse::BadGateway().json(ErrorBody {
|
|
error: "could not interpret the query".into(),
|
|
})
|
|
})
|
|
}
|
|
|
|
/// Slice a sorted hit list at `[offset, offset+limit)`, tolerating
|
|
/// out-of-range offsets (empty page).
|
|
fn paginate(hits: &[(f32, String)], offset: usize, limit: usize) -> Vec<(f32, String)> {
|
|
if offset >= hits.len() {
|
|
return Vec::new();
|
|
}
|
|
let end = (offset + limit).min(hits.len());
|
|
hits[offset..end].to_vec()
|
|
}
|
|
|
|
fn interpreted(sq: &StructuredQuery, place: Option<ResolvedPlace>) -> Interpreted {
|
|
Interpreted {
|
|
semantic: sq.semantic.clone(),
|
|
tag_ids: sq.tag_ids.clone(),
|
|
exclude_tag_ids: sq.exclude_tag_ids.clone(),
|
|
unmatched_tags: sq.unmatched_tags.clone(),
|
|
camera_make: sq.camera_make.clone(),
|
|
camera_model: sq.camera_model.clone(),
|
|
lens_model: sq.lens_model.clone(),
|
|
date_from: sq.date_from,
|
|
date_to: sq.date_to,
|
|
media_type: sq.media_type.clone(),
|
|
place,
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::ai::nl_query::StructuredQuery;
|
|
|
|
#[test]
|
|
fn effective_semantic_combines_semantic_and_unmatched() {
|
|
let sq = StructuredQuery {
|
|
semantic: Some("sunset".into()),
|
|
unmatched_tags: vec!["golden hour".into()],
|
|
..Default::default()
|
|
};
|
|
assert_eq!(
|
|
effective_semantic(&sq).as_deref(),
|
|
Some("sunset golden hour")
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn effective_semantic_none_when_empty() {
|
|
let sq = StructuredQuery::default();
|
|
assert_eq!(effective_semantic(&sq), None);
|
|
}
|
|
|
|
#[test]
|
|
fn effective_semantic_unmatched_only() {
|
|
let sq = StructuredQuery {
|
|
unmatched_tags: vec!["disco".into()],
|
|
..Default::default()
|
|
};
|
|
assert_eq!(effective_semantic(&sq).as_deref(), Some("disco"));
|
|
}
|
|
|
|
fn parse(qs: &str) -> UnifiedQuery {
|
|
Query::<UnifiedQuery>::from_query(qs)
|
|
.expect("valid query string")
|
|
.into_inner()
|
|
}
|
|
|
|
fn model_guess() -> StructuredQuery {
|
|
StructuredQuery {
|
|
semantic: Some("dog".into()),
|
|
tag_ids: vec![1, 2],
|
|
exclude_tag_ids: vec![3],
|
|
camera_make: Some("Canon".into()),
|
|
date_from: Some(100),
|
|
date_to: Some(200),
|
|
media_type: Some("photo".into()),
|
|
place: Some("Paris".into()),
|
|
..Default::default()
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn query_defaults_to_interpreting_with_no_explicit_filters() {
|
|
let q = parse("q=dog");
|
|
assert!(q.interpret);
|
|
let (effective, echo, tags, gps) = merge_explicit(model_guess(), &q).unwrap();
|
|
assert_eq!(effective, model_guess());
|
|
assert_eq!(echo, model_guess());
|
|
assert_eq!(tags, None);
|
|
assert_eq!(gps, None);
|
|
}
|
|
|
|
#[test]
|
|
fn explicit_filters_replace_the_models_and_leave_the_echo() {
|
|
let q = parse(
|
|
"q=dog&interpret=false&media_type=video&date_from=500&camera_model=Nexus%206\
|
|
&gps_lat=1.5&gps_lon=2.5&gps_radius_km=10",
|
|
);
|
|
assert!(!q.interpret);
|
|
let (effective, echo, _, gps) = merge_explicit(model_guess(), &q).unwrap();
|
|
|
|
assert_eq!(effective.media_type.as_deref(), Some("video"));
|
|
// A date range is one filter: the model's upper bound goes too.
|
|
assert_eq!((effective.date_from, effective.date_to), (Some(500), None));
|
|
assert_eq!(effective.camera_model.as_deref(), Some("Nexus 6"));
|
|
// Not overridden: the model's make still applies.
|
|
assert_eq!(effective.camera_make.as_deref(), Some("Canon"));
|
|
assert_eq!(effective.place, None);
|
|
assert_eq!(gps, Some((1.5, 2.5, 10.0)));
|
|
|
|
// The echo keeps only what the model added on top.
|
|
assert_eq!(echo.media_type, None);
|
|
assert_eq!((echo.date_from, echo.date_to), (None, None));
|
|
assert_eq!(echo.camera_model, None);
|
|
assert_eq!(echo.camera_make.as_deref(), Some("Canon"));
|
|
assert_eq!(echo.place, None);
|
|
}
|
|
|
|
#[test]
|
|
fn explicit_tags_become_their_own_predicate() {
|
|
let q = parse("q=dog&tag_ids=2,7&exclude_tag_ids=1&tag_filter_mode=Any");
|
|
let (effective, echo, tags, _) = merge_explicit(model_guess(), &q).unwrap();
|
|
assert_eq!(
|
|
tags,
|
|
Some(ExplicitTags {
|
|
include: vec![2, 7],
|
|
exclude: vec![1],
|
|
mode: FilterMode::Any,
|
|
})
|
|
);
|
|
// The model's guesses lose tags the user already decided on.
|
|
assert!(effective.tag_ids.is_empty());
|
|
assert!(echo.tag_ids.is_empty());
|
|
assert_eq!(echo.exclude_tag_ids, vec![3]);
|
|
}
|
|
|
|
#[test]
|
|
fn explicit_tags_default_to_all_mode() {
|
|
let q = parse("q=dog&tag_ids=5");
|
|
let (_, _, tags, _) = merge_explicit(StructuredQuery::default(), &q).unwrap();
|
|
assert_eq!(tags.unwrap().mode, FilterMode::All);
|
|
}
|
|
|
|
#[test]
|
|
fn media_type_all_is_no_filter() {
|
|
let q = parse("q=dog&media_type=all");
|
|
let (effective, _, _, _) = merge_explicit(model_guess(), &q).unwrap();
|
|
assert_eq!(effective.media_type.as_deref(), Some("photo"));
|
|
}
|
|
|
|
#[test]
|
|
fn partial_gps_and_bad_ids_are_rejected() {
|
|
assert!(merge_explicit(StructuredQuery::default(), &parse("q=a&gps_lat=1")).is_err());
|
|
assert!(merge_explicit(StructuredQuery::default(), &parse("q=a&tag_ids=1,x")).is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn folder_prefix_matches_whole_segments() {
|
|
assert_eq!(folder_prefix(None), None);
|
|
assert_eq!(folder_prefix(Some("/")), None);
|
|
let p = folder_prefix(Some("2019\\Trip/")).unwrap();
|
|
assert_eq!(p, "2019/Trip/");
|
|
assert!(in_folder("2019/Trip/a.jpg", &p));
|
|
assert!(in_folder("2019\\Trip\\sub\\b.jpg", &p));
|
|
assert!(!in_folder("2019/Trips/c.jpg", &p));
|
|
}
|
|
|
|
#[test]
|
|
fn paginate_handles_out_of_range_offset() {
|
|
let hits = vec![(0.9, "a".to_string()), (0.8, "b".to_string())];
|
|
assert_eq!(paginate(&hits, 5, 10).len(), 0);
|
|
assert_eq!(paginate(&hits, 0, 1).len(), 1);
|
|
assert_eq!(paginate(&hits, 1, 10).len(), 1);
|
|
}
|
|
}
|