Apps/ImageApi
1
0 Fork 0
Code Issues 7 Pull Requests Releases Wiki Activity

feat(search): explicit filters on unified search; list camera bodies

Browse Source 
The app's new search screen keeps tags, type, date, camera and place as
chips and sends free text to the unified endpoint. For the AI to add to
those chips instead of replacing them, the endpoint now takes them:

- /photos/search/unified accepts the /photos filters explicitly (tag_ids
  with All/Any, exclude_tag_ids, media_type, dates, camera make/model,
  gps circle) plus a recursive `path`. Explicit tags are their own
  predicate on top of the model's ANY-mode tags; single-valued filters
  replace the model's guess. `interpreted` echoes only what the model
  added, so the client doesn't show the user's own chips twice.
- interpret=false skips the LLM and uses q verbatim as the CLIP term,
  which replaces the app's separate "Semantic" mode with a visual match
  inside the user's filters.
- /image/tags/all also returns `cameras` (make + model with counts) so
  the camera picker can offer e.g. "Nexus 6" directly.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Cameron CordesandClaude Opus 5.5 committed 2026-10-15 21:09:06 -04:00
1 parent ed52a07d9d
commit 015ddd6c77
4 files changed
+511 -69
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/unified_search.rs
+397 -68
View File
@@ -25,7 +25,7 @@ use crate::ai::nl_query::{StructuredQuery, translate_nl_query};
use crate::clip_search::{
SearchHit, parse_library_scope, resolve_hits, score_error_response, score_photos,
};
use crate::data::Claims;
use crate::data::{Claims, FilterMode, MediaType};
use crate::database::ExifDao;
use crate::file_types::{is_image_file, is_video_file};
use crate::geo::{forward_geocode, gps_bounding_box, haversine_distance};
@@ -58,6 +58,32 @@ pub struct UnifiedQuery {
/// (avoids a llama-swap eviction / cold start). Falls back to the configured
/// default local model when absent. Local only — no hybrid here.
pub model: Option<String>,
/// When false, skip the LLM translation and use `q` verbatim as the CLIP
/// term ("visual match only"). Explicit filters below still apply.
#[serde(default = "default_interpret")]
pub interpret: bool,
// ── Explicit filters ──
// Set by the client's filter chips. They're AND-ed with whatever the
// model extracted from `q`; for single-valued filters (camera, date,
// media type, place) the explicit value replaces the model's guess.
// Same encodings as `/photos`.
/// Comma-separated tag ids the results must carry (see `tag_filter_mode`).
pub tag_ids: Option<String>,
/// Comma-separated tag ids the results must not carry.
pub exclude_tag_ids: Option<String>,
/// `All` (default) or `Any` for `tag_ids`.
pub tag_filter_mode: Option<FilterMode>,
pub media_type: Option<MediaType>,
pub date_from: Option<i64>,
pub date_to: Option<i64>,
pub camera_make: Option<String>,
pub camera_model: Option<String>,
pub gps_lat: Option<f64>,
pub gps_lon: Option<f64>,
pub gps_radius_km: Option<f64>,
/// Folder (relative path) to search under, recursively.
pub path: Option<String>,
}
fn default_limit() -> usize {
@@ -66,6 +92,129 @@ fn default_limit() -> usize {
fn default_threshold() -> f32 {
0.20
}
fn default_interpret() -> bool {
true
}
/// The client's explicit tag filter, applied as its own predicate on top of
/// the model's (ANY-mode) tags.
#[derive(Debug, Clone, PartialEq)]
struct ExplicitTags {
include: Vec<i32>,
exclude: Vec<i32>,
mode: FilterMode,
}
/// Parse a comma-separated id list, ignoring blanks.
fn parse_ids(raw: Option<&str>, name: &str) -> Result<Vec<i32>, String> {
raw.unwrap_or("")
.split(',')
.map(str::trim)
.filter(|s| !s.is_empty())
.map(|s| {
s.parse::<i32>()
.map_err(|_| format!("invalid id in `{name}`: {s}"))
})
.collect()
}
/// Merge the explicit filters in `query` into the model's interpretation.
///
/// Returns `(effective, echo, tags, gps)`:
/// - `effective` is what the search runs on: explicit single-valued filters
/// replace the model's, and the model's tags lose any the client already
/// included or excluded.
/// - `echo` is the model's interpretation minus everything the explicit
/// filters overrode, so the client's "interpreted" chips only show what
/// the model added.
/// - `tags` is the explicit tag predicate, if any.
/// - `gps` is the explicit place circle, if any (it replaces `place`).
#[allow(clippy::type_complexity)]
fn merge_explicit(
sq: StructuredQuery,
query: &UnifiedQuery,
) -> Result<
(
StructuredQuery,
StructuredQuery,
Option<ExplicitTags>,
Option<(f64, f64, f64)>,
),
String,
> {
let include = parse_ids(query.tag_ids.as_deref(), "tag_ids")?;
let exclude = parse_ids(query.exclude_tag_ids.as_deref(), "exclude_tag_ids")?;
let tags = (!include.is_empty() || !exclude.is_empty()).then(|| ExplicitTags {
include: include.clone(),
exclude: exclude.clone(),
mode: query.tag_filter_mode.unwrap_or(FilterMode::All),
});
let gps = match (query.gps_lat, query.gps_lon, query.gps_radius_km) {
(Some(lat), Some(lon), Some(r)) => Some((lat, lon, r)),
(None, None, None) => None,
_ => return Err("GPS search requires gps_lat, gps_lon and gps_radius_km".into()),
};
let media = match query.media_type {
Some(MediaType::Photo) => Some("photo".to_string()),
Some(MediaType::Video) => Some("video".to_string()),
Some(MediaType::All) | None => None,
};
let explicit_date = query.date_from.is_some() || query.date_to.is_some();
let explicit_make = query
.camera_make
.as_deref()
.filter(|s| !s.trim().is_empty());
let explicit_model = query
.camera_model
.as_deref()
.filter(|s| !s.trim().is_empty());
let mut echo = sq;
echo.tag_ids
.retain(|id| !include.contains(id) && !exclude.contains(id));
echo.exclude_tag_ids
.retain(|id| !include.contains(id) && !exclude.contains(id));
let mut effective = echo.clone();
if let Some(m) = media {
echo.media_type = None;
effective.media_type = Some(m);
}
if explicit_date {
// A date range is one filter: an explicit "after X" replaces the
// model's whole range, not just its lower bound.
echo.date_from = None;
echo.date_to = None;
effective.date_from = query.date_from;
effective.date_to = query.date_to;
}
if let Some(make) = explicit_make {
echo.camera_make = None;
effective.camera_make = Some(make.to_string());
}
if let Some(model) = explicit_model {
echo.camera_model = None;
effective.camera_model = Some(model.to_string());
}
if gps.is_some() {
echo.place = None;
effective.place = None;
}
Ok((effective, echo, tags, gps))
}
/// Normalized folder prefix for the `path` filter (`None` = whole library).
fn folder_prefix(path: Option<&str>) -> Option<String> {
let p = path?.replace('\\', "/");
let p = p.trim_matches('/');
(!p.is_empty()).then(|| format!("{p}/"))
}
fn in_folder(file_path: &str, prefix: &str) -> bool {
file_path.replace('\\', "/").starts_with(prefix)
}
/// A geocoded place echoed back so the client can show / edit the location
/// filter it actually searched.
@@ -158,73 +307,27 @@ pub async fn unified_search<TagD: TagDao>(
};
// ── 1. Translate the NL query, grounded on the real tag vocabulary ──
let tag_vocab: Vec<(i32, String)> = {
let mut dao = tag_dao.lock().expect("tag dao");
match dao.get_all_tags(None) {
Ok(tags) => tags.into_iter().map(|(_, t)| (t.id, t.name)).collect(),
Err(e) => {
log::warn!("unified_search: get_all_tags failed: {e:?}");
Vec::new()
}
// (or take it verbatim as the CLIP term when interpretation is off).
let sq = if query.interpret {
match translate(&state, &tag_dao, &query, &nl).await {
Ok(sq) => sq,
Err(resp) => return resp,
}
} else {
StructuredQuery {
semantic: Some(nl.clone()),
..Default::default()
}
};
// Respect env/config for the LLM backend (LLM_BACKEND → ollama or
// llama-swap); local only, no hybrid, per the feature's design.
//
// Translation-model precedence:
// 1. UNIFIED_SEARCH_MODEL env — pin a small, fast model that can stay
// co-resident with CLIP (and the chat model) so translation never
// evicts them. This is the recommended setup on a tight VRAM budget.
// 2. the client-selected model — routes translation to whatever the user
// already has loaded (no swap) when no dedicated model is pinned.
// 3. None → resolve_backend uses the configured default local model.
let translation_model = std::env::var("UNIFIED_SEARCH_MODEL")
.ok()
.filter(|m| !m.trim().is_empty())
.or_else(|| query.model.clone())
.filter(|m| !m.trim().is_empty());
let overrides = SamplingOverrides {
model: translation_model,
num_ctx: None,
temperature: None,
top_p: None,
top_k: None,
min_p: None,
enable_thinking: None,
};
let backend = match state
.insight_generator
.resolve_backend(BackendKind::Local, &overrides)
.await
{
Ok(b) => b,
Err(e) => {
log::warn!("unified_search: resolve_backend failed: {e:?}");
return HttpResponse::ServiceUnavailable().json(ErrorBody {
error: "LLM backend unavailable".into(),
});
}
};
log::info!("unified_search: translating with model={}", backend.model());
// The owner's day and zone, not UTC's: relative dates ("last week")
// anchor on the owner's today, and ISO dates become display-zone day
// bounds that the true-instant filters compare against (#152).
let display_zone = crate::date_view::owner_display_zone(&state.user_ai_prefs_dao);
let server_zone = crate::camera_tz::server_zone();
let today = chrono::Utc::now().with_timezone(&display_zone).date_naive();
let sq = match translate_nl_query(backend.chat(), &nl, &tag_vocab, today, display_zone).await {
Ok(sq) => sq,
Err(e) => {
log::warn!("unified_search: translate_nl_query failed: {e:?}");
return HttpResponse::BadGateway().json(ErrorBody {
error: "could not interpret the query".into(),
});
}
let (sq, echo, explicit_tags, explicit_gps) = match merge_explicit(sq, &query) {
Ok(merged) => merged,
Err(msg) => return bad_request(msg),
};
let folder = folder_prefix(query.path.as_deref());
// ── 2. Forward-geocode the place name into a gps circle ──
// (an explicit place from the client wins; merge_explicit dropped the
// model's place in that case, so no geocode call is spent on it).
let resolved_place = match sq.place.as_deref() {
Some(p) => forward_geocode(p).await.map(|g| ResolvedPlace {
display_name: g.display_name,
@@ -234,7 +337,7 @@ pub async fn unified_search<TagD: TagDao>(
}),
None => None,
};
let gps = resolved_place.as_ref().map(|p| (p.lat, p.lon, p.radius_km));
let gps = explicit_gps.or(resolved_place.as_ref().map(|p| (p.lat, p.lon, p.radius_km)));
let semantic = effective_semantic(&sq);
@@ -243,8 +346,14 @@ pub async fn unified_search<TagD: TagDao>(
|| sq.lens_model.is_some()
|| sq.date_from.is_some()
|| sq.date_to.is_some();
let has_struct =
has_exif_filter || gps.is_some() || !sq.tag_ids.is_empty() || sq.media_type.is_some();
let has_struct = has_exif_filter
|| gps.is_some()
|| !sq.tag_ids.is_empty()
|| sq.media_type.is_some()
|| explicit_tags.is_some()
|| folder.is_some();
let display_zone = crate::date_view::owner_display_zone(&state.user_ai_prefs_dao);
let server_zone = crate::camera_tz::server_zone();
// Stage trace: what the model extracted + whether a structured filter is
// active. The chips show this to the user too, but logging it makes the
@@ -284,6 +393,36 @@ pub async fn unified_search<TagD: TagDao>(
}
}
};
// The client's explicit tags: their own predicate with the user's
// ALL/ANY choice. Exclude-only becomes a "must not carry" set.
let explicit_tag_filter: Option<(bool, HashSet<String>)> =
explicit_tags.as_ref().map(|t| {
let mut dao = tag_dao.lock().expect("tag dao");
let (must_contain, files) = if t.include.is_empty() {
(
false,
dao.get_files_with_any_tag_ids(t.exclude.clone(), Vec::new()),
)
} else if t.mode == FilterMode::Any {
(
true,
dao.get_files_with_any_tag_ids(t.include.clone(), t.exclude.clone()),
)
} else {
(
true,
dao.get_files_with_all_tag_ids(t.include.clone(), t.exclude.clone()),
)
};
let files = files.unwrap_or_else(|e| {
log::warn!("unified_search: explicit tag filter failed: {e:?}");
Vec::new()
});
(
must_contain,
files.into_iter().map(|f| f.file_name).collect(),
)
});
log::info!(
"unified_search: tag_ids={:?} -> tag_set_files={:?}",
sq.tag_ids,
@@ -351,6 +490,17 @@ pub async fn unified_search<TagD: TagDao>(
{
return false;
}
if let Some((must_contain, ts)) = &explicit_tag_filter
&& ts.contains(&row.file_path) != *must_contain
{
return false;
}
// Folder scope.
if let Some(prefix) = &folder
&& !in_folder(&row.file_path, prefix)
{
return false;
}
true
})
.collect();
@@ -410,7 +560,7 @@ pub async fn unified_search<TagD: TagDao>(
let results = resolve_hits(&exif_dao, &page);
HttpResponse::Ok().json(UnifiedResponse {
query: nl,
interpreted: interpreted(&sq, resolved_place),
interpreted: interpreted(&echo, resolved_place),
model_version: Some(scored.model_version),
considered: scored.considered,
total_matching,
@@ -443,7 +593,7 @@ pub async fn unified_search<TagD: TagDao>(
};
HttpResponse::Ok().json(UnifiedResponse {
query: nl,
interpreted: interpreted(&sq, resolved_place),
interpreted: interpreted(&echo, resolved_place),
model_version: None,
considered: 0,
total_matching,
@@ -454,6 +604,79 @@ pub async fn unified_search<TagD: TagDao>(
}
}
/// Translate `nl` into a structured query with the local LLM, grounded on the
/// real tag vocabulary. Errors are ready-to-send responses.
async fn translate<TagD: TagDao>(
state: &AppState,
tag_dao: &Mutex<TagD>,
query: &UnifiedQuery,
nl: &str,
) -> Result<StructuredQuery, HttpResponse> {
let tag_vocab: Vec<(i32, String)> = {
let mut dao = tag_dao.lock().expect("tag dao");
match dao.get_all_tags(None) {
Ok(tags) => tags.into_iter().map(|(_, t)| (t.id, t.name)).collect(),
Err(e) => {
log::warn!("unified_search: get_all_tags failed: {e:?}");
Vec::new()
}
}
};
// Respect env/config for the LLM backend (LLM_BACKEND → ollama or
// llama-swap); local only, no hybrid, per the feature's design.
//
// Translation-model precedence:
// 1. UNIFIED_SEARCH_MODEL env — pin a small, fast model that can stay
// co-resident with CLIP (and the chat model) so translation never
// evicts them. This is the recommended setup on a tight VRAM budget.
// 2. the client-selected model — routes translation to whatever the user
// already has loaded (no swap) when no dedicated model is pinned.
// 3. None → resolve_backend uses the configured default local model.
let translation_model = std::env::var("UNIFIED_SEARCH_MODEL")
.ok()
.filter(|m| !m.trim().is_empty())
.or_else(|| query.model.clone())
.filter(|m| !m.trim().is_empty());
let overrides = SamplingOverrides {
model: translation_model,
num_ctx: None,
temperature: None,
top_p: None,
top_k: None,
min_p: None,
enable_thinking: None,
};
let backend = match state
.insight_generator
.resolve_backend(BackendKind::Local, &overrides)
.await
{
Ok(b) => b,
Err(e) => {
log::warn!("unified_search: resolve_backend failed: {e:?}");
return Err(HttpResponse::ServiceUnavailable().json(ErrorBody {
error: "LLM backend unavailable".into(),
}));
}
};
log::info!("unified_search: translating with model={}", backend.model());
// The owner's day and zone, not UTC's: relative dates ("last week")
// anchor on the owner's today, and ISO dates become display-zone day
// bounds that the true-instant filters compare against (#152).
let display_zone = crate::date_view::owner_display_zone(&state.user_ai_prefs_dao);
let today = chrono::Utc::now().with_timezone(&display_zone).date_naive();
translate_nl_query(backend.chat(), nl, &tag_vocab, today, display_zone)
.await
.map_err(|e| {
log::warn!("unified_search: translate_nl_query failed: {e:?}");
HttpResponse::BadGateway().json(ErrorBody {
error: "could not interpret the query".into(),
})
})
}
/// Slice a sorted hit list at `[offset, offset+limit)`, tolerating
/// out-of-range offsets (empty page).
fn paginate(hits: &[(f32, String)], offset: usize, limit: usize) -> Vec<(f32, String)> {
@@ -513,6 +736,112 @@ mod tests {
assert_eq!(effective_semantic(&sq).as_deref(), Some("disco"));
}
fn parse(qs: &str) -> UnifiedQuery {
Query::<UnifiedQuery>::from_query(qs)
.expect("valid query string")
.into_inner()
}
fn model_guess() -> StructuredQuery {
StructuredQuery {
semantic: Some("dog".into()),
tag_ids: vec![1, 2],
exclude_tag_ids: vec![3],
camera_make: Some("Canon".into()),
date_from: Some(100),
date_to: Some(200),
media_type: Some("photo".into()),
place: Some("Paris".into()),
..Default::default()
}
}
#[test]
fn query_defaults_to_interpreting_with_no_explicit_filters() {
let q = parse("q=dog");
assert!(q.interpret);
let (effective, echo, tags, gps) = merge_explicit(model_guess(), &q).unwrap();
assert_eq!(effective, model_guess());
assert_eq!(echo, model_guess());
assert_eq!(tags, None);
assert_eq!(gps, None);
}
#[test]
fn explicit_filters_replace_the_models_and_leave_the_echo() {
let q = parse(
"q=dog&interpret=false&media_type=video&date_from=500&camera_model=Nexus%206\
&gps_lat=1.5&gps_lon=2.5&gps_radius_km=10",
);
assert!(!q.interpret);
let (effective, echo, _, gps) = merge_explicit(model_guess(), &q).unwrap();
assert_eq!(effective.media_type.as_deref(), Some("video"));
// A date range is one filter: the model's upper bound goes too.
assert_eq!((effective.date_from, effective.date_to), (Some(500), None));
assert_eq!(effective.camera_model.as_deref(), Some("Nexus 6"));
// Not overridden: the model's make still applies.
assert_eq!(effective.camera_make.as_deref(), Some("Canon"));
assert_eq!(effective.place, None);
assert_eq!(gps, Some((1.5, 2.5, 10.0)));
// The echo keeps only what the model added on top.
assert_eq!(echo.media_type, None);
assert_eq!((echo.date_from, echo.date_to), (None, None));
assert_eq!(echo.camera_model, None);
assert_eq!(echo.camera_make.as_deref(), Some("Canon"));
assert_eq!(echo.place, None);
}
#[test]
fn explicit_tags_become_their_own_predicate() {
let q = parse("q=dog&tag_ids=2,7&exclude_tag_ids=1&tag_filter_mode=Any");
let (effective, echo, tags, _) = merge_explicit(model_guess(), &q).unwrap();
assert_eq!(
tags,
Some(ExplicitTags {
include: vec![2, 7],
exclude: vec![1],
mode: FilterMode::Any,
})
);
// The model's guesses lose tags the user already decided on.
assert!(effective.tag_ids.is_empty());
assert!(echo.tag_ids.is_empty());
assert_eq!(echo.exclude_tag_ids, vec![3]);
}
#[test]
fn explicit_tags_default_to_all_mode() {
let q = parse("q=dog&tag_ids=5");
let (_, _, tags, _) = merge_explicit(StructuredQuery::default(), &q).unwrap();
assert_eq!(tags.unwrap().mode, FilterMode::All);
}
#[test]
fn media_type_all_is_no_filter() {
let q = parse("q=dog&media_type=all");
let (effective, _, _, _) = merge_explicit(model_guess(), &q).unwrap();
assert_eq!(effective.media_type.as_deref(), Some("photo"));
}
#[test]
fn partial_gps_and_bad_ids_are_rejected() {
assert!(merge_explicit(StructuredQuery::default(), &parse("q=a&gps_lat=1")).is_err());
assert!(merge_explicit(StructuredQuery::default(), &parse("q=a&tag_ids=1,x")).is_err());
}
#[test]
fn folder_prefix_matches_whole_segments() {
assert_eq!(folder_prefix(None), None);
assert_eq!(folder_prefix(Some("/")), None);
let p = folder_prefix(Some("2019\\Trip/")).unwrap();
assert_eq!(p, "2019/Trip/");
assert!(in_folder("2019/Trip/a.jpg", &p));
assert!(in_folder("2019\\Trip\\sub\\b.jpg", &p));
assert!(!in_folder("2019/Trips/c.jpg", &p));
}
#[test]
fn paginate_handles_out_of_range_offset() {
let hits = vec![(0.9, "a".to_string()), (0.8, "b".to_string())];