diff --git a/CLAUDE.md b/CLAUDE.md index b10d25e..db4b757 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -437,6 +437,26 @@ a stronger tool comes online (exiftool install, new filename regex), re-resolve out-of-band rather than re-introducing the steady-state eligibility. +**True-instant drain (`backfill_utc_dates`, ticket #118).** Runs right +after `backfill_missing_date_taken` inside the per-library online gate, +sharing its `DATE_BACKFILL_MAX_PER_TICK` cap. Converts legacy +mixed-convention rows (`date_utc IS NULL AND date_taken IS NOT NULL`, +partial index `idx_image_exif_utc_backfill`) by re-running +`resolve_dates_batch` with empty priors (one exiftool subprocess per +batch) and writing **only** the new-contract triple — the legacy pair +stays frozen. Legacy `manual` rows convert via +`camera_tz::manual_date_resolution` instead of being re-resolved from +the file. Missing-file rows are paged past without spending the cap; +EXIF/exiftool-legacy rows wait when exiftool is absent (they'd otherwise +fall to fs_time). When a converted row's display-zone day differs from +the day its legacy value rendered on, the `day`-span `precomputed_reels` +rows covering either month-day (this library + `all`) are deleted and +unlinked; the nightly pre-gen regenerates. Daily conversation summaries +are SMS-derived and unaffected. Progress: Prometheus +`imageserver_date_backfill_remaining` (per tick) and +`date_backfill_remaining` per library on `GET /libraries` (live count of +`date_utc IS NULL`). + `/memories` is a single SQL query against this column (`get_memories_in_window` in `src/database/mod.rs`), using `strftime('%m-%d' | '%W' | '%m', date_taken, 'unixepoch', tz)` for diff --git a/migrations/2026-09-26-000000_add_utc_backfill_index/down.sql b/migrations/2026-09-26-000000_add_utc_backfill_index/down.sql new file mode 100644 index 0000000..bc9851d --- /dev/null +++ b/migrations/2026-09-26-000000_add_utc_backfill_index/down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS idx_image_exif_utc_backfill; diff --git a/migrations/2026-09-26-000000_add_utc_backfill_index/up.sql b/migrations/2026-09-26-000000_add_utc_backfill_index/up.sql new file mode 100644 index 0000000..0d97cff --- /dev/null +++ b/migrations/2026-09-26-000000_add_utc_backfill_index/up.sql @@ -0,0 +1,9 @@ +-- Partial index backing the true-instant backfill drain (ADR-0001, ticket +-- #118). The drain selects rows still on the legacy convention +-- (`date_utc IS NULL`) per library in id order, and the progress surface +-- (Prometheus `imageserver_date_backfill_remaining` + `/libraries`) counts +-- the same set. Mirrors `idx_image_exif_date_backfill`: the index shrinks +-- to nothing as the drain converges, so it costs ~0 after cutover. +CREATE INDEX idx_image_exif_utc_backfill + ON image_exif (library_id, id) + WHERE date_utc IS NULL; diff --git a/src/backfill.rs b/src/backfill.rs index c87c789..9e4abd4 100644 --- a/src/backfill.rs +++ b/src/backfill.rs @@ -6,14 +6,19 @@ //! wouldn't otherwise drain except during the once-an-hour full scan. //! Each function is bounded per call by a `*_PER_TICK` env-var cap. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::path::PathBuf; use std::sync::{Arc, Mutex}; +use chrono::{DateTime, Datelike, NaiveDate, TimeZone}; +use lazy_static::lazy_static; use log::{debug, info, warn}; +use prometheus::IntGauge; +use crate::camera_tz; use crate::content_hash; -use crate::database::ExifDao; +use crate::database::models::PrecomputedReel; +use crate::database::{ExifDao, PrecomputedReelDao, UtcBackfillCandidate}; use crate::date_resolver; use crate::face_watch; use crate::faces; @@ -219,6 +224,322 @@ pub fn backfill_missing_date_taken( backfilled } +lazy_static! { + /// Rows not yet on the true-instant date contract (`date_utc IS + /// NULL`), summed across libraries. Refreshed by the watcher every + /// tick; cutover (#120) waits on this reaching zero. + pub static ref DATE_BACKFILL_REMAINING: IntGauge = IntGauge::new( + "imageserver_date_backfill_remaining", + "Rows not yet converted to the true-instant date contract" + ) + .unwrap(); +} + +/// Upper bound on rows the true-instant drain pages past in one tick, +/// as a multiple of the cap. Unconvertible rows (missing files) don't +/// spend the cap, but a library full of them must not turn one tick into +/// a full-table walk. +const UTC_BACKFILL_SCAN_FACTOR: i64 = 20; + +/// What one tick of [`backfill_utc_dates`] did for one library. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct UtcBackfillOutcome { + /// Rows whose new-contract triple was written this tick. + pub converted: usize, + /// Rows paged past because their file is missing (didn't spend cap). + pub skipped_missing: usize, + /// Rows the waterfall couldn't convert faithfully (left for later). + pub unresolved: usize, + /// Converted rows whose display-zone calendar day moved. + pub day_changed: usize, + /// Pre-generated day reels deleted because a day they cover moved. + pub reels_invalidated: usize, +} + +/// True-instant backfill drain (ADR-0001, ticket #118): convert rows +/// still on a legacy mixed convention (`date_utc IS NULL AND date_taken +/// IS NOT NULL`) to true instants by re-running the canonical waterfall — +/// one exiftool subprocess per batch, same call shape as the #117 audit +/// preview (`resolve_dates_batch` with empty priors). Writes only the +/// new-contract triple; the legacy `date_taken` / `date_taken_source` +/// pair is frozen. Capped per tick by `DATE_BACKFILL_MAX_PER_TICK` +/// (default 500), shared with [`backfill_missing_date_taken`]. +/// +/// Legacy `manual` rows (overrides set before #115) aren't re-resolved +/// from the file — that would discard the operator's pick. Their legacy +/// value is the wall clock the UI sent, so they convert the same way a +/// manual edit does today (`camera_tz::manual_date_resolution` in the +/// display zone). +/// +/// Day-boundary invalidation: when a converted instant falls on a +/// different display-zone calendar day than the legacy value did (as the +/// legacy `/memories` SQL rendered it), the pre-generated `day` reels +/// covering either month-day are deleted so the nightly pre-gen +/// regenerates them. Daily conversation summaries are SMS-derived and +/// don't depend on photo dates, so they're not touched. +pub fn backfill_utc_dates( + context: &opentelemetry::Context, + library: &libraries::Library, + exif_dao: &Arc>>, + reel_dao: &Arc>>, + display_tz: &str, +) -> UtcBackfillOutcome { + let cap: i64 = dotenv::var("DATE_BACKFILL_MAX_PER_TICK") + .ok() + .and_then(|s| s.parse().ok()) + .filter(|n: &i64| *n > 0) + .unwrap_or(500); + backfill_utc_dates_capped(context, library, exif_dao, reel_dao, display_tz, cap) +} + +fn backfill_utc_dates_capped( + context: &opentelemetry::Context, + library: &libraries::Library, + exif_dao: &Arc>>, + reel_dao: &Arc>>, + display_tz: &str, + cap: i64, +) -> UtcBackfillOutcome { + let mut outcome = UtcBackfillOutcome::default(); + let base_path = std::path::Path::new(&library.root_path); + + // The watcher only calls this for online libraries; re-check here so + // an unmounted root can never partially convert (manual rows need no + // file and would otherwise convert while the rest waited). + if !base_path.is_dir() { + return outcome; + } + + let zone_name = camera_tz::display_zone_for(Some(display_tz)); + let Ok(zone) = zone_name.parse::() else { + return outcome; + }; + // Without exiftool, legacy EXIF/exiftool rows would fall through the + // empty-prior waterfall to fs_time and land a wrong instant. Leave + // them for a deploy that has it. + let exiftool_ok = date_resolver::exiftool_available(); + + // Page by id until `cap` convertible rows are collected. Rows that + // can't convert this tick are paged past, not counted against the + // cap, so a pile of missing files at low ids can't stall the drain. + let mut manual: Vec = Vec::new(); + let mut filed: Vec<(UtcBackfillCandidate, PathBuf)> = Vec::new(); + let mut cursor = 0; + let mut scanned = 0i64; + 'paging: loop { + let page = { + let mut dao = exif_dao.lock().expect("Unable to lock ExifDao"); + dao.get_rows_needing_utc_backfill(context, library.id, cursor, cap) + .unwrap_or_default() + }; + let page_len = page.len() as i64; + for row in page { + cursor = row.id; + scanned += 1; + match row.date_taken_source.as_deref() { + Some("manual") => manual.push(row), + Some("exif") | Some("exiftool") if !exiftool_ok => outcome.unresolved += 1, + _ => { + let abs = base_path.join(&row.rel_path); + if abs.exists() { + filed.push((row, abs)); + } else { + // The missing-file scan retires these rows. + outcome.skipped_missing += 1; + } + } + } + if (manual.len() + filed.len()) as i64 >= cap { + break 'paging; + } + } + if page_len < cap || scanned >= cap * UTC_BACKFILL_SCAN_FACTOR { + break; + } + } + let more_remain = (manual.len() + filed.len()) as i64 >= cap; + + // (candidate, date_utc, camera_offset_minutes, tz_source) + let mut conversions: Vec<(UtcBackfillCandidate, i64, Option, &'static str)> = + Vec::with_capacity(manual.len() + filed.len()); + for row in manual { + match camera_tz::manual_date_resolution(Some(&zone_name), row.date_taken) { + Some((utc, offset)) => { + conversions.push((row, utc, Some(offset), camera_tz::TzSource::Manual.as_str())) + } + None => outcome.unresolved += 1, + } + } + if !filed.is_empty() { + // One exiftool subprocess for the whole batch — byte-identical + // call shape to the #117 audit preview. + let paths: Vec = filed.iter().map(|(_, p)| p.clone()).collect(); + let resolved = date_resolver::resolve_dates_batch(&paths, &HashMap::new()); + for (row, abs) in filed { + let legacy_had_tag = matches!( + row.date_taken_source.as_deref(), + Some("exif") | Some("exiftool") + ); + match resolved.get(&abs) { + Some(rd) if !(legacy_had_tag && rd.source == date_resolver::DateSource::FsTime) => { + match (rd.date_utc, rd.tz_source) { + (Some(utc), Some(tz)) => { + conversions.push((row, utc, rd.camera_offset_minutes, tz.as_str())) + } + _ => outcome.unresolved += 1, + } + } + // No result, or a row whose legacy date came from a tag + // now resolving to fs_time (exiftool failed on it): don't + // bake a filesystem time in over a real capture date. + _ => { + debug!( + "utc_backfill: no faithful resolution for lib {} {}", + library.id, row.rel_path + ); + outcome.unresolved += 1; + } + } + } + } + + let mut affected_days: HashSet<(u32, u32)> = HashSet::new(); + { + let mut dao = exif_dao.lock().expect("Unable to lock ExifDao"); + for (row, utc, offset, tz_source) in &conversions { + match dao.backfill_date_utc(context, row.id, *utc, *offset, tz_source) { + Ok(true) => { + outcome.converted += 1; + let before = display_day(zone, row.date_taken); + let after = display_day(zone, *utc); + if let (Some(before), Some(after)) = (before, after) + && before != after + { + outcome.day_changed += 1; + affected_days.insert((before.month(), before.day())); + affected_days.insert((after.month(), after.day())); + } + } + // A manual edit / ingest raced us and already wrote it. + Ok(false) => {} + Err(e) => warn!( + "utc_backfill: update failed for lib {} {}: {:?}", + library.id, row.rel_path, e + ), + } + } + } + + if !affected_days.is_empty() { + outcome.reels_invalidated = + invalidate_day_reels(context, library, reel_dao, zone, &affected_days); + } + + if outcome.converted > 0 || outcome.unresolved > 0 || more_remain { + info!( + "utc_backfill: library '{}': {:?}, cap={}, more_remain={}", + library.name, outcome, cap, more_remain + ); + } + outcome +} + +/// Calendar day of a unix instant in the display zone. The legacy value +/// goes through here too: the legacy `/memories` SQL rendered +/// `date_taken` as an instant in the client's zone, so that is the day +/// the row's derived day data was built from. +fn display_day(zone: chrono_tz::Tz, epoch: i64) -> Option { + zone.timestamp_opt(epoch, 0) + .single() + .map(|dt| dt.date_naive()) +} + +/// The month-day an "on this day" reel covers: the date it was generated +/// on, in the zone offset it was generated with. A pre-gen fallback path +/// has stored seconds rather than minutes; an out-of-range offset falls +/// back to the display zone. +fn reel_covered_day(reel: &PrecomputedReel, zone: chrono_tz::Tz) -> Option { + if reel.tz_offset_minutes.abs() <= 14 * 60 { + DateTime::from_timestamp(reel.generated_at + reel.tz_offset_minutes as i64 * 60, 0) + .map(|dt| dt.date_naive()) + } else { + display_day(zone, reel.generated_at) + } +} + +/// Delete the pre-generated `day` reels (this library's and the +/// all-libraries one) covering any of `affected_days`, plus their files. +/// With the ledger row gone the nightly pre-gen's freshness gate fails +/// and it renders a new one on its normal schedule. Returns rows deleted. +fn invalidate_day_reels( + context: &opentelemetry::Context, + library: &libraries::Library, + reel_dao: &Arc>>, + zone: chrono_tz::Tz, + affected_days: &HashSet<(u32, u32)>, +) -> usize { + let keys = vec!["all".to_string(), library.id.to_string()]; + let mut dao = reel_dao.lock().expect("Unable to lock PrecomputedReelDao"); + let reels = match dao.list_for_span(context, "day", &keys) { + Ok(r) => r, + Err(e) => { + warn!("utc_backfill: listing day reels failed: {:?}", e); + return 0; + } + }; + let stale: Vec<&PrecomputedReel> = reels + .iter() + .filter(|r| { + reel_covered_day(r, zone).is_some_and(|d| affected_days.contains(&(d.month(), d.day()))) + }) + .collect(); + if stale.is_empty() { + return 0; + } + let ids: Vec = stale.iter().map(|r| r.id).collect(); + match dao.delete_ids(context, &ids) { + Ok(n) => { + // Unlink too, so the on-demand cache can't serve the stale + // render under the same key. + for r in &stale { + let mp4 = std::path::Path::new(&r.output_path); + let _ = std::fs::remove_file(mp4); + let _ = std::fs::remove_file(mp4.with_extension("json")); + } + info!( + "utc_backfill: invalidated {} day reel(s) for library '{}' (re-dated days {:?})", + n, library.name, affected_days + ); + n + } + Err(e) => { + warn!("utc_backfill: deleting stale day reels failed: {:?}", e); + 0 + } + } +} + +/// Refresh the progress surface for one online library: store its +/// remaining count in `remaining` and set [`DATE_BACKFILL_REMAINING`] to +/// the sum. Offline libraries keep their last-known count (the watcher +/// doesn't call this for them), so a flapping share doesn't make the +/// gauge dip. +pub fn refresh_utc_backfill_progress( + context: &opentelemetry::Context, + library: &libraries::Library, + exif_dao: &Arc>>, + remaining: &mut HashMap, +) { + let count = { + let mut dao = exif_dao.lock().expect("Unable to lock ExifDao"); + dao.count_rows_needing_utc_backfill(context, library.id) + }; + if let Ok(n) = count { + remaining.insert(library.id, n); + DATE_BACKFILL_REMAINING.set(remaining.values().sum()); + } +} + /// Per-tick face-detection drain. Pulls a capped batch of hashed-but- /// unscanned image_exif rows directly via the FaceDao anti-join and /// hands them to the existing detection pass. Runs on every tick (not @@ -799,4 +1120,540 @@ mod tests { assert_eq!(candidates[0].rel_path, "fresh.jpg"); assert_eq!(candidates[0].content_hash, "hash-fresh"); } + + // --- True-instant backfill drain (ADR-0001, ticket #118) ------------- + + use crate::database::SqlitePrecomputedReelDao; + use crate::database::models::{ImageExif, InsertablePrecomputedReel}; + use chrono::NaiveDateTime; + + const NY: &str = "America/New_York"; + + fn utc(y: i32, mo: u32, d: u32, h: u32, mi: u32, s: u32) -> i64 { + NaiveDate::from_ymd_opt(y, mo, d) + .unwrap() + .and_hms_opt(h, mi, s) + .unwrap() + .and_utc() + .timestamp() + } + + fn reel_dao_for( + conn: &Arc>, + ) -> Arc>> { + Arc::new(Mutex::new(Box::new( + SqlitePrecomputedReelDao::from_connection(Arc::clone(conn)), + ))) + } + + /// Seed a row on the legacy convention: legacy pair set, new-contract + /// triple NULL. Returns the row id. + fn insert_legacy( + exif_dao: &Arc>>, + lib_id: i32, + rel: &str, + date_taken: Option, + source: Option<&str>, + ) -> i32 { + let mut dao = exif_dao.lock().unwrap(); + dao.store_exif( + &ctx(), + InsertImageExif { + library_id: lib_id, + file_path: rel.to_string(), + camera_make: None, + camera_model: None, + lens_model: None, + width: None, + height: None, + orientation: None, + gps_latitude: None, + gps_longitude: None, + gps_altitude: None, + focal_length: None, + aperture: None, + shutter_speed: None, + iso: None, + date_taken, + created_time: 0, + last_modified: 0, + content_hash: None, + size_bytes: None, + phash_64: None, + dhash_64: None, + date_taken_source: source.map(str::to_string), + date_utc: None, + camera_offset_minutes: None, + date_taken_tz_source: None, + }, + ) + .expect("insert") + .id + } + + fn row(conn: &Arc>, id: i32) -> ImageExif { + let mut c = conn.lock().unwrap(); + schema::image_exif::table + .find(id) + .first::(&mut *c) + .expect("row") + } + + fn remaining(exif_dao: &Arc>>, lib_id: i32) -> i64 { + exif_dao + .lock() + .unwrap() + .count_rows_needing_utc_backfill(&ctx(), lib_id) + .unwrap() + } + + fn tools_available() -> bool { + let ok = |bin: &str, arg: &str| { + std::process::Command::new(bin) + .arg(arg) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .map(|s| s.success()) + .unwrap_or(false) + }; + ok("ffmpeg", "-version") && ok("exiftool", "-ver") + } + + fn run_ok(cmd: &mut std::process::Command) { + let out = cmd.output().expect("spawn fixture tool"); + assert!( + out.status.success(), + "fixture command failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + } + + /// Tiny H.264 MP4 whose container `creation_time` exiftool reads as + /// QuickTime / Track* groups (→ `utc_direct`). + fn write_container_video(root: &std::path::Path, rel: &str, creation_time: &str) { + run_ok( + std::process::Command::new("ffmpeg") + .args([ + "-y", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "color=c=black:s=32x32:d=0.4", + "-r", + "10", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-metadata", + ]) + .arg(format!("creation_time={creation_time}")) + .arg(root.join(rel)), + ); + } + + /// Tiny JPEG carrying `DateTimeOriginal` + `OffsetTimeOriginal`. + fn write_exif_jpeg(root: &std::path::Path, rel: &str, dto: &str, offset: &str) { + let abs = root.join(rel); + run_ok( + std::process::Command::new("ffmpeg") + .args([ + "-y", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "color=c=gray:s=16x16", + "-frames:v", + "1", + ]) + .arg(&abs), + ); + run_ok( + std::process::Command::new("exiftool") + .arg("-q") + .arg("-overwrite_original") + .arg(format!("-DateTimeOriginal={dto}")) + .arg(format!("-OffsetTimeOriginal={offset}")) + .arg(&abs), + ); + } + + #[test] + fn utc_backfill_converts_one_row_per_source_and_is_idempotent() { + if !tools_available() { + eprintln!("skipping: ffmpeg/exiftool not on PATH"); + return; + } + let (tmp, library, conn, exif_dao, _face_dao) = setup(); + let reel_dao = reel_dao_for(&conn); + + // exif: camera wall 22:30 at +02:00 → 20:30Z. Legacy = wall-as-UTC. + write_exif_jpeg(tmp.path(), "IMG_0001.jpg", "2024:07:04 22:30:00", "+02:00"); + let exif_legacy = utc(2024, 7, 4, 22, 30, 0); + let exif_id = insert_legacy( + &exif_dao, + 1, + "IMG_0001.jpg", + Some(exif_legacy), + Some("exif"), + ); + + // exiftool/container: creation_time is UTC. Legacy = the old + // server-local interpretation (EDT, +4h) — the "videos off by 4–5 + // hours" row this drain exists to fix. + write_container_video(tmp.path(), "clip.mp4", "2017-06-11T15:58:23Z"); + let video_legacy = utc(2017, 6, 11, 19, 58, 23); + let video_id = insert_legacy( + &exif_dao, + 1, + "clip.mp4", + Some(video_legacy), + Some("exiftool"), + ); + + // filename (wall pattern): legacy wall-as-UTC; no evidence → server zone. + let fname = "Screenshot_2014-06-01-20-44-50.png"; + write_image(tmp.path(), fname, b"not-a-png"); + let fname_wall: NaiveDateTime = NaiveDate::from_ymd_opt(2014, 6, 1) + .unwrap() + .and_hms_opt(20, 44, 50) + .unwrap(); + let fname_legacy = fname_wall.and_utc().timestamp(); + let fname_id = insert_legacy(&exif_dao, 1, fname, Some(fname_legacy), Some("filename")); + + // fs_time: no date anywhere in the file; legacy is already a true instant. + write_image(tmp.path(), "scan.jpg", b"no-exif-here"); + let meta = fs::metadata(tmp.path().join("scan.jpg")).unwrap(); + let fs_instant: chrono::DateTime = + crate::utils::earliest_fs_time(&meta).unwrap().into(); + let fs_instant = fs_instant.timestamp(); + let fs_id = insert_legacy(&exif_dao, 1, "scan.jpg", Some(fs_instant), Some("fs_time")); + + // manual (pre-#115 override): the operator's wall in the display + // zone — never re-resolved from the file. + write_image(tmp.path(), "edited.jpg", b"no-exif-here-either"); + let manual_legacy = utc(2020, 1, 1, 12, 0, 0); + let manual_id = insert_legacy( + &exif_dao, + 1, + "edited.jpg", + Some(manual_legacy), + Some("manual"), + ); + + assert_eq!(remaining(&exif_dao, 1), 5); + let out = backfill_utc_dates_capped(&ctx(), &library, &exif_dao, &reel_dao, NY, 500); + assert_eq!(out.converted, 5, "{out:?}"); + assert_eq!(out.unresolved, 0, "{out:?}"); + + let legacy = |r: &ImageExif| (r.date_taken, r.date_taken_source.clone()); + + let r = row(&conn, exif_id); + assert_eq!(r.date_utc, Some(utc(2024, 7, 4, 20, 30, 0))); + assert_eq!(r.camera_offset_minutes, Some(120)); + assert_eq!(r.date_taken_tz_source.as_deref(), Some("offset_tag")); + assert_eq!(legacy(&r), (Some(exif_legacy), Some("exif".into()))); + + let r = row(&conn, video_id); + assert_eq!(r.date_utc, Some(utc(2017, 6, 11, 15, 58, 23))); + assert_eq!(r.camera_offset_minutes, None); + assert_eq!(r.date_taken_tz_source.as_deref(), Some("utc_direct")); + assert_eq!(legacy(&r), (Some(video_legacy), Some("exiftool".into()))); + + let r = row(&conn, fname_id); + assert_eq!(r.date_taken_tz_source.as_deref(), Some("assumed_server")); + if let Some(zone) = crate::camera_tz::server_zone() { + let off = crate::camera_tz::offset_minutes_for_zone(zone, fname_wall); + assert_eq!(r.camera_offset_minutes, Some(off)); + assert_eq!( + r.date_utc, + Some(crate::camera_tz::wall_to_instant(fname_wall, off)) + ); + } + assert_eq!(legacy(&r), (Some(fname_legacy), Some("filename".into()))); + + let r = row(&conn, fs_id); + assert_eq!(r.date_utc, Some(fs_instant)); + assert_eq!(r.camera_offset_minutes, None); + assert_eq!(r.date_taken_tz_source.as_deref(), Some("utc_direct")); + assert_eq!(legacy(&r), (Some(fs_instant), Some("fs_time".into()))); + + let r = row(&conn, manual_id); + // 2020-01-01 12:00 EST (-300) → 17:00Z. + assert_eq!(r.date_utc, Some(utc(2020, 1, 1, 17, 0, 0))); + assert_eq!(r.camera_offset_minutes, Some(-300)); + assert_eq!(r.date_taken_tz_source.as_deref(), Some("manual")); + assert_eq!(legacy(&r), (Some(manual_legacy), Some("manual".into()))); + + // Second pass: nothing eligible, nothing changes, progress is 0. + let ids = [exif_id, video_id, fname_id, fs_id, manual_id]; + let snapshot = |conn: &Arc>| { + ids.iter() + .map(|id| { + let r = row(conn, *id); + ( + r.date_taken, + r.date_taken_source, + r.date_utc, + r.camera_offset_minutes, + r.date_taken_tz_source, + ) + }) + .collect::>() + }; + let before = snapshot(&conn); + let again = backfill_utc_dates_capped(&ctx(), &library, &exif_dao, &reel_dao, NY, 500); + assert_eq!(again, UtcBackfillOutcome::default()); + assert_eq!(snapshot(&conn), before); + let mut progress = HashMap::new(); + refresh_utc_backfill_progress(&ctx(), &library, &exif_dao, &mut progress); + assert_eq!(progress.get(&1), Some(&0)); + } + + #[test] + fn utc_backfill_leaves_null_date_rows_and_other_libraries_alone() { + let (tmp, library, conn, exif_dao, _face_dao) = setup(); + let reel_dao = reel_dao_for(&conn); + write_image(tmp.path(), "1500000000.jpg", b"x"); + write_image(tmp.path(), "undated.jpg", b"y"); + let ts = 1_500_000_000; + let converted = insert_legacy(&exif_dao, 1, "1500000000.jpg", Some(ts), Some("filename")); + // Both NULL → the legacy drain's job (its writer fills the triple). + let undated = insert_legacy(&exif_dao, 1, "undated.jpg", None, None); + let other = insert_legacy(&exif_dao, 2, "1500000000.jpg", Some(ts), Some("filename")); + + let out = backfill_utc_dates_capped(&ctx(), &library, &exif_dao, &reel_dao, NY, 500); + assert_eq!(out.converted, 1); + assert_eq!(row(&conn, converted).date_utc, Some(ts)); + assert_eq!(row(&conn, undated).date_utc, None); + assert_eq!(row(&conn, other).date_utc, None); + // Progress counts every row not yet on the new contract. + assert_eq!(remaining(&exif_dao, 1), 1); + } + + #[test] + fn utc_backfill_respects_cap_and_missing_files_dont_spend_it() { + let (tmp, library, conn, exif_dao, _face_dao) = setup(); + let reel_dao = reel_dao_for(&conn); + // Missing files at the lowest ids, ahead of the present ones. + // Timestamp filenames keep the fixture hermetic (never hit exiftool). + let mut missing = Vec::new(); + for i in 0..3 { + let ts = 1_400_000_000 + i; + let rel = format!("{ts}.jpg"); + missing.push(insert_legacy( + &exif_dao, + 1, + &rel, + Some(ts), + Some("filename"), + )); + } + for i in 0..5 { + let ts = 1_500_000_000 + i; + let rel = format!("{ts}.jpg"); + write_image(tmp.path(), &rel, b"z"); + insert_legacy(&exif_dao, 1, &rel, Some(ts), Some("filename")); + } + assert_eq!(remaining(&exif_dao, 1), 8); + + let mut progress = HashMap::new(); + let mut seen = Vec::new(); + for _ in 0..4 { + let out = backfill_utc_dates_capped(&ctx(), &library, &exif_dao, &reel_dao, NY, 2); + refresh_utc_backfill_progress(&ctx(), &library, &exif_dao, &mut progress); + seen.push((out.converted, progress[&1])); + } + // Cap holds (2 per tick) and the 3 missing rows never spend it. + assert_eq!(seen, vec![(2, 6), (2, 4), (1, 3), (0, 3)]); + + // The missing-file scan retires the rest → progress reaches 0. + { + let mut c = conn.lock().unwrap(); + diesel::delete( + schema::image_exif::table.filter(schema::image_exif::id.eq_any(&missing)), + ) + .execute(&mut *c) + .unwrap(); + } + refresh_utc_backfill_progress(&ctx(), &library, &exif_dao, &mut progress); + assert_eq!(progress[&1], 0); + } + + #[test] + fn utc_backfill_skips_unavailable_library_then_resumes() { + let (tmp, library, conn, exif_dao, _face_dao) = setup(); + let reel_dao = reel_dao_for(&conn); + write_image(tmp.path(), "1500000000.jpg", b"z"); + let filed = insert_legacy( + &exif_dao, + 1, + "1500000000.jpg", + Some(1_500_000_000), + Some("filename"), + ); + // A manual row needs no file — it must still wait for the mount. + let manual = insert_legacy( + &exif_dao, + 1, + "edited.jpg", + Some(utc(2020, 1, 1, 12, 0, 0)), + Some("manual"), + ); + + let unmounted = Library { + root_path: tmp.path().join("unmounted").to_string_lossy().to_string(), + ..library.clone() + }; + // The watcher gate sees it stale… + let health_map = libraries::new_health_map(std::slice::from_ref(&unmounted)); + assert!(!libraries::refresh_health(&health_map, &unmounted, true).is_online()); + // …and the drain itself converts nothing even if called. + let out = backfill_utc_dates_capped(&ctx(), &unmounted, &exif_dao, &reel_dao, NY, 500); + assert_eq!(out, UtcBackfillOutcome::default()); + assert_eq!(row(&conn, filed).date_utc, None); + assert_eq!(row(&conn, manual).date_utc, None); + + // Share returns → picked up next tick. + assert!(libraries::refresh_health(&health_map, &library, true).is_online()); + let out = backfill_utc_dates_capped(&ctx(), &library, &exif_dao, &reel_dao, NY, 500); + assert_eq!(out.converted, 2); + assert_eq!(remaining(&exif_dao, 1), 0); + } + + /// Seed a pre-generated reel covering `month`-`day` (generated 03:00 + /// EDT that day, as the nightly pre-gen does). Returns (id, mp4 path). + fn seed_reel( + reel_dao: &Arc>>, + dir: &std::path::Path, + span: &str, + library_key: &str, + month: u32, + day: u32, + ) -> (i32, PathBuf) { + let mp4 = dir.join(format!("reel-{span}-{library_key}-{month}-{day}.mp4")); + fs::write(&mp4, b"mp4").unwrap(); + let id = reel_dao + .lock() + .unwrap() + .record_reel( + &ctx(), + &InsertablePrecomputedReel { + span: span.to_string(), + library_key: library_key.to_string(), + cache_key: format!("{span}-{library_key}-{month}-{day}"), + output_path: mp4.to_string_lossy().to_string(), + title: "t".to_string(), + media_count: 1, + render_version: 1, + tz_offset_minutes: -240, + voice: None, + generated_at: utc(2025, month, day, 7, 0, 0), + }, + ) + .unwrap(); + (id, mp4) + } + + fn reel_ids(conn: &Arc>) -> Vec { + let mut c = conn.lock().unwrap(); + schema::precomputed_reels::table + .select(schema::precomputed_reels::id) + .order(schema::precomputed_reels::id.asc()) + .load::(&mut *c) + .unwrap() + } + + #[test] + fn utc_backfill_invalidates_exactly_the_moved_days_reels() { + let (tmp, library, conn, exif_dao, _face_dao) = setup(); + let reel_dao = reel_dao_for(&conn); + let reels_dir = TempDir::new().unwrap(); + + // Manual wall 2017-06-12 02:00 in NY. The legacy value, read as an + // instant in NY by the old /memories SQL, showed 06-11 22:00; the + // true instant is 06-12 06:00Z = 06-12 02:00 EDT → the day moves + // 06-11 → 06-12. + let moved = insert_legacy( + &exif_dao, + 1, + "moved.jpg", + Some(utc(2017, 6, 12, 2, 0, 0)), + Some("manual"), + ); + // Unchanged: a timestamp-filename row's legacy value is already the instant. + write_image(tmp.path(), "1400000000.jpg", b"z"); + insert_legacy( + &exif_dao, + 1, + "1400000000.jpg", + Some(1_400_000_000), + Some("filename"), + ); + + let (_, mp4_11) = seed_reel(&reel_dao, reels_dir.path(), "day", "1", 6, 11); + let (_, mp4_12) = seed_reel(&reel_dao, reels_dir.path(), "day", "all", 6, 12); + let (other_lib, _) = seed_reel(&reel_dao, reels_dir.path(), "day", "2", 6, 12); + let (other_day, mp4_0301) = seed_reel(&reel_dao, reels_dir.path(), "day", "1", 3, 1); + let (week, _) = seed_reel(&reel_dao, reels_dir.path(), "week", "1", 6, 12); + + let out = backfill_utc_dates_capped(&ctx(), &library, &exif_dao, &reel_dao, NY, 500); + assert_eq!(out.converted, 2); + assert_eq!(out.day_changed, 1, "{out:?}"); + assert_eq!(out.reels_invalidated, 2, "{out:?}"); + assert_eq!(row(&conn, moved).date_utc, Some(utc(2017, 6, 12, 6, 0, 0))); + + // Exactly the moved days' day reels for this library (+ "all") are gone. + assert_eq!(reel_ids(&conn), vec![other_lib, other_day, week]); + assert!( + !mp4_11.exists() && !mp4_12.exists(), + "stale renders unlinked" + ); + assert!(mp4_0301.exists()); + + // With the ledger row gone the nightly pre-gen's freshness gate + // (`latest_for` must match the new cache key) can't skip → it + // regenerates on its normal schedule; no new scheduler involved. + let latest = reel_dao + .lock() + .unwrap() + .latest_for(&ctx(), "day", "all") + .unwrap(); + assert!(latest.is_none()); + } + + #[test] + fn utc_backfill_utc_midnight_straddle_marks_nothing() { + let (_tmp, library, conn, exif_dao, _face_dao) = setup(); + let reel_dao = reel_dao_for(&conn); + let reels_dir = TempDir::new().unwrap(); + + // Manual wall 2017-06-11 23:30 in NY. Legacy instant 06-11 23:30Z + // (NY 19:30); true instant 06-12 03:30Z (NY 23:30). The UTC date + // flips 06-11 → 06-12, but the display-zone day stays 06-11. + let straddle = insert_legacy( + &exif_dao, + 1, + "late.jpg", + Some(utc(2017, 6, 11, 23, 30, 0)), + Some("manual"), + ); + seed_reel(&reel_dao, reels_dir.path(), "day", "1", 6, 11); + seed_reel(&reel_dao, reels_dir.path(), "day", "all", 6, 12); + + let out = backfill_utc_dates_capped(&ctx(), &library, &exif_dao, &reel_dao, NY, 500); + assert_eq!(out.converted, 1); + assert_eq!(out.day_changed, 0); + assert_eq!(out.reels_invalidated, 0); + assert_eq!( + row(&conn, straddle).date_utc, + Some(utc(2017, 6, 12, 3, 30, 0)) + ); + assert_eq!(reel_ids(&conn).len(), 2); + } } diff --git a/src/database/mod.rs b/src/database/mod.rs index 04be69a..67ad787 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -43,6 +43,17 @@ pub struct DuplicateRow { pub duplicate_decided_at: Option, } +/// One row the true-instant backfill drain (ADR-0001, ticket #118) still +/// has to convert: on the legacy convention (`date_utc IS NULL`) with a +/// legacy `date_taken` to compare the converted day against. +#[derive(Debug, Clone, Queryable)] +pub struct UtcBackfillCandidate { + pub id: i32, + pub rel_path: String, + pub date_taken: i64, + pub date_taken_source: Option, +} + pub mod calendar_dao; pub mod daily_summary_dao; pub mod insight_generation_job_dao; @@ -513,6 +524,46 @@ pub trait ExifDao: Sync + Send { date_taken_tz_source: Option<&str>, ) -> Result<(), DbError>; + /// Rows the true-instant drain (ADR-0001, ticket #118) still has to + /// convert: `date_utc IS NULL AND date_taken IS NOT NULL` in + /// `library_id`, `id > after_id`, ordered by id ASC. Rows with both + /// NULL belong to `get_rows_needing_date_backfill` (whose writer fills + /// the new triple too). The `after_id` cursor lets the drain page past + /// rows it can't convert this tick (missing files) without spending + /// its per-tick cap on them. Backed by `idx_image_exif_utc_backfill`. + fn get_rows_needing_utc_backfill( + &mut self, + context: &opentelemetry::Context, + library_id: i32, + after_id: i32, + limit: i64, + ) -> Result, DbError>; + + /// Persist a converted row's new-contract triple (`date_utc`, + /// `camera_offset_minutes`, `date_taken_tz_source`) by primary key. + /// Never touches the frozen legacy `date_taken` / `date_taken_source` + /// pair. Guarded on `date_utc IS NULL` so a manual edit or ingest + /// that raced the drain wins. Returns whether a row was written. + fn backfill_date_utc( + &mut self, + context: &opentelemetry::Context, + id: i32, + date_utc: i64, + camera_offset_minutes: Option, + date_taken_tz_source: &str, + ) -> Result; + + /// Progress surface for the true-instant drain: rows in `library_id` + /// not yet on the new contract (`date_utc IS NULL`). Counts every such + /// row — including NULL-date rows the legacy drain resolves and rows + /// whose file is missing (until the missing-file scan retires them) — + /// because cutover (#120) waits on exactly this set reaching zero. + fn count_rows_needing_utc_backfill( + &mut self, + context: &opentelemetry::Context, + library_id: i32, + ) -> Result; + /// Find image_exif rows needing a CLIP embedding for semantic search: /// `clip_embedding IS NULL AND content_hash IS NOT NULL`, ordered by id /// ASC, limited. Hash-less rows wait for `backfill_unhashed_backlog` to @@ -1501,6 +1552,95 @@ impl ExifDao for SqliteExifDao { }) } + fn get_rows_needing_utc_backfill( + &mut self, + context: &opentelemetry::Context, + library_id_val: i32, + after_id: i32, + limit: i64, + ) -> Result, DbError> { + trace_db_call(context, "query", "get_rows_needing_utc_backfill", |_span| { + use schema::image_exif::dsl::*; + + let mut connection = self.connection.lock().expect("Unable to get ExifDao"); + + // Partial index `(library_id, id) WHERE date_utc IS NULL` + // serves the filter + order directly. + image_exif + .filter(library_id.eq(library_id_val)) + .filter(date_utc.is_null()) + .filter(date_taken.is_not_null()) + .filter(id.gt(after_id)) + .select(( + id, + rel_path, + date_taken.assume_not_null(), + date_taken_source, + )) + .order(id.asc()) + .limit(limit) + .load::(connection.deref_mut()) + .map_err(|e| anyhow::anyhow!("Query error: {}", e)) + }) + .map_err(|e| DbError::log(DbErrorKind::QueryError, e)) + } + + fn backfill_date_utc( + &mut self, + context: &opentelemetry::Context, + id_val: i32, + date_utc_val: i64, + camera_offset_minutes_val: Option, + date_taken_tz_source_val: &str, + ) -> Result { + trace_db_call(context, "update", "backfill_date_utc", |_span| { + use schema::image_exif::dsl::*; + + let mut connection = self.connection.lock().expect("Unable to get ExifDao"); + + // New-contract triple only — the legacy pair is frozen + // (ADR-0001). + diesel::update(image_exif.find(id_val).filter(date_utc.is_null())) + .set(( + date_utc.eq(Some(date_utc_val)), + camera_offset_minutes.eq(camera_offset_minutes_val), + date_taken_tz_source.eq(Some(date_taken_tz_source_val)), + )) + .execute(connection.deref_mut()) + .map(|rows| rows > 0) + .map_err(|e| anyhow::anyhow!("diesel update failed (id={}): {}", id_val, e)) + }) + .map_err(|e| { + log::warn!("backfill_date_utc: {}", e); + DbError::new(DbErrorKind::UpdateError) + }) + } + + fn count_rows_needing_utc_backfill( + &mut self, + context: &opentelemetry::Context, + library_id_val: i32, + ) -> Result { + trace_db_call( + context, + "query", + "count_rows_needing_utc_backfill", + |_span| { + use schema::image_exif::dsl::*; + + let mut connection = self.connection.lock().expect("Unable to get ExifDao"); + + image_exif + .filter(library_id.eq(library_id_val)) + .filter(date_utc.is_null()) + .count() + .get_result::(connection.deref_mut()) + .map_err(|e| anyhow::anyhow!("Query error: {}", e)) + }, + ) + .map_err(|e| DbError::log(DbErrorKind::QueryError, e)) + } + fn list_clip_unencoded_candidates( &mut self, context: &opentelemetry::Context, diff --git a/src/database/precomputed_reel_dao.rs b/src/database/precomputed_reel_dao.rs index b66573b..0a83561 100644 --- a/src/database/precomputed_reel_dao.rs +++ b/src/database/precomputed_reel_dao.rs @@ -58,6 +58,24 @@ pub trait PrecomputedReelDao: Sync + Send { /// to protect files a ledger row still points at. #[allow(dead_code)] fn all_cache_keys(&mut self, context: &opentelemetry::Context) -> Result, DbError>; + + /// Every reel row for `span` whose `library_key` is one of + /// `library_keys`. Used by the true-instant date drain (ticket #118) + /// to find the day reels a re-dated photo invalidates. + fn list_for_span( + &mut self, + context: &opentelemetry::Context, + span: &str, + library_keys: &[String], + ) -> Result, DbError>; + + /// Delete the given reel rows (stale-marking: the nightly pre-gen + /// finds no current row and regenerates). Returns rows deleted. + fn delete_ids( + &mut self, + context: &opentelemetry::Context, + ids: &[i32], + ) -> Result; } pub struct SqlitePrecomputedReelDao { @@ -219,6 +237,49 @@ impl PrecomputedReelDao for SqlitePrecomputedReelDao { }) .map_err(|e| DbError::log(DbErrorKind::QueryError, e)) } + + fn list_for_span( + &mut self, + context: &opentelemetry::Context, + span: &str, + library_keys: &[String], + ) -> Result, DbError> { + trace_db_call(context, "query", "list_for_span", |_span| { + use schema::precomputed_reels::dsl; + + let mut connection = self + .connection + .lock() + .expect("Unable to lock PrecomputedReelDao"); + + dsl::precomputed_reels + .filter(dsl::span.eq(span)) + .filter(dsl::library_key.eq_any(library_keys)) + .load::(connection.deref_mut()) + .map_err(|e| anyhow::anyhow!("Failed to list reels for span: {}", e)) + }) + .map_err(|e| DbError::log(DbErrorKind::QueryError, e)) + } + + fn delete_ids( + &mut self, + context: &opentelemetry::Context, + ids: &[i32], + ) -> Result { + trace_db_call(context, "delete", "delete_ids", |_span| { + use schema::precomputed_reels::dsl; + + let mut connection = self + .connection + .lock() + .expect("Unable to lock PrecomputedReelDao"); + + diesel::delete(dsl::precomputed_reels.filter(dsl::id.eq_any(ids))) + .execute(connection.deref_mut()) + .map_err(|e| anyhow::anyhow!("Failed to delete reels: {}", e)) + }) + .map_err(|e| DbError::log(DbErrorKind::UpdateError, e)) + } } #[cfg(test)] diff --git a/src/date_resolver.rs b/src/date_resolver.rs index 430ffba..3d9b48c 100644 --- a/src/date_resolver.rs +++ b/src/date_resolver.rs @@ -351,7 +351,7 @@ const EXIFTOOL_EVIDENCE_TAGS: &[&str] = &[ /// Cache the "exiftool exists on PATH" check across the process lifetime so /// the per-tick backfill doesn't fork a doomed subprocess every iteration on /// deploys without exiftool installed. -fn exiftool_available() -> bool { +pub fn exiftool_available() -> bool { static AVAIL: OnceLock = OnceLock::new(); *AVAIL.get_or_init(|| { let ok = Command::new("exiftool") diff --git a/src/files.rs b/src/files.rs index ed1d72c..9cec9fd 100644 --- a/src/files.rs +++ b/src/files.rs @@ -1756,6 +1756,35 @@ mod tests { Ok(()) } + fn get_rows_needing_utc_backfill( + &mut self, + _context: &opentelemetry::Context, + _library_id: i32, + _after_id: i32, + _limit: i64, + ) -> Result, DbError> { + Ok(Vec::new()) + } + + fn backfill_date_utc( + &mut self, + _context: &opentelemetry::Context, + _id: i32, + _date_utc: i64, + _camera_offset_minutes: Option, + _date_taken_tz_source: &str, + ) -> Result { + Ok(false) + } + + fn count_rows_needing_utc_backfill( + &mut self, + _context: &opentelemetry::Context, + _library_id: i32, + ) -> Result { + Ok(0) + } + fn set_manual_date_taken( &mut self, _context: &opentelemetry::Context, diff --git a/src/libraries.rs b/src/libraries.rs index 377b442..a916f94 100644 --- a/src/libraries.rs +++ b/src/libraries.rs @@ -460,6 +460,10 @@ pub struct LibraryStatus { #[serde(flatten)] pub library: Library, pub health: LibraryHealth, + /// Rows in this library not yet converted to the true-instant date + /// contract (`date_utc IS NULL`, ADR-0001 / ticket #118). Drains to 0 + /// over watcher ticks; `None` when the count query failed. + pub date_backfill_remaining: Option, } #[derive(serde::Serialize)] @@ -488,6 +492,12 @@ pub async fn list_libraries(_claims: Claims, app_state: Data) -> impl .library_health .read() .unwrap_or_else(|e| e.into_inner()); + let context = opentelemetry::Context::new(); + let mut exif_dao = app_state + .insight_generator + .exif_dao() + .lock() + .expect("Unable to lock ExifDao"); let libraries = live_guard .iter() .map(|lib| LibraryStatus { @@ -496,6 +506,11 @@ pub async fn list_libraries(_claims: Claims, app_state: Data) -> impl .get(&lib.id) .cloned() .unwrap_or(LibraryHealth::Online), + // Live count off the partial index — cheap, and exact even for + // a stale library the watcher isn't refreshing. + date_backfill_remaining: exif_dao + .count_rows_needing_utc_backfill(&context, lib.id) + .ok(), }) .collect(); HttpResponse::Ok().json(LibrariesResponse { diff --git a/src/main.rs b/src/main.rs index b3aee61..f33abd0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -151,6 +151,12 @@ fn main() -> std::io::Result<()> { .registry .register(Box::new(thumbnails::VIDEO_GAUGE.clone())) .unwrap(); + // True-instant date drain progress (#118). Refreshed per watcher + // tick; see `backfill::refresh_utc_backfill_progress`. + prometheus + .registry + .register(Box::new(backfill::DATE_BACKFILL_REMAINING.clone())) + .unwrap(); // HLS readiness gauges. Updated by the watcher every full-scan // tick and on every `/hls/stats` request. See `hls_stats`. prometheus diff --git a/src/watcher.rs b/src/watcher.rs index 28d8390..21ada9c 100644 --- a/src/watcher.rs +++ b/src/watcher.rs @@ -26,7 +26,10 @@ use log::{debug, error, info, warn}; use crate::backfill; use crate::content_hash; use crate::database::models::InsertImageExif; -use crate::database::{ExifDao, PreviewDao, SqliteExifDao, SqlitePreviewDao}; +use crate::database::{ + ExifDao, PrecomputedReelDao, PreviewDao, SqliteExifDao, SqlitePrecomputedReelDao, + SqlitePreviewDao, SqliteUserAiPrefsDao, UserAiPrefsDao, +}; use crate::date_resolver; use crate::exif; use crate::face_watch; @@ -336,6 +339,17 @@ pub fn watch_files( Box::new(SqliteTagDao::default()) as Box )); + // True-instant date drain (#118): day-reel invalidation + the + // display zone the day comparison runs in. + let precomputed_reel_dao = Arc::new(Mutex::new( + Box::new(SqlitePrecomputedReelDao::new()) as Box + )); + let user_ai_prefs_dao: Mutex> = + Mutex::new(Box::new(SqliteUserAiPrefsDao::new())); + // Last-known per-library remaining count behind the + // `imageserver_date_backfill_remaining` gauge. + let mut utc_backfill_remaining: HashMap = HashMap::new(); + let mut last_quick_scan = SystemTime::now(); // Initialize to UNIX_EPOCH so the *first* tick is treated as a // full scan. That replaces the legacy startup ScanDirectoryMessage @@ -494,6 +508,35 @@ pub fn watch_files( { let context = opentelemetry::Context::new(); backfill::backfill_missing_date_taken(&context, lib, &exif_dao); + + // True-instant drain (ADR-0001, #118): convert + // legacy mixed-convention rows, invalidating day + // reels whose day moved. Inside the online gate, so a + // stale library is skipped whole and resumes next + // tick. The display zone is re-read per tick so a + // prefs change applies without restart. + let display_tz = { + let mut prefs = user_ai_prefs_dao.lock().expect("prefs dao poisoned"); + match prefs.get_prefs(&context) { + Ok(Some(p)) => { + crate::camera_tz::display_zone_for(p.display_tz.as_deref()) + } + _ => crate::camera_tz::display_zone_for(None), + } + }; + backfill::backfill_utc_dates( + &context, + lib, + &exif_dao, + &precomputed_reel_dao, + &display_tz, + ); + backfill::refresh_utc_backfill_progress( + &context, + lib, + &exif_dao, + &mut utc_backfill_remaining, + ); } if is_full_scan {