feat(dates): backfill drain converts mixed-convention rows to true instants (#118)

Per-tick drain (backfill_utc_dates) at the existing date-drain seam,
inside the per-library online gate. Selects date_utc IS NULL AND
date_taken IS NOT NULL (new partial index idx_image_exif_utc_backfill),
re-runs resolve_dates_batch with empty priors (one exiftool subprocess
per batch, same call shape as the #117 audit), and writes only the
new-contract triple; the legacy pair stays frozen. Capped by
DATE_BACKFILL_MAX_PER_TICK; missing-file rows are paged past without
spending the cap. Legacy manual overrides convert via
manual_date_resolution rather than being re-resolved from the file.

Day-boundary invalidation deletes the day-span precomputed reels
covering a re-dated row's old/new display-zone month-day so the nightly
pre-gen regenerates them. Progress: imageserver_date_backfill_remaining
gauge + date_backfill_remaining on GET /libraries.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Cameron CordesandClaude Opus 5.5 committed 2026-09-26 09:24:14 -04:00
1 parent 3ac011abd7
commit 1fad08dea3
11 files changed
+1185 -4

No files matched your search

+20
View File
@@ -437,6 +437,26 @@ a stronger tool comes online (exiftool install, new filename regex),
re-resolve out-of-band rather than re-introducing the steady-state
eligibility.
**True-instant drain (`backfill_utc_dates`, ticket #118).** Runs right
after `backfill_missing_date_taken` inside the per-library online gate,
sharing its `DATE_BACKFILL_MAX_PER_TICK` cap. Converts legacy
mixed-convention rows (`date_utc IS NULL AND date_taken IS NOT NULL`,
partial index `idx_image_exif_utc_backfill`) by re-running
`resolve_dates_batch` with empty priors (one exiftool subprocess per
batch) and writing **only** the new-contract triple — the legacy pair
stays frozen. Legacy `manual` rows convert via
`camera_tz::manual_date_resolution` instead of being re-resolved from
the file. Missing-file rows are paged past without spending the cap;
EXIF/exiftool-legacy rows wait when exiftool is absent (they'd otherwise
fall to fs_time). When a converted row's display-zone day differs from
the day its legacy value rendered on, the `day`-span `precomputed_reels`
rows covering either month-day (this library + `all`) are deleted and
unlinked; the nightly pre-gen regenerates. Daily conversation summaries
are SMS-derived and unaffected. Progress: Prometheus
`imageserver_date_backfill_remaining` (per tick) and
`date_backfill_remaining` per library on `GET /libraries` (live count of
`date_utc IS NULL`).
`/memories` is a single SQL query against this column
(`get_memories_in_window` in `src/database/mod.rs`), using
`strftime('%m-%d' | '%W' | '%m', date_taken, 'unixepoch', tz)` for
@@ -0,0 +1 @@
DROP INDEX IF EXISTS idx_image_exif_utc_backfill;
@@ -0,0 +1,9 @@
-- Partial index backing the true-instant backfill drain (ADR-0001, ticket
-- #118). The drain selects rows still on the legacy convention
-- (`date_utc IS NULL`) per library in id order, and the progress surface
-- (Prometheus `imageserver_date_backfill_remaining` + `/libraries`) counts
-- the same set. Mirrors `idx_image_exif_date_backfill`: the index shrinks
-- to nothing as the drain converges, so it costs ~0 after cutover.
CREATE INDEX idx_image_exif_utc_backfill
ON image_exif (library_id, id)
WHERE date_utc IS NULL;
+859 -2
View File
@@ -6,14 +6,19 @@
//! wouldn't otherwise drain except during the once-an-hour full scan.
//! Each function is bounded per call by a `*_PER_TICK` env-var cap.
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use chrono::{DateTime, Datelike, NaiveDate, TimeZone};
use lazy_static::lazy_static;
use log::{debug, info, warn};
use prometheus::IntGauge;
use crate::camera_tz;
use crate::content_hash;
use crate::database::ExifDao;
use crate::database::models::PrecomputedReel;
use crate::database::{ExifDao, PrecomputedReelDao, UtcBackfillCandidate};
use crate::date_resolver;
use crate::face_watch;
use crate::faces;
@@ -219,6 +224,322 @@ pub fn backfill_missing_date_taken(
backfilled
}
lazy_static! {
/// Rows not yet on the true-instant date contract (`date_utc IS
/// NULL`), summed across libraries. Refreshed by the watcher every
/// tick; cutover (#120) waits on this reaching zero.
pub static ref DATE_BACKFILL_REMAINING: IntGauge = IntGauge::new(
"imageserver_date_backfill_remaining",
"Rows not yet converted to the true-instant date contract"
)
.unwrap();
}
/// Upper bound on rows the true-instant drain pages past in one tick,
/// as a multiple of the cap. Unconvertible rows (missing files) don't
/// spend the cap, but a library full of them must not turn one tick into
/// a full-table walk.
const UTC_BACKFILL_SCAN_FACTOR: i64 = 20;
/// What one tick of [`backfill_utc_dates`] did for one library.
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct UtcBackfillOutcome {
/// Rows whose new-contract triple was written this tick.
pub converted: usize,
/// Rows paged past because their file is missing (didn't spend cap).
pub skipped_missing: usize,
/// Rows the waterfall couldn't convert faithfully (left for later).
pub unresolved: usize,
/// Converted rows whose display-zone calendar day moved.
pub day_changed: usize,
/// Pre-generated day reels deleted because a day they cover moved.
pub reels_invalidated: usize,
}
/// True-instant backfill drain (ADR-0001, ticket #118): convert rows
/// still on a legacy mixed convention (`date_utc IS NULL AND date_taken
/// IS NOT NULL`) to true instants by re-running the canonical waterfall —
/// one exiftool subprocess per batch, same call shape as the #117 audit
/// preview (`resolve_dates_batch` with empty priors). Writes only the
/// new-contract triple; the legacy `date_taken` / `date_taken_source`
/// pair is frozen. Capped per tick by `DATE_BACKFILL_MAX_PER_TICK`
/// (default 500), shared with [`backfill_missing_date_taken`].
///
/// Legacy `manual` rows (overrides set before #115) aren't re-resolved
/// from the file — that would discard the operator's pick. Their legacy
/// value is the wall clock the UI sent, so they convert the same way a
/// manual edit does today (`camera_tz::manual_date_resolution` in the
/// display zone).
///
/// Day-boundary invalidation: when a converted instant falls on a
/// different display-zone calendar day than the legacy value did (as the
/// legacy `/memories` SQL rendered it), the pre-generated `day` reels
/// covering either month-day are deleted so the nightly pre-gen
/// regenerates them. Daily conversation summaries are SMS-derived and
/// don't depend on photo dates, so they're not touched.
pub fn backfill_utc_dates(
context: &opentelemetry::Context,
library: &libraries::Library,
exif_dao: &Arc<Mutex<Box<dyn ExifDao>>>,
reel_dao: &Arc<Mutex<Box<dyn PrecomputedReelDao>>>,
display_tz: &str,
) -> UtcBackfillOutcome {
let cap: i64 = dotenv::var("DATE_BACKFILL_MAX_PER_TICK")
.ok()
.and_then(|s| s.parse().ok())
.filter(|n: &i64| *n > 0)
.unwrap_or(500);
backfill_utc_dates_capped(context, library, exif_dao, reel_dao, display_tz, cap)
}
fn backfill_utc_dates_capped(
context: &opentelemetry::Context,
library: &libraries::Library,
exif_dao: &Arc<Mutex<Box<dyn ExifDao>>>,
reel_dao: &Arc<Mutex<Box<dyn PrecomputedReelDao>>>,
display_tz: &str,
cap: i64,
) -> UtcBackfillOutcome {
let mut outcome = UtcBackfillOutcome::default();
let base_path = std::path::Path::new(&library.root_path);
// The watcher only calls this for online libraries; re-check here so
// an unmounted root can never partially convert (manual rows need no
// file and would otherwise convert while the rest waited).
if !base_path.is_dir() {
return outcome;
}
let zone_name = camera_tz::display_zone_for(Some(display_tz));
let Ok(zone) = zone_name.parse::<chrono_tz::Tz>() else {
return outcome;
};
// Without exiftool, legacy EXIF/exiftool rows would fall through the
// empty-prior waterfall to fs_time and land a wrong instant. Leave
// them for a deploy that has it.
let exiftool_ok = date_resolver::exiftool_available();
// Page by id until `cap` convertible rows are collected. Rows that
// can't convert this tick are paged past, not counted against the
// cap, so a pile of missing files at low ids can't stall the drain.
let mut manual: Vec<UtcBackfillCandidate> = Vec::new();
let mut filed: Vec<(UtcBackfillCandidate, PathBuf)> = Vec::new();
let mut cursor = 0;
let mut scanned = 0i64;
'paging: loop {
let page = {
let mut dao = exif_dao.lock().expect("Unable to lock ExifDao");
dao.get_rows_needing_utc_backfill(context, library.id, cursor, cap)
.unwrap_or_default()
};
let page_len = page.len() as i64;
for row in page {
cursor = row.id;
scanned += 1;
match row.date_taken_source.as_deref() {
Some("manual") => manual.push(row),
Some("exif") | Some("exiftool") if !exiftool_ok => outcome.unresolved += 1,
_ => {
let abs = base_path.join(&row.rel_path);
if abs.exists() {
filed.push((row, abs));
} else {
// The missing-file scan retires these rows.
outcome.skipped_missing += 1;
}
}
}
if (manual.len() + filed.len()) as i64 >= cap {
break 'paging;
}
}
if page_len < cap || scanned >= cap * UTC_BACKFILL_SCAN_FACTOR {
break;
}
}
let more_remain = (manual.len() + filed.len()) as i64 >= cap;
// (candidate, date_utc, camera_offset_minutes, tz_source)
let mut conversions: Vec<(UtcBackfillCandidate, i64, Option<i32>, &'static str)> =
Vec::with_capacity(manual.len() + filed.len());
for row in manual {
match camera_tz::manual_date_resolution(Some(&zone_name), row.date_taken) {
Some((utc, offset)) => {
conversions.push((row, utc, Some(offset), camera_tz::TzSource::Manual.as_str()))
}
None => outcome.unresolved += 1,
}
}
if !filed.is_empty() {
// One exiftool subprocess for the whole batch — byte-identical
// call shape to the #117 audit preview.
let paths: Vec<PathBuf> = filed.iter().map(|(_, p)| p.clone()).collect();
let resolved = date_resolver::resolve_dates_batch(&paths, &HashMap::new());
for (row, abs) in filed {
let legacy_had_tag = matches!(
row.date_taken_source.as_deref(),
Some("exif") | Some("exiftool")
);
match resolved.get(&abs) {
Some(rd) if !(legacy_had_tag && rd.source == date_resolver::DateSource::FsTime) => {
match (rd.date_utc, rd.tz_source) {
(Some(utc), Some(tz)) => {
conversions.push((row, utc, rd.camera_offset_minutes, tz.as_str()))
}
_ => outcome.unresolved += 1,
}
}
// No result, or a row whose legacy date came from a tag
// now resolving to fs_time (exiftool failed on it): don't
// bake a filesystem time in over a real capture date.
_ => {
debug!(
"utc_backfill: no faithful resolution for lib {} {}",
library.id, row.rel_path
);
outcome.unresolved += 1;
}
}
}
}
let mut affected_days: HashSet<(u32, u32)> = HashSet::new();
{
let mut dao = exif_dao.lock().expect("Unable to lock ExifDao");
for (row, utc, offset, tz_source) in &conversions {
match dao.backfill_date_utc(context, row.id, *utc, *offset, tz_source) {
Ok(true) => {
outcome.converted += 1;
let before = display_day(zone, row.date_taken);
let after = display_day(zone, *utc);
if let (Some(before), Some(after)) = (before, after)
&& before != after
{
outcome.day_changed += 1;
affected_days.insert((before.month(), before.day()));
affected_days.insert((after.month(), after.day()));
}
}
// A manual edit / ingest raced us and already wrote it.
Ok(false) => {}
Err(e) => warn!(
"utc_backfill: update failed for lib {} {}: {:?}",
library.id, row.rel_path, e
),
}
}
}
if !affected_days.is_empty() {
outcome.reels_invalidated =
invalidate_day_reels(context, library, reel_dao, zone, &affected_days);
}
if outcome.converted > 0 || outcome.unresolved > 0 || more_remain {
info!(
"utc_backfill: library '{}': {:?}, cap={}, more_remain={}",
library.name, outcome, cap, more_remain
);
}
outcome
}
/// Calendar day of a unix instant in the display zone. The legacy value
/// goes through here too: the legacy `/memories` SQL rendered
/// `date_taken` as an instant in the client's zone, so that is the day
/// the row's derived day data was built from.
fn display_day(zone: chrono_tz::Tz, epoch: i64) -> Option<NaiveDate> {
zone.timestamp_opt(epoch, 0)
.single()
.map(|dt| dt.date_naive())
}
/// The month-day an "on this day" reel covers: the date it was generated
/// on, in the zone offset it was generated with. A pre-gen fallback path
/// has stored seconds rather than minutes; an out-of-range offset falls
/// back to the display zone.
fn reel_covered_day(reel: &PrecomputedReel, zone: chrono_tz::Tz) -> Option<NaiveDate> {
if reel.tz_offset_minutes.abs() <= 14 * 60 {
DateTime::from_timestamp(reel.generated_at + reel.tz_offset_minutes as i64 * 60, 0)
.map(|dt| dt.date_naive())
} else {
display_day(zone, reel.generated_at)
}
}
/// Delete the pre-generated `day` reels (this library's and the
/// all-libraries one) covering any of `affected_days`, plus their files.
/// With the ledger row gone the nightly pre-gen's freshness gate fails
/// and it renders a new one on its normal schedule. Returns rows deleted.
fn invalidate_day_reels(
context: &opentelemetry::Context,
library: &libraries::Library,
reel_dao: &Arc<Mutex<Box<dyn PrecomputedReelDao>>>,
zone: chrono_tz::Tz,
affected_days: &HashSet<(u32, u32)>,
) -> usize {
let keys = vec!["all".to_string(), library.id.to_string()];
let mut dao = reel_dao.lock().expect("Unable to lock PrecomputedReelDao");
let reels = match dao.list_for_span(context, "day", &keys) {
Ok(r) => r,
Err(e) => {
warn!("utc_backfill: listing day reels failed: {:?}", e);
return 0;
}
};
let stale: Vec<&PrecomputedReel> = reels
.iter()
.filter(|r| {
reel_covered_day(r, zone).is_some_and(|d| affected_days.contains(&(d.month(), d.day())))
})
.collect();
if stale.is_empty() {
return 0;
}
let ids: Vec<i32> = stale.iter().map(|r| r.id).collect();
match dao.delete_ids(context, &ids) {
Ok(n) => {
// Unlink too, so the on-demand cache can't serve the stale
// render under the same key.
for r in &stale {
let mp4 = std::path::Path::new(&r.output_path);
let _ = std::fs::remove_file(mp4);
let _ = std::fs::remove_file(mp4.with_extension("json"));
}
info!(
"utc_backfill: invalidated {} day reel(s) for library '{}' (re-dated days {:?})",
n, library.name, affected_days
);
n
}
Err(e) => {
warn!("utc_backfill: deleting stale day reels failed: {:?}", e);
0
}
}
}
/// Refresh the progress surface for one online library: store its
/// remaining count in `remaining` and set [`DATE_BACKFILL_REMAINING`] to
/// the sum. Offline libraries keep their last-known count (the watcher
/// doesn't call this for them), so a flapping share doesn't make the
/// gauge dip.
pub fn refresh_utc_backfill_progress(
context: &opentelemetry::Context,
library: &libraries::Library,
exif_dao: &Arc<Mutex<Box<dyn ExifDao>>>,
remaining: &mut HashMap<i32, i64>,
) {
let count = {
let mut dao = exif_dao.lock().expect("Unable to lock ExifDao");
dao.count_rows_needing_utc_backfill(context, library.id)
};
if let Ok(n) = count {
remaining.insert(library.id, n);
DATE_BACKFILL_REMAINING.set(remaining.values().sum());
}
}
/// Per-tick face-detection drain. Pulls a capped batch of hashed-but-
/// unscanned image_exif rows directly via the FaceDao anti-join and
/// hands them to the existing detection pass. Runs on every tick (not
@@ -799,4 +1120,540 @@ mod tests {
assert_eq!(candidates[0].rel_path, "fresh.jpg");
assert_eq!(candidates[0].content_hash, "hash-fresh");
}
// --- True-instant backfill drain (ADR-0001, ticket #118) -------------
use crate::database::SqlitePrecomputedReelDao;
use crate::database::models::{ImageExif, InsertablePrecomputedReel};
use chrono::NaiveDateTime;
const NY: &str = "America/New_York";
fn utc(y: i32, mo: u32, d: u32, h: u32, mi: u32, s: u32) -> i64 {
NaiveDate::from_ymd_opt(y, mo, d)
.unwrap()
.and_hms_opt(h, mi, s)
.unwrap()
.and_utc()
.timestamp()
}
fn reel_dao_for(
conn: &Arc<Mutex<diesel::SqliteConnection>>,
) -> Arc<Mutex<Box<dyn PrecomputedReelDao>>> {
Arc::new(Mutex::new(Box::new(
SqlitePrecomputedReelDao::from_connection(Arc::clone(conn)),
)))
}
/// Seed a row on the legacy convention: legacy pair set, new-contract
/// triple NULL. Returns the row id.
fn insert_legacy(
exif_dao: &Arc<Mutex<Box<dyn ExifDao>>>,
lib_id: i32,
rel: &str,
date_taken: Option<i64>,
source: Option<&str>,
) -> i32 {
let mut dao = exif_dao.lock().unwrap();
dao.store_exif(
&ctx(),
InsertImageExif {
library_id: lib_id,
file_path: rel.to_string(),
camera_make: None,
camera_model: None,
lens_model: None,
width: None,
height: None,
orientation: None,
gps_latitude: None,
gps_longitude: None,
gps_altitude: None,
focal_length: None,
aperture: None,
shutter_speed: None,
iso: None,
date_taken,
created_time: 0,
last_modified: 0,
content_hash: None,
size_bytes: None,
phash_64: None,
dhash_64: None,
date_taken_source: source.map(str::to_string),
date_utc: None,
camera_offset_minutes: None,
date_taken_tz_source: None,
},
)
.expect("insert")
.id
}
fn row(conn: &Arc<Mutex<diesel::SqliteConnection>>, id: i32) -> ImageExif {
let mut c = conn.lock().unwrap();
schema::image_exif::table
.find(id)
.first::<ImageExif>(&mut *c)
.expect("row")
}
fn remaining(exif_dao: &Arc<Mutex<Box<dyn ExifDao>>>, lib_id: i32) -> i64 {
exif_dao
.lock()
.unwrap()
.count_rows_needing_utc_backfill(&ctx(), lib_id)
.unwrap()
}
fn tools_available() -> bool {
let ok = |bin: &str, arg: &str| {
std::process::Command::new(bin)
.arg(arg)
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status()
.map(|s| s.success())
.unwrap_or(false)
};
ok("ffmpeg", "-version") && ok("exiftool", "-ver")
}
fn run_ok(cmd: &mut std::process::Command) {
let out = cmd.output().expect("spawn fixture tool");
assert!(
out.status.success(),
"fixture command failed: {}",
String::from_utf8_lossy(&out.stderr)
);
}
/// Tiny H.264 MP4 whose container `creation_time` exiftool reads as
/// QuickTime / Track* groups (→ `utc_direct`).
fn write_container_video(root: &std::path::Path, rel: &str, creation_time: &str) {
run_ok(
std::process::Command::new("ffmpeg")
.args([
"-y",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
"color=c=black:s=32x32:d=0.4",
"-r",
"10",
"-c:v",
"libx264",
"-pix_fmt",
"yuv420p",
"-metadata",
])
.arg(format!("creation_time={creation_time}"))
.arg(root.join(rel)),
);
}
/// Tiny JPEG carrying `DateTimeOriginal` + `OffsetTimeOriginal`.
fn write_exif_jpeg(root: &std::path::Path, rel: &str, dto: &str, offset: &str) {
let abs = root.join(rel);
run_ok(
std::process::Command::new("ffmpeg")
.args([
"-y",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
"color=c=gray:s=16x16",
"-frames:v",
"1",
])
.arg(&abs),
);
run_ok(
std::process::Command::new("exiftool")
.arg("-q")
.arg("-overwrite_original")
.arg(format!("-DateTimeOriginal={dto}"))
.arg(format!("-OffsetTimeOriginal={offset}"))
.arg(&abs),
);
}
#[test]
fn utc_backfill_converts_one_row_per_source_and_is_idempotent() {
if !tools_available() {
eprintln!("skipping: ffmpeg/exiftool not on PATH");
return;
}
let (tmp, library, conn, exif_dao, _face_dao) = setup();
let reel_dao = reel_dao_for(&conn);
// exif: camera wall 22:30 at +02:00 → 20:30Z. Legacy = wall-as-UTC.
write_exif_jpeg(tmp.path(), "IMG_0001.jpg", "2024:07:04 22:30:00", "+02:00");
let exif_legacy = utc(2024, 7, 4, 22, 30, 0);
let exif_id = insert_legacy(
&exif_dao,
1,
"IMG_0001.jpg",
Some(exif_legacy),
Some("exif"),
);
// exiftool/container: creation_time is UTC. Legacy = the old
// server-local interpretation (EDT, +4h) — the "videos off by 4–5
// hours" row this drain exists to fix.
write_container_video(tmp.path(), "clip.mp4", "2017-06-11T15:58:23Z");
let video_legacy = utc(2017, 6, 11, 19, 58, 23);
let video_id = insert_legacy(
&exif_dao,
1,
"clip.mp4",
Some(video_legacy),
Some("exiftool"),
);
// filename (wall pattern): legacy wall-as-UTC; no evidence → server zone.
let fname = "Screenshot_2014-06-01-20-44-50.png";
write_image(tmp.path(), fname, b"not-a-png");
let fname_wall: NaiveDateTime = NaiveDate::from_ymd_opt(2014, 6, 1)
.unwrap()
.and_hms_opt(20, 44, 50)
.unwrap();
let fname_legacy = fname_wall.and_utc().timestamp();
let fname_id = insert_legacy(&exif_dao, 1, fname, Some(fname_legacy), Some("filename"));
// fs_time: no date anywhere in the file; legacy is already a true instant.
write_image(tmp.path(), "scan.jpg", b"no-exif-here");
let meta = fs::metadata(tmp.path().join("scan.jpg")).unwrap();
let fs_instant: chrono::DateTime<chrono::Utc> =
crate::utils::earliest_fs_time(&meta).unwrap().into();
let fs_instant = fs_instant.timestamp();
let fs_id = insert_legacy(&exif_dao, 1, "scan.jpg", Some(fs_instant), Some("fs_time"));
// manual (pre-#115 override): the operator's wall in the display
// zone — never re-resolved from the file.
write_image(tmp.path(), "edited.jpg", b"no-exif-here-either");
let manual_legacy = utc(2020, 1, 1, 12, 0, 0);
let manual_id = insert_legacy(
&exif_dao,
1,
"edited.jpg",
Some(manual_legacy),
Some("manual"),
);
assert_eq!(remaining(&exif_dao, 1), 5);
let out = backfill_utc_dates_capped(&ctx(), &library, &exif_dao, &reel_dao, NY, 500);
assert_eq!(out.converted, 5, "{out:?}");
assert_eq!(out.unresolved, 0, "{out:?}");
let legacy = |r: &ImageExif| (r.date_taken, r.date_taken_source.clone());
let r = row(&conn, exif_id);
assert_eq!(r.date_utc, Some(utc(2024, 7, 4, 20, 30, 0)));
assert_eq!(r.camera_offset_minutes, Some(120));
assert_eq!(r.date_taken_tz_source.as_deref(), Some("offset_tag"));
assert_eq!(legacy(&r), (Some(exif_legacy), Some("exif".into())));
let r = row(&conn, video_id);
assert_eq!(r.date_utc, Some(utc(2017, 6, 11, 15, 58, 23)));
assert_eq!(r.camera_offset_minutes, None);
assert_eq!(r.date_taken_tz_source.as_deref(), Some("utc_direct"));
assert_eq!(legacy(&r), (Some(video_legacy), Some("exiftool".into())));
let r = row(&conn, fname_id);
assert_eq!(r.date_taken_tz_source.as_deref(), Some("assumed_server"));
if let Some(zone) = crate::camera_tz::server_zone() {
let off = crate::camera_tz::offset_minutes_for_zone(zone, fname_wall);
assert_eq!(r.camera_offset_minutes, Some(off));
assert_eq!(
r.date_utc,
Some(crate::camera_tz::wall_to_instant(fname_wall, off))
);
}
assert_eq!(legacy(&r), (Some(fname_legacy), Some("filename".into())));
let r = row(&conn, fs_id);
assert_eq!(r.date_utc, Some(fs_instant));
assert_eq!(r.camera_offset_minutes, None);
assert_eq!(r.date_taken_tz_source.as_deref(), Some("utc_direct"));
assert_eq!(legacy(&r), (Some(fs_instant), Some("fs_time".into())));
let r = row(&conn, manual_id);
// 2020-01-01 12:00 EST (-300) → 17:00Z.
assert_eq!(r.date_utc, Some(utc(2020, 1, 1, 17, 0, 0)));
assert_eq!(r.camera_offset_minutes, Some(-300));
assert_eq!(r.date_taken_tz_source.as_deref(), Some("manual"));
assert_eq!(legacy(&r), (Some(manual_legacy), Some("manual".into())));
// Second pass: nothing eligible, nothing changes, progress is 0.
let ids = [exif_id, video_id, fname_id, fs_id, manual_id];
let snapshot = |conn: &Arc<Mutex<diesel::SqliteConnection>>| {
ids.iter()
.map(|id| {
let r = row(conn, *id);
(
r.date_taken,
r.date_taken_source,
r.date_utc,
r.camera_offset_minutes,
r.date_taken_tz_source,
)
})
.collect::<Vec<_>>()
};
let before = snapshot(&conn);
let again = backfill_utc_dates_capped(&ctx(), &library, &exif_dao, &reel_dao, NY, 500);
assert_eq!(again, UtcBackfillOutcome::default());
assert_eq!(snapshot(&conn), before);
let mut progress = HashMap::new();
refresh_utc_backfill_progress(&ctx(), &library, &exif_dao, &mut progress);
assert_eq!(progress.get(&1), Some(&0));
}
#[test]
fn utc_backfill_leaves_null_date_rows_and_other_libraries_alone() {
let (tmp, library, conn, exif_dao, _face_dao) = setup();
let reel_dao = reel_dao_for(&conn);
write_image(tmp.path(), "1500000000.jpg", b"x");
write_image(tmp.path(), "undated.jpg", b"y");
let ts = 1_500_000_000;
let converted = insert_legacy(&exif_dao, 1, "1500000000.jpg", Some(ts), Some("filename"));
// Both NULL → the legacy drain's job (its writer fills the triple).
let undated = insert_legacy(&exif_dao, 1, "undated.jpg", None, None);
let other = insert_legacy(&exif_dao, 2, "1500000000.jpg", Some(ts), Some("filename"));
let out = backfill_utc_dates_capped(&ctx(), &library, &exif_dao, &reel_dao, NY, 500);
assert_eq!(out.converted, 1);
assert_eq!(row(&conn, converted).date_utc, Some(ts));
assert_eq!(row(&conn, undated).date_utc, None);
assert_eq!(row(&conn, other).date_utc, None);
// Progress counts every row not yet on the new contract.
assert_eq!(remaining(&exif_dao, 1), 1);
}
#[test]
fn utc_backfill_respects_cap_and_missing_files_dont_spend_it() {
let (tmp, library, conn, exif_dao, _face_dao) = setup();
let reel_dao = reel_dao_for(&conn);
// Missing files at the lowest ids, ahead of the present ones.
// Timestamp filenames keep the fixture hermetic (never hit exiftool).
let mut missing = Vec::new();
for i in 0..3 {
let ts = 1_400_000_000 + i;
let rel = format!("{ts}.jpg");
missing.push(insert_legacy(
&exif_dao,
1,
&rel,
Some(ts),
Some("filename"),
));
}
for i in 0..5 {
let ts = 1_500_000_000 + i;
let rel = format!("{ts}.jpg");
write_image(tmp.path(), &rel, b"z");
insert_legacy(&exif_dao, 1, &rel, Some(ts), Some("filename"));
}
assert_eq!(remaining(&exif_dao, 1), 8);
let mut progress = HashMap::new();
let mut seen = Vec::new();
for _ in 0..4 {
let out = backfill_utc_dates_capped(&ctx(), &library, &exif_dao, &reel_dao, NY, 2);
refresh_utc_backfill_progress(&ctx(), &library, &exif_dao, &mut progress);
seen.push((out.converted, progress[&1]));
}
// Cap holds (2 per tick) and the 3 missing rows never spend it.
assert_eq!(seen, vec![(2, 6), (2, 4), (1, 3), (0, 3)]);
// The missing-file scan retires the rest → progress reaches 0.
{
let mut c = conn.lock().unwrap();
diesel::delete(
schema::image_exif::table.filter(schema::image_exif::id.eq_any(&missing)),
)
.execute(&mut *c)
.unwrap();
}
refresh_utc_backfill_progress(&ctx(), &library, &exif_dao, &mut progress);
assert_eq!(progress[&1], 0);
}
#[test]
fn utc_backfill_skips_unavailable_library_then_resumes() {
let (tmp, library, conn, exif_dao, _face_dao) = setup();
let reel_dao = reel_dao_for(&conn);
write_image(tmp.path(), "1500000000.jpg", b"z");
let filed = insert_legacy(
&exif_dao,
1,
"1500000000.jpg",
Some(1_500_000_000),
Some("filename"),
);
// A manual row needs no file — it must still wait for the mount.
let manual = insert_legacy(
&exif_dao,
1,
"edited.jpg",
Some(utc(2020, 1, 1, 12, 0, 0)),
Some("manual"),
);
let unmounted = Library {
root_path: tmp.path().join("unmounted").to_string_lossy().to_string(),
..library.clone()
};
// The watcher gate sees it stale…
let health_map = libraries::new_health_map(std::slice::from_ref(&unmounted));
assert!(!libraries::refresh_health(&health_map, &unmounted, true).is_online());
// …and the drain itself converts nothing even if called.
let out = backfill_utc_dates_capped(&ctx(), &unmounted, &exif_dao, &reel_dao, NY, 500);
assert_eq!(out, UtcBackfillOutcome::default());
assert_eq!(row(&conn, filed).date_utc, None);
assert_eq!(row(&conn, manual).date_utc, None);
// Share returns → picked up next tick.
assert!(libraries::refresh_health(&health_map, &library, true).is_online());
let out = backfill_utc_dates_capped(&ctx(), &library, &exif_dao, &reel_dao, NY, 500);
assert_eq!(out.converted, 2);
assert_eq!(remaining(&exif_dao, 1), 0);
}
/// Seed a pre-generated reel covering `month`-`day` (generated 03:00
/// EDT that day, as the nightly pre-gen does). Returns (id, mp4 path).
fn seed_reel(
reel_dao: &Arc<Mutex<Box<dyn PrecomputedReelDao>>>,
dir: &std::path::Path,
span: &str,
library_key: &str,
month: u32,
day: u32,
) -> (i32, PathBuf) {
let mp4 = dir.join(format!("reel-{span}-{library_key}-{month}-{day}.mp4"));
fs::write(&mp4, b"mp4").unwrap();
let id = reel_dao
.lock()
.unwrap()
.record_reel(
&ctx(),
&InsertablePrecomputedReel {
span: span.to_string(),
library_key: library_key.to_string(),
cache_key: format!("{span}-{library_key}-{month}-{day}"),
output_path: mp4.to_string_lossy().to_string(),
title: "t".to_string(),
media_count: 1,
render_version: 1,
tz_offset_minutes: -240,
voice: None,
generated_at: utc(2025, month, day, 7, 0, 0),
},
)
.unwrap();
(id, mp4)
}
fn reel_ids(conn: &Arc<Mutex<diesel::SqliteConnection>>) -> Vec<i32> {
let mut c = conn.lock().unwrap();
schema::precomputed_reels::table
.select(schema::precomputed_reels::id)
.order(schema::precomputed_reels::id.asc())
.load::<i32>(&mut *c)
.unwrap()
}
#[test]
fn utc_backfill_invalidates_exactly_the_moved_days_reels() {
let (tmp, library, conn, exif_dao, _face_dao) = setup();
let reel_dao = reel_dao_for(&conn);
let reels_dir = TempDir::new().unwrap();
// Manual wall 2017-06-12 02:00 in NY. The legacy value, read as an
// instant in NY by the old /memories SQL, showed 06-11 22:00; the
// true instant is 06-12 06:00Z = 06-12 02:00 EDT → the day moves
// 06-11 → 06-12.
let moved = insert_legacy(
&exif_dao,
1,
"moved.jpg",
Some(utc(2017, 6, 12, 2, 0, 0)),
Some("manual"),
);
// Unchanged: a timestamp-filename row's legacy value is already the instant.
write_image(tmp.path(), "1400000000.jpg", b"z");
insert_legacy(
&exif_dao,
1,
"1400000000.jpg",
Some(1_400_000_000),
Some("filename"),
);
let (_, mp4_11) = seed_reel(&reel_dao, reels_dir.path(), "day", "1", 6, 11);
let (_, mp4_12) = seed_reel(&reel_dao, reels_dir.path(), "day", "all", 6, 12);
let (other_lib, _) = seed_reel(&reel_dao, reels_dir.path(), "day", "2", 6, 12);
let (other_day, mp4_0301) = seed_reel(&reel_dao, reels_dir.path(), "day", "1", 3, 1);
let (week, _) = seed_reel(&reel_dao, reels_dir.path(), "week", "1", 6, 12);
let out = backfill_utc_dates_capped(&ctx(), &library, &exif_dao, &reel_dao, NY, 500);
assert_eq!(out.converted, 2);
assert_eq!(out.day_changed, 1, "{out:?}");
assert_eq!(out.reels_invalidated, 2, "{out:?}");
assert_eq!(row(&conn, moved).date_utc, Some(utc(2017, 6, 12, 6, 0, 0)));
// Exactly the moved days' day reels for this library (+ "all") are gone.
assert_eq!(reel_ids(&conn), vec![other_lib, other_day, week]);
assert!(
!mp4_11.exists() && !mp4_12.exists(),
"stale renders unlinked"
);
assert!(mp4_0301.exists());
// With the ledger row gone the nightly pre-gen's freshness gate
// (`latest_for` must match the new cache key) can't skip → it
// regenerates on its normal schedule; no new scheduler involved.
let latest = reel_dao
.lock()
.unwrap()
.latest_for(&ctx(), "day", "all")
.unwrap();
assert!(latest.is_none());
}
#[test]
fn utc_backfill_utc_midnight_straddle_marks_nothing() {
let (_tmp, library, conn, exif_dao, _face_dao) = setup();
let reel_dao = reel_dao_for(&conn);
let reels_dir = TempDir::new().unwrap();
// Manual wall 2017-06-11 23:30 in NY. Legacy instant 06-11 23:30Z
// (NY 19:30); true instant 06-12 03:30Z (NY 23:30). The UTC date
// flips 06-11 → 06-12, but the display-zone day stays 06-11.
let straddle = insert_legacy(
&exif_dao,
1,
"late.jpg",
Some(utc(2017, 6, 11, 23, 30, 0)),
Some("manual"),
);
seed_reel(&reel_dao, reels_dir.path(), "day", "1", 6, 11);
seed_reel(&reel_dao, reels_dir.path(), "day", "all", 6, 12);
let out = backfill_utc_dates_capped(&ctx(), &library, &exif_dao, &reel_dao, NY, 500);
assert_eq!(out.converted, 1);
assert_eq!(out.day_changed, 0);
assert_eq!(out.reels_invalidated, 0);
assert_eq!(
row(&conn, straddle).date_utc,
Some(utc(2017, 6, 12, 3, 30, 0))
);
assert_eq!(reel_ids(&conn).len(), 2);
}
}
+140
View File
@@ -43,6 +43,17 @@ pub struct DuplicateRow {
pub duplicate_decided_at: Option<i64>,
}
/// One row the true-instant backfill drain (ADR-0001, ticket #118) still
/// has to convert: on the legacy convention (`date_utc IS NULL`) with a
/// legacy `date_taken` to compare the converted day against.
#[derive(Debug, Clone, Queryable)]
pub struct UtcBackfillCandidate {
pub id: i32,
pub rel_path: String,
pub date_taken: i64,
pub date_taken_source: Option<String>,
}
pub mod calendar_dao;
pub mod daily_summary_dao;
pub mod insight_generation_job_dao;
@@ -513,6 +524,46 @@ pub trait ExifDao: Sync + Send {
date_taken_tz_source: Option<&str>,
) -> Result<(), DbError>;
/// Rows the true-instant drain (ADR-0001, ticket #118) still has to
/// convert: `date_utc IS NULL AND date_taken IS NOT NULL` in
/// `library_id`, `id > after_id`, ordered by id ASC. Rows with both
/// NULL belong to `get_rows_needing_date_backfill` (whose writer fills
/// the new triple too). The `after_id` cursor lets the drain page past
/// rows it can't convert this tick (missing files) without spending
/// its per-tick cap on them. Backed by `idx_image_exif_utc_backfill`.
fn get_rows_needing_utc_backfill(
&mut self,
context: &opentelemetry::Context,
library_id: i32,
after_id: i32,
limit: i64,
) -> Result<Vec<UtcBackfillCandidate>, DbError>;
/// Persist a converted row's new-contract triple (`date_utc`,
/// `camera_offset_minutes`, `date_taken_tz_source`) by primary key.
/// Never touches the frozen legacy `date_taken` / `date_taken_source`
/// pair. Guarded on `date_utc IS NULL` so a manual edit or ingest
/// that raced the drain wins. Returns whether a row was written.
fn backfill_date_utc(
&mut self,
context: &opentelemetry::Context,
id: i32,
date_utc: i64,
camera_offset_minutes: Option<i32>,
date_taken_tz_source: &str,
) -> Result<bool, DbError>;
/// Progress surface for the true-instant drain: rows in `library_id`
/// not yet on the new contract (`date_utc IS NULL`). Counts every such
/// row — including NULL-date rows the legacy drain resolves and rows
/// whose file is missing (until the missing-file scan retires them) —
/// because cutover (#120) waits on exactly this set reaching zero.
fn count_rows_needing_utc_backfill(
&mut self,
context: &opentelemetry::Context,
library_id: i32,
) -> Result<i64, DbError>;
/// Find image_exif rows needing a CLIP embedding for semantic search:
/// `clip_embedding IS NULL AND content_hash IS NOT NULL`, ordered by id
/// ASC, limited. Hash-less rows wait for `backfill_unhashed_backlog` to
@@ -1501,6 +1552,95 @@ impl ExifDao for SqliteExifDao {
})
}
fn get_rows_needing_utc_backfill(
&mut self,
context: &opentelemetry::Context,
library_id_val: i32,
after_id: i32,
limit: i64,
) -> Result<Vec<UtcBackfillCandidate>, DbError> {
trace_db_call(context, "query", "get_rows_needing_utc_backfill", |_span| {
use schema::image_exif::dsl::*;
let mut connection = self.connection.lock().expect("Unable to get ExifDao");
// Partial index `(library_id, id) WHERE date_utc IS NULL`
// serves the filter + order directly.
image_exif
.filter(library_id.eq(library_id_val))
.filter(date_utc.is_null())
.filter(date_taken.is_not_null())
.filter(id.gt(after_id))
.select((
id,
rel_path,
date_taken.assume_not_null(),
date_taken_source,
))
.order(id.asc())
.limit(limit)
.load::<UtcBackfillCandidate>(connection.deref_mut())
.map_err(|e| anyhow::anyhow!("Query error: {}", e))
})
.map_err(|e| DbError::log(DbErrorKind::QueryError, e))
}
fn backfill_date_utc(
&mut self,
context: &opentelemetry::Context,
id_val: i32,
date_utc_val: i64,
camera_offset_minutes_val: Option<i32>,
date_taken_tz_source_val: &str,
) -> Result<bool, DbError> {
trace_db_call(context, "update", "backfill_date_utc", |_span| {
use schema::image_exif::dsl::*;
let mut connection = self.connection.lock().expect("Unable to get ExifDao");
// New-contract triple only — the legacy pair is frozen
// (ADR-0001).
diesel::update(image_exif.find(id_val).filter(date_utc.is_null()))
.set((
date_utc.eq(Some(date_utc_val)),
camera_offset_minutes.eq(camera_offset_minutes_val),
date_taken_tz_source.eq(Some(date_taken_tz_source_val)),
))
.execute(connection.deref_mut())
.map(|rows| rows > 0)
.map_err(|e| anyhow::anyhow!("diesel update failed (id={}): {}", id_val, e))
})
.map_err(|e| {
log::warn!("backfill_date_utc: {}", e);
DbError::new(DbErrorKind::UpdateError)
})
}
fn count_rows_needing_utc_backfill(
&mut self,
context: &opentelemetry::Context,
library_id_val: i32,
) -> Result<i64, DbError> {
trace_db_call(
context,
"query",
"count_rows_needing_utc_backfill",
|_span| {
use schema::image_exif::dsl::*;
let mut connection = self.connection.lock().expect("Unable to get ExifDao");
image_exif
.filter(library_id.eq(library_id_val))
.filter(date_utc.is_null())
.count()
.get_result::<i64>(connection.deref_mut())
.map_err(|e| anyhow::anyhow!("Query error: {}", e))
},
)
.map_err(|e| DbError::log(DbErrorKind::QueryError, e))
}
fn list_clip_unencoded_candidates(
&mut self,
context: &opentelemetry::Context,
+61
View File
@@ -58,6 +58,24 @@ pub trait PrecomputedReelDao: Sync + Send {
/// to protect files a ledger row still points at.
#[allow(dead_code)]
fn all_cache_keys(&mut self, context: &opentelemetry::Context) -> Result<Vec<String>, DbError>;
/// Every reel row for `span` whose `library_key` is one of
/// `library_keys`. Used by the true-instant date drain (ticket #118)
/// to find the day reels a re-dated photo invalidates.
fn list_for_span(
&mut self,
context: &opentelemetry::Context,
span: &str,
library_keys: &[String],
) -> Result<Vec<PrecomputedReel>, DbError>;
/// Delete the given reel rows (stale-marking: the nightly pre-gen
/// finds no current row and regenerates). Returns rows deleted.
fn delete_ids(
&mut self,
context: &opentelemetry::Context,
ids: &[i32],
) -> Result<usize, DbError>;
}
pub struct SqlitePrecomputedReelDao {
@@ -219,6 +237,49 @@ impl PrecomputedReelDao for SqlitePrecomputedReelDao {
})
.map_err(|e| DbError::log(DbErrorKind::QueryError, e))
}
fn list_for_span(
&mut self,
context: &opentelemetry::Context,
span: &str,
library_keys: &[String],
) -> Result<Vec<PrecomputedReel>, DbError> {
trace_db_call(context, "query", "list_for_span", |_span| {
use schema::precomputed_reels::dsl;
let mut connection = self
.connection
.lock()
.expect("Unable to lock PrecomputedReelDao");
dsl::precomputed_reels
.filter(dsl::span.eq(span))
.filter(dsl::library_key.eq_any(library_keys))
.load::<PrecomputedReel>(connection.deref_mut())
.map_err(|e| anyhow::anyhow!("Failed to list reels for span: {}", e))
})
.map_err(|e| DbError::log(DbErrorKind::QueryError, e))
}
fn delete_ids(
&mut self,
context: &opentelemetry::Context,
ids: &[i32],
) -> Result<usize, DbError> {
trace_db_call(context, "delete", "delete_ids", |_span| {
use schema::precomputed_reels::dsl;
let mut connection = self
.connection
.lock()
.expect("Unable to lock PrecomputedReelDao");
diesel::delete(dsl::precomputed_reels.filter(dsl::id.eq_any(ids)))
.execute(connection.deref_mut())
.map_err(|e| anyhow::anyhow!("Failed to delete reels: {}", e))
})
.map_err(|e| DbError::log(DbErrorKind::UpdateError, e))
}
}
#[cfg(test)]
+1 -1
View File
@@ -351,7 +351,7 @@ const EXIFTOOL_EVIDENCE_TAGS: &[&str] = &[
/// Cache the "exiftool exists on PATH" check across the process lifetime so
/// the per-tick backfill doesn't fork a doomed subprocess every iteration on
/// deploys without exiftool installed.
fn exiftool_available() -> bool {
pub fn exiftool_available() -> bool {
static AVAIL: OnceLock<bool> = OnceLock::new();
*AVAIL.get_or_init(|| {
let ok = Command::new("exiftool")
+29
View File
@@ -1756,6 +1756,35 @@ mod tests {
Ok(())
}
fn get_rows_needing_utc_backfill(
&mut self,
_context: &opentelemetry::Context,
_library_id: i32,
_after_id: i32,
_limit: i64,
) -> Result<Vec<crate::database::UtcBackfillCandidate>, DbError> {
Ok(Vec::new())
}
fn backfill_date_utc(
&mut self,
_context: &opentelemetry::Context,
_id: i32,
_date_utc: i64,
_camera_offset_minutes: Option<i32>,
_date_taken_tz_source: &str,
) -> Result<bool, DbError> {
Ok(false)
}
fn count_rows_needing_utc_backfill(
&mut self,
_context: &opentelemetry::Context,
_library_id: i32,
) -> Result<i64, DbError> {
Ok(0)
}
fn set_manual_date_taken(
&mut self,
_context: &opentelemetry::Context,
+15
View File
@@ -460,6 +460,10 @@ pub struct LibraryStatus {
#[serde(flatten)]
pub library: Library,
pub health: LibraryHealth,
/// Rows in this library not yet converted to the true-instant date
/// contract (`date_utc IS NULL`, ADR-0001 / ticket #118). Drains to 0
/// over watcher ticks; `None` when the count query failed.
pub date_backfill_remaining: Option<i64>,
}
#[derive(serde::Serialize)]
@@ -488,6 +492,12 @@ pub async fn list_libraries(_claims: Claims, app_state: Data<AppState>) -> impl
.library_health
.read()
.unwrap_or_else(|e| e.into_inner());
let context = opentelemetry::Context::new();
let mut exif_dao = app_state
.insight_generator
.exif_dao()
.lock()
.expect("Unable to lock ExifDao");
let libraries = live_guard
.iter()
.map(|lib| LibraryStatus {
@@ -496,6 +506,11 @@ pub async fn list_libraries(_claims: Claims, app_state: Data<AppState>) -> impl
.get(&lib.id)
.cloned()
.unwrap_or(LibraryHealth::Online),
// Live count off the partial index — cheap, and exact even for
// a stale library the watcher isn't refreshing.
date_backfill_remaining: exif_dao
.count_rows_needing_utc_backfill(&context, lib.id)
.ok(),
})
.collect();
HttpResponse::Ok().json(LibrariesResponse {
+6
View File
@@ -151,6 +151,12 @@ fn main() -> std::io::Result<()> {
.registry
.register(Box::new(thumbnails::VIDEO_GAUGE.clone()))
.unwrap();
// True-instant date drain progress (#118). Refreshed per watcher
// tick; see `backfill::refresh_utc_backfill_progress`.
prometheus
.registry
.register(Box::new(backfill::DATE_BACKFILL_REMAINING.clone()))
.unwrap();
// HLS readiness gauges. Updated by the watcher every full-scan
// tick and on every `/hls/stats` request. See `hls_stats`.
prometheus
+44 -1
View File
@@ -26,7 +26,10 @@ use log::{debug, error, info, warn};
use crate::backfill;
use crate::content_hash;
use crate::database::models::InsertImageExif;
use crate::database::{ExifDao, PreviewDao, SqliteExifDao, SqlitePreviewDao};
use crate::database::{
ExifDao, PrecomputedReelDao, PreviewDao, SqliteExifDao, SqlitePrecomputedReelDao,
SqlitePreviewDao, SqliteUserAiPrefsDao, UserAiPrefsDao,
};
use crate::date_resolver;
use crate::exif;
use crate::face_watch;
@@ -336,6 +339,17 @@ pub fn watch_files(
Box::new(SqliteTagDao::default()) as Box<dyn tags::TagDao>
));
// True-instant date drain (#118): day-reel invalidation + the
// display zone the day comparison runs in.
let precomputed_reel_dao = Arc::new(Mutex::new(
Box::new(SqlitePrecomputedReelDao::new()) as Box<dyn PrecomputedReelDao>
));
let user_ai_prefs_dao: Mutex<Box<dyn UserAiPrefsDao>> =
Mutex::new(Box::new(SqliteUserAiPrefsDao::new()));
// Last-known per-library remaining count behind the
// `imageserver_date_backfill_remaining` gauge.
let mut utc_backfill_remaining: HashMap<i32, i64> = HashMap::new();
let mut last_quick_scan = SystemTime::now();
// Initialize to UNIX_EPOCH so the *first* tick is treated as a
// full scan. That replaces the legacy startup ScanDirectoryMessage
@@ -494,6 +508,35 @@ pub fn watch_files(
{
let context = opentelemetry::Context::new();
backfill::backfill_missing_date_taken(&context, lib, &exif_dao);
// True-instant drain (ADR-0001, #118): convert
// legacy mixed-convention rows, invalidating day
// reels whose day moved. Inside the online gate, so a
// stale library is skipped whole and resumes next
// tick. The display zone is re-read per tick so a
// prefs change applies without restart.
let display_tz = {
let mut prefs = user_ai_prefs_dao.lock().expect("prefs dao poisoned");
match prefs.get_prefs(&context) {
Ok(Some(p)) => {
crate::camera_tz::display_zone_for(p.display_tz.as_deref())
}
_ => crate::camera_tz::display_zone_for(None),
}
};
backfill::backfill_utc_dates(
&context,
lib,
&exif_dao,
&precomputed_reel_dao,
&display_tz,
);
backfill::refresh_utc_backfill_progress(
&context,
lib,
&exif_dao,
&mut utc_backfill_remaining,
);
}
if is_full_scan {