feat(dates): backfill drain converts mixed-convention rows to true instants (#118)
Per-tick drain (backfill_utc_dates) at the existing date-drain seam, inside the per-library online gate. Selects date_utc IS NULL AND date_taken IS NOT NULL (new partial index idx_image_exif_utc_backfill), re-runs resolve_dates_batch with empty priors (one exiftool subprocess per batch, same call shape as the #117 audit), and writes only the new-contract triple; the legacy pair stays frozen. Capped by DATE_BACKFILL_MAX_PER_TICK; missing-file rows are paged past without spending the cap. Legacy manual overrides convert via manual_date_resolution rather than being re-resolved from the file. Day-boundary invalidation deletes the day-span precomputed reels covering a re-dated row's old/new display-zone month-day so the nightly pre-gen regenerates them. Progress: imageserver_date_backfill_remaining gauge + date_backfill_remaining on GET /libraries. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3ac011abd7
commit
1fad08dea3
11 files changed
+1185
-4
No files matched your search
@@ -437,6 +437,26 @@ a stronger tool comes online (exiftool install, new filename regex),
|
|||||||
re-resolve out-of-band rather than re-introducing the steady-state
|
re-resolve out-of-band rather than re-introducing the steady-state
|
||||||
eligibility.
|
eligibility.
|
||||||
|
|
||||||
|
**True-instant drain (`backfill_utc_dates`, ticket #118).** Runs right
|
||||||
|
after `backfill_missing_date_taken` inside the per-library online gate,
|
||||||
|
sharing its `DATE_BACKFILL_MAX_PER_TICK` cap. Converts legacy
|
||||||
|
mixed-convention rows (`date_utc IS NULL AND date_taken IS NOT NULL`,
|
||||||
|
partial index `idx_image_exif_utc_backfill`) by re-running
|
||||||
|
`resolve_dates_batch` with empty priors (one exiftool subprocess per
|
||||||
|
batch) and writing **only** the new-contract triple — the legacy pair
|
||||||
|
stays frozen. Legacy `manual` rows convert via
|
||||||
|
`camera_tz::manual_date_resolution` instead of being re-resolved from
|
||||||
|
the file. Missing-file rows are paged past without spending the cap;
|
||||||
|
EXIF/exiftool-legacy rows wait when exiftool is absent (they'd otherwise
|
||||||
|
fall to fs_time). When a converted row's display-zone day differs from
|
||||||
|
the day its legacy value rendered on, the `day`-span `precomputed_reels`
|
||||||
|
rows covering either month-day (this library + `all`) are deleted and
|
||||||
|
unlinked; the nightly pre-gen regenerates. Daily conversation summaries
|
||||||
|
are SMS-derived and unaffected. Progress: Prometheus
|
||||||
|
`imageserver_date_backfill_remaining` (per tick) and
|
||||||
|
`date_backfill_remaining` per library on `GET /libraries` (live count of
|
||||||
|
`date_utc IS NULL`).
|
||||||
|
|
||||||
`/memories` is a single SQL query against this column
|
`/memories` is a single SQL query against this column
|
||||||
(`get_memories_in_window` in `src/database/mod.rs`), using
|
(`get_memories_in_window` in `src/database/mod.rs`), using
|
||||||
`strftime('%m-%d' | '%W' | '%m', date_taken, 'unixepoch', tz)` for
|
`strftime('%m-%d' | '%W' | '%m', date_taken, 'unixepoch', tz)` for
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
DROP INDEX IF EXISTS idx_image_exif_utc_backfill;
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
-- Partial index backing the true-instant backfill drain (ADR-0001, ticket
|
||||||
|
-- #118). The drain selects rows still on the legacy convention
|
||||||
|
-- (`date_utc IS NULL`) per library in id order, and the progress surface
|
||||||
|
-- (Prometheus `imageserver_date_backfill_remaining` + `/libraries`) counts
|
||||||
|
-- the same set. Mirrors `idx_image_exif_date_backfill`: the index shrinks
|
||||||
|
-- to nothing as the drain converges, so it costs ~0 after cutover.
|
||||||
|
CREATE INDEX idx_image_exif_utc_backfill
|
||||||
|
ON image_exif (library_id, id)
|
||||||
|
WHERE date_utc IS NULL;
|
||||||
+859
-2
@@ -6,14 +6,19 @@
|
|||||||
//! wouldn't otherwise drain except during the once-an-hour full scan.
|
//! wouldn't otherwise drain except during the once-an-hour full scan.
|
||||||
//! Each function is bounded per call by a `*_PER_TICK` env-var cap.
|
//! Each function is bounded per call by a `*_PER_TICK` env-var cap.
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::{HashMap, HashSet};
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
|
use chrono::{DateTime, Datelike, NaiveDate, TimeZone};
|
||||||
|
use lazy_static::lazy_static;
|
||||||
use log::{debug, info, warn};
|
use log::{debug, info, warn};
|
||||||
|
use prometheus::IntGauge;
|
||||||
|
|
||||||
|
use crate::camera_tz;
|
||||||
use crate::content_hash;
|
use crate::content_hash;
|
||||||
use crate::database::ExifDao;
|
use crate::database::models::PrecomputedReel;
|
||||||
|
use crate::database::{ExifDao, PrecomputedReelDao, UtcBackfillCandidate};
|
||||||
use crate::date_resolver;
|
use crate::date_resolver;
|
||||||
use crate::face_watch;
|
use crate::face_watch;
|
||||||
use crate::faces;
|
use crate::faces;
|
||||||
@@ -219,6 +224,322 @@ pub fn backfill_missing_date_taken(
|
|||||||
backfilled
|
backfilled
|
||||||
}
|
}
|
||||||
|
|
||||||
|
lazy_static! {
|
||||||
|
/// Rows not yet on the true-instant date contract (`date_utc IS
|
||||||
|
/// NULL`), summed across libraries. Refreshed by the watcher every
|
||||||
|
/// tick; cutover (#120) waits on this reaching zero.
|
||||||
|
pub static ref DATE_BACKFILL_REMAINING: IntGauge = IntGauge::new(
|
||||||
|
"imageserver_date_backfill_remaining",
|
||||||
|
"Rows not yet converted to the true-instant date contract"
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Upper bound on rows the true-instant drain pages past in one tick,
|
||||||
|
/// as a multiple of the cap. Unconvertible rows (missing files) don't
|
||||||
|
/// spend the cap, but a library full of them must not turn one tick into
|
||||||
|
/// a full-table walk.
|
||||||
|
const UTC_BACKFILL_SCAN_FACTOR: i64 = 20;
|
||||||
|
|
||||||
|
/// What one tick of [`backfill_utc_dates`] did for one library.
|
||||||
|
#[derive(Debug, Default, Clone, PartialEq, Eq)]
|
||||||
|
pub struct UtcBackfillOutcome {
|
||||||
|
/// Rows whose new-contract triple was written this tick.
|
||||||
|
pub converted: usize,
|
||||||
|
/// Rows paged past because their file is missing (didn't spend cap).
|
||||||
|
pub skipped_missing: usize,
|
||||||
|
/// Rows the waterfall couldn't convert faithfully (left for later).
|
||||||
|
pub unresolved: usize,
|
||||||
|
/// Converted rows whose display-zone calendar day moved.
|
||||||
|
pub day_changed: usize,
|
||||||
|
/// Pre-generated day reels deleted because a day they cover moved.
|
||||||
|
pub reels_invalidated: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// True-instant backfill drain (ADR-0001, ticket #118): convert rows
|
||||||
|
/// still on a legacy mixed convention (`date_utc IS NULL AND date_taken
|
||||||
|
/// IS NOT NULL`) to true instants by re-running the canonical waterfall —
|
||||||
|
/// one exiftool subprocess per batch, same call shape as the #117 audit
|
||||||
|
/// preview (`resolve_dates_batch` with empty priors). Writes only the
|
||||||
|
/// new-contract triple; the legacy `date_taken` / `date_taken_source`
|
||||||
|
/// pair is frozen. Capped per tick by `DATE_BACKFILL_MAX_PER_TICK`
|
||||||
|
/// (default 500), shared with [`backfill_missing_date_taken`].
|
||||||
|
///
|
||||||
|
/// Legacy `manual` rows (overrides set before #115) aren't re-resolved
|
||||||
|
/// from the file — that would discard the operator's pick. Their legacy
|
||||||
|
/// value is the wall clock the UI sent, so they convert the same way a
|
||||||
|
/// manual edit does today (`camera_tz::manual_date_resolution` in the
|
||||||
|
/// display zone).
|
||||||
|
///
|
||||||
|
/// Day-boundary invalidation: when a converted instant falls on a
|
||||||
|
/// different display-zone calendar day than the legacy value did (as the
|
||||||
|
/// legacy `/memories` SQL rendered it), the pre-generated `day` reels
|
||||||
|
/// covering either month-day are deleted so the nightly pre-gen
|
||||||
|
/// regenerates them. Daily conversation summaries are SMS-derived and
|
||||||
|
/// don't depend on photo dates, so they're not touched.
|
||||||
|
pub fn backfill_utc_dates(
|
||||||
|
context: &opentelemetry::Context,
|
||||||
|
library: &libraries::Library,
|
||||||
|
exif_dao: &Arc<Mutex<Box<dyn ExifDao>>>,
|
||||||
|
reel_dao: &Arc<Mutex<Box<dyn PrecomputedReelDao>>>,
|
||||||
|
display_tz: &str,
|
||||||
|
) -> UtcBackfillOutcome {
|
||||||
|
let cap: i64 = dotenv::var("DATE_BACKFILL_MAX_PER_TICK")
|
||||||
|
.ok()
|
||||||
|
.and_then(|s| s.parse().ok())
|
||||||
|
.filter(|n: &i64| *n > 0)
|
||||||
|
.unwrap_or(500);
|
||||||
|
backfill_utc_dates_capped(context, library, exif_dao, reel_dao, display_tz, cap)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn backfill_utc_dates_capped(
|
||||||
|
context: &opentelemetry::Context,
|
||||||
|
library: &libraries::Library,
|
||||||
|
exif_dao: &Arc<Mutex<Box<dyn ExifDao>>>,
|
||||||
|
reel_dao: &Arc<Mutex<Box<dyn PrecomputedReelDao>>>,
|
||||||
|
display_tz: &str,
|
||||||
|
cap: i64,
|
||||||
|
) -> UtcBackfillOutcome {
|
||||||
|
let mut outcome = UtcBackfillOutcome::default();
|
||||||
|
let base_path = std::path::Path::new(&library.root_path);
|
||||||
|
|
||||||
|
// The watcher only calls this for online libraries; re-check here so
|
||||||
|
// an unmounted root can never partially convert (manual rows need no
|
||||||
|
// file and would otherwise convert while the rest waited).
|
||||||
|
if !base_path.is_dir() {
|
||||||
|
return outcome;
|
||||||
|
}
|
||||||
|
|
||||||
|
let zone_name = camera_tz::display_zone_for(Some(display_tz));
|
||||||
|
let Ok(zone) = zone_name.parse::<chrono_tz::Tz>() else {
|
||||||
|
return outcome;
|
||||||
|
};
|
||||||
|
// Without exiftool, legacy EXIF/exiftool rows would fall through the
|
||||||
|
// empty-prior waterfall to fs_time and land a wrong instant. Leave
|
||||||
|
// them for a deploy that has it.
|
||||||
|
let exiftool_ok = date_resolver::exiftool_available();
|
||||||
|
|
||||||
|
// Page by id until `cap` convertible rows are collected. Rows that
|
||||||
|
// can't convert this tick are paged past, not counted against the
|
||||||
|
// cap, so a pile of missing files at low ids can't stall the drain.
|
||||||
|
let mut manual: Vec<UtcBackfillCandidate> = Vec::new();
|
||||||
|
let mut filed: Vec<(UtcBackfillCandidate, PathBuf)> = Vec::new();
|
||||||
|
let mut cursor = 0;
|
||||||
|
let mut scanned = 0i64;
|
||||||
|
'paging: loop {
|
||||||
|
let page = {
|
||||||
|
let mut dao = exif_dao.lock().expect("Unable to lock ExifDao");
|
||||||
|
dao.get_rows_needing_utc_backfill(context, library.id, cursor, cap)
|
||||||
|
.unwrap_or_default()
|
||||||
|
};
|
||||||
|
let page_len = page.len() as i64;
|
||||||
|
for row in page {
|
||||||
|
cursor = row.id;
|
||||||
|
scanned += 1;
|
||||||
|
match row.date_taken_source.as_deref() {
|
||||||
|
Some("manual") => manual.push(row),
|
||||||
|
Some("exif") | Some("exiftool") if !exiftool_ok => outcome.unresolved += 1,
|
||||||
|
_ => {
|
||||||
|
let abs = base_path.join(&row.rel_path);
|
||||||
|
if abs.exists() {
|
||||||
|
filed.push((row, abs));
|
||||||
|
} else {
|
||||||
|
// The missing-file scan retires these rows.
|
||||||
|
outcome.skipped_missing += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (manual.len() + filed.len()) as i64 >= cap {
|
||||||
|
break 'paging;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if page_len < cap || scanned >= cap * UTC_BACKFILL_SCAN_FACTOR {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let more_remain = (manual.len() + filed.len()) as i64 >= cap;
|
||||||
|
|
||||||
|
// (candidate, date_utc, camera_offset_minutes, tz_source)
|
||||||
|
let mut conversions: Vec<(UtcBackfillCandidate, i64, Option<i32>, &'static str)> =
|
||||||
|
Vec::with_capacity(manual.len() + filed.len());
|
||||||
|
for row in manual {
|
||||||
|
match camera_tz::manual_date_resolution(Some(&zone_name), row.date_taken) {
|
||||||
|
Some((utc, offset)) => {
|
||||||
|
conversions.push((row, utc, Some(offset), camera_tz::TzSource::Manual.as_str()))
|
||||||
|
}
|
||||||
|
None => outcome.unresolved += 1,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !filed.is_empty() {
|
||||||
|
// One exiftool subprocess for the whole batch — byte-identical
|
||||||
|
// call shape to the #117 audit preview.
|
||||||
|
let paths: Vec<PathBuf> = filed.iter().map(|(_, p)| p.clone()).collect();
|
||||||
|
let resolved = date_resolver::resolve_dates_batch(&paths, &HashMap::new());
|
||||||
|
for (row, abs) in filed {
|
||||||
|
let legacy_had_tag = matches!(
|
||||||
|
row.date_taken_source.as_deref(),
|
||||||
|
Some("exif") | Some("exiftool")
|
||||||
|
);
|
||||||
|
match resolved.get(&abs) {
|
||||||
|
Some(rd) if !(legacy_had_tag && rd.source == date_resolver::DateSource::FsTime) => {
|
||||||
|
match (rd.date_utc, rd.tz_source) {
|
||||||
|
(Some(utc), Some(tz)) => {
|
||||||
|
conversions.push((row, utc, rd.camera_offset_minutes, tz.as_str()))
|
||||||
|
}
|
||||||
|
_ => outcome.unresolved += 1,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// No result, or a row whose legacy date came from a tag
|
||||||
|
// now resolving to fs_time (exiftool failed on it): don't
|
||||||
|
// bake a filesystem time in over a real capture date.
|
||||||
|
_ => {
|
||||||
|
debug!(
|
||||||
|
"utc_backfill: no faithful resolution for lib {} {}",
|
||||||
|
library.id, row.rel_path
|
||||||
|
);
|
||||||
|
outcome.unresolved += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut affected_days: HashSet<(u32, u32)> = HashSet::new();
|
||||||
|
{
|
||||||
|
let mut dao = exif_dao.lock().expect("Unable to lock ExifDao");
|
||||||
|
for (row, utc, offset, tz_source) in &conversions {
|
||||||
|
match dao.backfill_date_utc(context, row.id, *utc, *offset, tz_source) {
|
||||||
|
Ok(true) => {
|
||||||
|
outcome.converted += 1;
|
||||||
|
let before = display_day(zone, row.date_taken);
|
||||||
|
let after = display_day(zone, *utc);
|
||||||
|
if let (Some(before), Some(after)) = (before, after)
|
||||||
|
&& before != after
|
||||||
|
{
|
||||||
|
outcome.day_changed += 1;
|
||||||
|
affected_days.insert((before.month(), before.day()));
|
||||||
|
affected_days.insert((after.month(), after.day()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A manual edit / ingest raced us and already wrote it.
|
||||||
|
Ok(false) => {}
|
||||||
|
Err(e) => warn!(
|
||||||
|
"utc_backfill: update failed for lib {} {}: {:?}",
|
||||||
|
library.id, row.rel_path, e
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !affected_days.is_empty() {
|
||||||
|
outcome.reels_invalidated =
|
||||||
|
invalidate_day_reels(context, library, reel_dao, zone, &affected_days);
|
||||||
|
}
|
||||||
|
|
||||||
|
if outcome.converted > 0 || outcome.unresolved > 0 || more_remain {
|
||||||
|
info!(
|
||||||
|
"utc_backfill: library '{}': {:?}, cap={}, more_remain={}",
|
||||||
|
library.name, outcome, cap, more_remain
|
||||||
|
);
|
||||||
|
}
|
||||||
|
outcome
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Calendar day of a unix instant in the display zone. The legacy value
|
||||||
|
/// goes through here too: the legacy `/memories` SQL rendered
|
||||||
|
/// `date_taken` as an instant in the client's zone, so that is the day
|
||||||
|
/// the row's derived day data was built from.
|
||||||
|
fn display_day(zone: chrono_tz::Tz, epoch: i64) -> Option<NaiveDate> {
|
||||||
|
zone.timestamp_opt(epoch, 0)
|
||||||
|
.single()
|
||||||
|
.map(|dt| dt.date_naive())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The month-day an "on this day" reel covers: the date it was generated
|
||||||
|
/// on, in the zone offset it was generated with. A pre-gen fallback path
|
||||||
|
/// has stored seconds rather than minutes; an out-of-range offset falls
|
||||||
|
/// back to the display zone.
|
||||||
|
fn reel_covered_day(reel: &PrecomputedReel, zone: chrono_tz::Tz) -> Option<NaiveDate> {
|
||||||
|
if reel.tz_offset_minutes.abs() <= 14 * 60 {
|
||||||
|
DateTime::from_timestamp(reel.generated_at + reel.tz_offset_minutes as i64 * 60, 0)
|
||||||
|
.map(|dt| dt.date_naive())
|
||||||
|
} else {
|
||||||
|
display_day(zone, reel.generated_at)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Delete the pre-generated `day` reels (this library's and the
|
||||||
|
/// all-libraries one) covering any of `affected_days`, plus their files.
|
||||||
|
/// With the ledger row gone the nightly pre-gen's freshness gate fails
|
||||||
|
/// and it renders a new one on its normal schedule. Returns rows deleted.
|
||||||
|
fn invalidate_day_reels(
|
||||||
|
context: &opentelemetry::Context,
|
||||||
|
library: &libraries::Library,
|
||||||
|
reel_dao: &Arc<Mutex<Box<dyn PrecomputedReelDao>>>,
|
||||||
|
zone: chrono_tz::Tz,
|
||||||
|
affected_days: &HashSet<(u32, u32)>,
|
||||||
|
) -> usize {
|
||||||
|
let keys = vec!["all".to_string(), library.id.to_string()];
|
||||||
|
let mut dao = reel_dao.lock().expect("Unable to lock PrecomputedReelDao");
|
||||||
|
let reels = match dao.list_for_span(context, "day", &keys) {
|
||||||
|
Ok(r) => r,
|
||||||
|
Err(e) => {
|
||||||
|
warn!("utc_backfill: listing day reels failed: {:?}", e);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let stale: Vec<&PrecomputedReel> = reels
|
||||||
|
.iter()
|
||||||
|
.filter(|r| {
|
||||||
|
reel_covered_day(r, zone).is_some_and(|d| affected_days.contains(&(d.month(), d.day())))
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
if stale.is_empty() {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
let ids: Vec<i32> = stale.iter().map(|r| r.id).collect();
|
||||||
|
match dao.delete_ids(context, &ids) {
|
||||||
|
Ok(n) => {
|
||||||
|
// Unlink too, so the on-demand cache can't serve the stale
|
||||||
|
// render under the same key.
|
||||||
|
for r in &stale {
|
||||||
|
let mp4 = std::path::Path::new(&r.output_path);
|
||||||
|
let _ = std::fs::remove_file(mp4);
|
||||||
|
let _ = std::fs::remove_file(mp4.with_extension("json"));
|
||||||
|
}
|
||||||
|
info!(
|
||||||
|
"utc_backfill: invalidated {} day reel(s) for library '{}' (re-dated days {:?})",
|
||||||
|
n, library.name, affected_days
|
||||||
|
);
|
||||||
|
n
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
warn!("utc_backfill: deleting stale day reels failed: {:?}", e);
|
||||||
|
0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Refresh the progress surface for one online library: store its
|
||||||
|
/// remaining count in `remaining` and set [`DATE_BACKFILL_REMAINING`] to
|
||||||
|
/// the sum. Offline libraries keep their last-known count (the watcher
|
||||||
|
/// doesn't call this for them), so a flapping share doesn't make the
|
||||||
|
/// gauge dip.
|
||||||
|
pub fn refresh_utc_backfill_progress(
|
||||||
|
context: &opentelemetry::Context,
|
||||||
|
library: &libraries::Library,
|
||||||
|
exif_dao: &Arc<Mutex<Box<dyn ExifDao>>>,
|
||||||
|
remaining: &mut HashMap<i32, i64>,
|
||||||
|
) {
|
||||||
|
let count = {
|
||||||
|
let mut dao = exif_dao.lock().expect("Unable to lock ExifDao");
|
||||||
|
dao.count_rows_needing_utc_backfill(context, library.id)
|
||||||
|
};
|
||||||
|
if let Ok(n) = count {
|
||||||
|
remaining.insert(library.id, n);
|
||||||
|
DATE_BACKFILL_REMAINING.set(remaining.values().sum());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Per-tick face-detection drain. Pulls a capped batch of hashed-but-
|
/// Per-tick face-detection drain. Pulls a capped batch of hashed-but-
|
||||||
/// unscanned image_exif rows directly via the FaceDao anti-join and
|
/// unscanned image_exif rows directly via the FaceDao anti-join and
|
||||||
/// hands them to the existing detection pass. Runs on every tick (not
|
/// hands them to the existing detection pass. Runs on every tick (not
|
||||||
@@ -799,4 +1120,540 @@ mod tests {
|
|||||||
assert_eq!(candidates[0].rel_path, "fresh.jpg");
|
assert_eq!(candidates[0].rel_path, "fresh.jpg");
|
||||||
assert_eq!(candidates[0].content_hash, "hash-fresh");
|
assert_eq!(candidates[0].content_hash, "hash-fresh");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// --- True-instant backfill drain (ADR-0001, ticket #118) -------------
|
||||||
|
|
||||||
|
use crate::database::SqlitePrecomputedReelDao;
|
||||||
|
use crate::database::models::{ImageExif, InsertablePrecomputedReel};
|
||||||
|
use chrono::NaiveDateTime;
|
||||||
|
|
||||||
|
const NY: &str = "America/New_York";
|
||||||
|
|
||||||
|
fn utc(y: i32, mo: u32, d: u32, h: u32, mi: u32, s: u32) -> i64 {
|
||||||
|
NaiveDate::from_ymd_opt(y, mo, d)
|
||||||
|
.unwrap()
|
||||||
|
.and_hms_opt(h, mi, s)
|
||||||
|
.unwrap()
|
||||||
|
.and_utc()
|
||||||
|
.timestamp()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reel_dao_for(
|
||||||
|
conn: &Arc<Mutex<diesel::SqliteConnection>>,
|
||||||
|
) -> Arc<Mutex<Box<dyn PrecomputedReelDao>>> {
|
||||||
|
Arc::new(Mutex::new(Box::new(
|
||||||
|
SqlitePrecomputedReelDao::from_connection(Arc::clone(conn)),
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Seed a row on the legacy convention: legacy pair set, new-contract
|
||||||
|
/// triple NULL. Returns the row id.
|
||||||
|
fn insert_legacy(
|
||||||
|
exif_dao: &Arc<Mutex<Box<dyn ExifDao>>>,
|
||||||
|
lib_id: i32,
|
||||||
|
rel: &str,
|
||||||
|
date_taken: Option<i64>,
|
||||||
|
source: Option<&str>,
|
||||||
|
) -> i32 {
|
||||||
|
let mut dao = exif_dao.lock().unwrap();
|
||||||
|
dao.store_exif(
|
||||||
|
&ctx(),
|
||||||
|
InsertImageExif {
|
||||||
|
library_id: lib_id,
|
||||||
|
file_path: rel.to_string(),
|
||||||
|
camera_make: None,
|
||||||
|
camera_model: None,
|
||||||
|
lens_model: None,
|
||||||
|
width: None,
|
||||||
|
height: None,
|
||||||
|
orientation: None,
|
||||||
|
gps_latitude: None,
|
||||||
|
gps_longitude: None,
|
||||||
|
gps_altitude: None,
|
||||||
|
focal_length: None,
|
||||||
|
aperture: None,
|
||||||
|
shutter_speed: None,
|
||||||
|
iso: None,
|
||||||
|
date_taken,
|
||||||
|
created_time: 0,
|
||||||
|
last_modified: 0,
|
||||||
|
content_hash: None,
|
||||||
|
size_bytes: None,
|
||||||
|
phash_64: None,
|
||||||
|
dhash_64: None,
|
||||||
|
date_taken_source: source.map(str::to_string),
|
||||||
|
date_utc: None,
|
||||||
|
camera_offset_minutes: None,
|
||||||
|
date_taken_tz_source: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("insert")
|
||||||
|
.id
|
||||||
|
}
|
||||||
|
|
||||||
|
fn row(conn: &Arc<Mutex<diesel::SqliteConnection>>, id: i32) -> ImageExif {
|
||||||
|
let mut c = conn.lock().unwrap();
|
||||||
|
schema::image_exif::table
|
||||||
|
.find(id)
|
||||||
|
.first::<ImageExif>(&mut *c)
|
||||||
|
.expect("row")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remaining(exif_dao: &Arc<Mutex<Box<dyn ExifDao>>>, lib_id: i32) -> i64 {
|
||||||
|
exif_dao
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.count_rows_needing_utc_backfill(&ctx(), lib_id)
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn tools_available() -> bool {
|
||||||
|
let ok = |bin: &str, arg: &str| {
|
||||||
|
std::process::Command::new(bin)
|
||||||
|
.arg(arg)
|
||||||
|
.stdout(std::process::Stdio::null())
|
||||||
|
.stderr(std::process::Stdio::null())
|
||||||
|
.status()
|
||||||
|
.map(|s| s.success())
|
||||||
|
.unwrap_or(false)
|
||||||
|
};
|
||||||
|
ok("ffmpeg", "-version") && ok("exiftool", "-ver")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run_ok(cmd: &mut std::process::Command) {
|
||||||
|
let out = cmd.output().expect("spawn fixture tool");
|
||||||
|
assert!(
|
||||||
|
out.status.success(),
|
||||||
|
"fixture command failed: {}",
|
||||||
|
String::from_utf8_lossy(&out.stderr)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Tiny H.264 MP4 whose container `creation_time` exiftool reads as
|
||||||
|
/// QuickTime / Track* groups (→ `utc_direct`).
|
||||||
|
fn write_container_video(root: &std::path::Path, rel: &str, creation_time: &str) {
|
||||||
|
run_ok(
|
||||||
|
std::process::Command::new("ffmpeg")
|
||||||
|
.args([
|
||||||
|
"-y",
|
||||||
|
"-loglevel",
|
||||||
|
"error",
|
||||||
|
"-f",
|
||||||
|
"lavfi",
|
||||||
|
"-i",
|
||||||
|
"color=c=black:s=32x32:d=0.4",
|
||||||
|
"-r",
|
||||||
|
"10",
|
||||||
|
"-c:v",
|
||||||
|
"libx264",
|
||||||
|
"-pix_fmt",
|
||||||
|
"yuv420p",
|
||||||
|
"-metadata",
|
||||||
|
])
|
||||||
|
.arg(format!("creation_time={creation_time}"))
|
||||||
|
.arg(root.join(rel)),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Tiny JPEG carrying `DateTimeOriginal` + `OffsetTimeOriginal`.
|
||||||
|
fn write_exif_jpeg(root: &std::path::Path, rel: &str, dto: &str, offset: &str) {
|
||||||
|
let abs = root.join(rel);
|
||||||
|
run_ok(
|
||||||
|
std::process::Command::new("ffmpeg")
|
||||||
|
.args([
|
||||||
|
"-y",
|
||||||
|
"-loglevel",
|
||||||
|
"error",
|
||||||
|
"-f",
|
||||||
|
"lavfi",
|
||||||
|
"-i",
|
||||||
|
"color=c=gray:s=16x16",
|
||||||
|
"-frames:v",
|
||||||
|
"1",
|
||||||
|
])
|
||||||
|
.arg(&abs),
|
||||||
|
);
|
||||||
|
run_ok(
|
||||||
|
std::process::Command::new("exiftool")
|
||||||
|
.arg("-q")
|
||||||
|
.arg("-overwrite_original")
|
||||||
|
.arg(format!("-DateTimeOriginal={dto}"))
|
||||||
|
.arg(format!("-OffsetTimeOriginal={offset}"))
|
||||||
|
.arg(&abs),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn utc_backfill_converts_one_row_per_source_and_is_idempotent() {
|
||||||
|
if !tools_available() {
|
||||||
|
eprintln!("skipping: ffmpeg/exiftool not on PATH");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let (tmp, library, conn, exif_dao, _face_dao) = setup();
|
||||||
|
let reel_dao = reel_dao_for(&conn);
|
||||||
|
|
||||||
|
// exif: camera wall 22:30 at +02:00 → 20:30Z. Legacy = wall-as-UTC.
|
||||||
|
write_exif_jpeg(tmp.path(), "IMG_0001.jpg", "2024:07:04 22:30:00", "+02:00");
|
||||||
|
let exif_legacy = utc(2024, 7, 4, 22, 30, 0);
|
||||||
|
let exif_id = insert_legacy(
|
||||||
|
&exif_dao,
|
||||||
|
1,
|
||||||
|
"IMG_0001.jpg",
|
||||||
|
Some(exif_legacy),
|
||||||
|
Some("exif"),
|
||||||
|
);
|
||||||
|
|
||||||
|
// exiftool/container: creation_time is UTC. Legacy = the old
|
||||||
|
// server-local interpretation (EDT, +4h) — the "videos off by 4–5
|
||||||
|
// hours" row this drain exists to fix.
|
||||||
|
write_container_video(tmp.path(), "clip.mp4", "2017-06-11T15:58:23Z");
|
||||||
|
let video_legacy = utc(2017, 6, 11, 19, 58, 23);
|
||||||
|
let video_id = insert_legacy(
|
||||||
|
&exif_dao,
|
||||||
|
1,
|
||||||
|
"clip.mp4",
|
||||||
|
Some(video_legacy),
|
||||||
|
Some("exiftool"),
|
||||||
|
);
|
||||||
|
|
||||||
|
// filename (wall pattern): legacy wall-as-UTC; no evidence → server zone.
|
||||||
|
let fname = "Screenshot_2014-06-01-20-44-50.png";
|
||||||
|
write_image(tmp.path(), fname, b"not-a-png");
|
||||||
|
let fname_wall: NaiveDateTime = NaiveDate::from_ymd_opt(2014, 6, 1)
|
||||||
|
.unwrap()
|
||||||
|
.and_hms_opt(20, 44, 50)
|
||||||
|
.unwrap();
|
||||||
|
let fname_legacy = fname_wall.and_utc().timestamp();
|
||||||
|
let fname_id = insert_legacy(&exif_dao, 1, fname, Some(fname_legacy), Some("filename"));
|
||||||
|
|
||||||
|
// fs_time: no date anywhere in the file; legacy is already a true instant.
|
||||||
|
write_image(tmp.path(), "scan.jpg", b"no-exif-here");
|
||||||
|
let meta = fs::metadata(tmp.path().join("scan.jpg")).unwrap();
|
||||||
|
let fs_instant: chrono::DateTime<chrono::Utc> =
|
||||||
|
crate::utils::earliest_fs_time(&meta).unwrap().into();
|
||||||
|
let fs_instant = fs_instant.timestamp();
|
||||||
|
let fs_id = insert_legacy(&exif_dao, 1, "scan.jpg", Some(fs_instant), Some("fs_time"));
|
||||||
|
|
||||||
|
// manual (pre-#115 override): the operator's wall in the display
|
||||||
|
// zone — never re-resolved from the file.
|
||||||
|
write_image(tmp.path(), "edited.jpg", b"no-exif-here-either");
|
||||||
|
let manual_legacy = utc(2020, 1, 1, 12, 0, 0);
|
||||||
|
let manual_id = insert_legacy(
|
||||||
|
&exif_dao,
|
||||||
|
1,
|
||||||
|
"edited.jpg",
|
||||||
|
Some(manual_legacy),
|
||||||
|
Some("manual"),
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(remaining(&exif_dao, 1), 5);
|
||||||
|
let out = backfill_utc_dates_capped(&ctx(), &library, &exif_dao, &reel_dao, NY, 500);
|
||||||
|
assert_eq!(out.converted, 5, "{out:?}");
|
||||||
|
assert_eq!(out.unresolved, 0, "{out:?}");
|
||||||
|
|
||||||
|
let legacy = |r: &ImageExif| (r.date_taken, r.date_taken_source.clone());
|
||||||
|
|
||||||
|
let r = row(&conn, exif_id);
|
||||||
|
assert_eq!(r.date_utc, Some(utc(2024, 7, 4, 20, 30, 0)));
|
||||||
|
assert_eq!(r.camera_offset_minutes, Some(120));
|
||||||
|
assert_eq!(r.date_taken_tz_source.as_deref(), Some("offset_tag"));
|
||||||
|
assert_eq!(legacy(&r), (Some(exif_legacy), Some("exif".into())));
|
||||||
|
|
||||||
|
let r = row(&conn, video_id);
|
||||||
|
assert_eq!(r.date_utc, Some(utc(2017, 6, 11, 15, 58, 23)));
|
||||||
|
assert_eq!(r.camera_offset_minutes, None);
|
||||||
|
assert_eq!(r.date_taken_tz_source.as_deref(), Some("utc_direct"));
|
||||||
|
assert_eq!(legacy(&r), (Some(video_legacy), Some("exiftool".into())));
|
||||||
|
|
||||||
|
let r = row(&conn, fname_id);
|
||||||
|
assert_eq!(r.date_taken_tz_source.as_deref(), Some("assumed_server"));
|
||||||
|
if let Some(zone) = crate::camera_tz::server_zone() {
|
||||||
|
let off = crate::camera_tz::offset_minutes_for_zone(zone, fname_wall);
|
||||||
|
assert_eq!(r.camera_offset_minutes, Some(off));
|
||||||
|
assert_eq!(
|
||||||
|
r.date_utc,
|
||||||
|
Some(crate::camera_tz::wall_to_instant(fname_wall, off))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert_eq!(legacy(&r), (Some(fname_legacy), Some("filename".into())));
|
||||||
|
|
||||||
|
let r = row(&conn, fs_id);
|
||||||
|
assert_eq!(r.date_utc, Some(fs_instant));
|
||||||
|
assert_eq!(r.camera_offset_minutes, None);
|
||||||
|
assert_eq!(r.date_taken_tz_source.as_deref(), Some("utc_direct"));
|
||||||
|
assert_eq!(legacy(&r), (Some(fs_instant), Some("fs_time".into())));
|
||||||
|
|
||||||
|
let r = row(&conn, manual_id);
|
||||||
|
// 2020-01-01 12:00 EST (-300) → 17:00Z.
|
||||||
|
assert_eq!(r.date_utc, Some(utc(2020, 1, 1, 17, 0, 0)));
|
||||||
|
assert_eq!(r.camera_offset_minutes, Some(-300));
|
||||||
|
assert_eq!(r.date_taken_tz_source.as_deref(), Some("manual"));
|
||||||
|
assert_eq!(legacy(&r), (Some(manual_legacy), Some("manual".into())));
|
||||||
|
|
||||||
|
// Second pass: nothing eligible, nothing changes, progress is 0.
|
||||||
|
let ids = [exif_id, video_id, fname_id, fs_id, manual_id];
|
||||||
|
let snapshot = |conn: &Arc<Mutex<diesel::SqliteConnection>>| {
|
||||||
|
ids.iter()
|
||||||
|
.map(|id| {
|
||||||
|
let r = row(conn, *id);
|
||||||
|
(
|
||||||
|
r.date_taken,
|
||||||
|
r.date_taken_source,
|
||||||
|
r.date_utc,
|
||||||
|
r.camera_offset_minutes,
|
||||||
|
r.date_taken_tz_source,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
};
|
||||||
|
let before = snapshot(&conn);
|
||||||
|
let again = backfill_utc_dates_capped(&ctx(), &library, &exif_dao, &reel_dao, NY, 500);
|
||||||
|
assert_eq!(again, UtcBackfillOutcome::default());
|
||||||
|
assert_eq!(snapshot(&conn), before);
|
||||||
|
let mut progress = HashMap::new();
|
||||||
|
refresh_utc_backfill_progress(&ctx(), &library, &exif_dao, &mut progress);
|
||||||
|
assert_eq!(progress.get(&1), Some(&0));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn utc_backfill_leaves_null_date_rows_and_other_libraries_alone() {
|
||||||
|
let (tmp, library, conn, exif_dao, _face_dao) = setup();
|
||||||
|
let reel_dao = reel_dao_for(&conn);
|
||||||
|
write_image(tmp.path(), "1500000000.jpg", b"x");
|
||||||
|
write_image(tmp.path(), "undated.jpg", b"y");
|
||||||
|
let ts = 1_500_000_000;
|
||||||
|
let converted = insert_legacy(&exif_dao, 1, "1500000000.jpg", Some(ts), Some("filename"));
|
||||||
|
// Both NULL → the legacy drain's job (its writer fills the triple).
|
||||||
|
let undated = insert_legacy(&exif_dao, 1, "undated.jpg", None, None);
|
||||||
|
let other = insert_legacy(&exif_dao, 2, "1500000000.jpg", Some(ts), Some("filename"));
|
||||||
|
|
||||||
|
let out = backfill_utc_dates_capped(&ctx(), &library, &exif_dao, &reel_dao, NY, 500);
|
||||||
|
assert_eq!(out.converted, 1);
|
||||||
|
assert_eq!(row(&conn, converted).date_utc, Some(ts));
|
||||||
|
assert_eq!(row(&conn, undated).date_utc, None);
|
||||||
|
assert_eq!(row(&conn, other).date_utc, None);
|
||||||
|
// Progress counts every row not yet on the new contract.
|
||||||
|
assert_eq!(remaining(&exif_dao, 1), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn utc_backfill_respects_cap_and_missing_files_dont_spend_it() {
|
||||||
|
let (tmp, library, conn, exif_dao, _face_dao) = setup();
|
||||||
|
let reel_dao = reel_dao_for(&conn);
|
||||||
|
// Missing files at the lowest ids, ahead of the present ones.
|
||||||
|
// Timestamp filenames keep the fixture hermetic (never hit exiftool).
|
||||||
|
let mut missing = Vec::new();
|
||||||
|
for i in 0..3 {
|
||||||
|
let ts = 1_400_000_000 + i;
|
||||||
|
let rel = format!("{ts}.jpg");
|
||||||
|
missing.push(insert_legacy(
|
||||||
|
&exif_dao,
|
||||||
|
1,
|
||||||
|
&rel,
|
||||||
|
Some(ts),
|
||||||
|
Some("filename"),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
for i in 0..5 {
|
||||||
|
let ts = 1_500_000_000 + i;
|
||||||
|
let rel = format!("{ts}.jpg");
|
||||||
|
write_image(tmp.path(), &rel, b"z");
|
||||||
|
insert_legacy(&exif_dao, 1, &rel, Some(ts), Some("filename"));
|
||||||
|
}
|
||||||
|
assert_eq!(remaining(&exif_dao, 1), 8);
|
||||||
|
|
||||||
|
let mut progress = HashMap::new();
|
||||||
|
let mut seen = Vec::new();
|
||||||
|
for _ in 0..4 {
|
||||||
|
let out = backfill_utc_dates_capped(&ctx(), &library, &exif_dao, &reel_dao, NY, 2);
|
||||||
|
refresh_utc_backfill_progress(&ctx(), &library, &exif_dao, &mut progress);
|
||||||
|
seen.push((out.converted, progress[&1]));
|
||||||
|
}
|
||||||
|
// Cap holds (2 per tick) and the 3 missing rows never spend it.
|
||||||
|
assert_eq!(seen, vec![(2, 6), (2, 4), (1, 3), (0, 3)]);
|
||||||
|
|
||||||
|
// The missing-file scan retires the rest → progress reaches 0.
|
||||||
|
{
|
||||||
|
let mut c = conn.lock().unwrap();
|
||||||
|
diesel::delete(
|
||||||
|
schema::image_exif::table.filter(schema::image_exif::id.eq_any(&missing)),
|
||||||
|
)
|
||||||
|
.execute(&mut *c)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
refresh_utc_backfill_progress(&ctx(), &library, &exif_dao, &mut progress);
|
||||||
|
assert_eq!(progress[&1], 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn utc_backfill_skips_unavailable_library_then_resumes() {
|
||||||
|
let (tmp, library, conn, exif_dao, _face_dao) = setup();
|
||||||
|
let reel_dao = reel_dao_for(&conn);
|
||||||
|
write_image(tmp.path(), "1500000000.jpg", b"z");
|
||||||
|
let filed = insert_legacy(
|
||||||
|
&exif_dao,
|
||||||
|
1,
|
||||||
|
"1500000000.jpg",
|
||||||
|
Some(1_500_000_000),
|
||||||
|
Some("filename"),
|
||||||
|
);
|
||||||
|
// A manual row needs no file — it must still wait for the mount.
|
||||||
|
let manual = insert_legacy(
|
||||||
|
&exif_dao,
|
||||||
|
1,
|
||||||
|
"edited.jpg",
|
||||||
|
Some(utc(2020, 1, 1, 12, 0, 0)),
|
||||||
|
Some("manual"),
|
||||||
|
);
|
||||||
|
|
||||||
|
let unmounted = Library {
|
||||||
|
root_path: tmp.path().join("unmounted").to_string_lossy().to_string(),
|
||||||
|
..library.clone()
|
||||||
|
};
|
||||||
|
// The watcher gate sees it stale…
|
||||||
|
let health_map = libraries::new_health_map(std::slice::from_ref(&unmounted));
|
||||||
|
assert!(!libraries::refresh_health(&health_map, &unmounted, true).is_online());
|
||||||
|
// …and the drain itself converts nothing even if called.
|
||||||
|
let out = backfill_utc_dates_capped(&ctx(), &unmounted, &exif_dao, &reel_dao, NY, 500);
|
||||||
|
assert_eq!(out, UtcBackfillOutcome::default());
|
||||||
|
assert_eq!(row(&conn, filed).date_utc, None);
|
||||||
|
assert_eq!(row(&conn, manual).date_utc, None);
|
||||||
|
|
||||||
|
// Share returns → picked up next tick.
|
||||||
|
assert!(libraries::refresh_health(&health_map, &library, true).is_online());
|
||||||
|
let out = backfill_utc_dates_capped(&ctx(), &library, &exif_dao, &reel_dao, NY, 500);
|
||||||
|
assert_eq!(out.converted, 2);
|
||||||
|
assert_eq!(remaining(&exif_dao, 1), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Seed a pre-generated reel covering `month`-`day` (generated 03:00
|
||||||
|
/// EDT that day, as the nightly pre-gen does). Returns (id, mp4 path).
|
||||||
|
fn seed_reel(
|
||||||
|
reel_dao: &Arc<Mutex<Box<dyn PrecomputedReelDao>>>,
|
||||||
|
dir: &std::path::Path,
|
||||||
|
span: &str,
|
||||||
|
library_key: &str,
|
||||||
|
month: u32,
|
||||||
|
day: u32,
|
||||||
|
) -> (i32, PathBuf) {
|
||||||
|
let mp4 = dir.join(format!("reel-{span}-{library_key}-{month}-{day}.mp4"));
|
||||||
|
fs::write(&mp4, b"mp4").unwrap();
|
||||||
|
let id = reel_dao
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.record_reel(
|
||||||
|
&ctx(),
|
||||||
|
&InsertablePrecomputedReel {
|
||||||
|
span: span.to_string(),
|
||||||
|
library_key: library_key.to_string(),
|
||||||
|
cache_key: format!("{span}-{library_key}-{month}-{day}"),
|
||||||
|
output_path: mp4.to_string_lossy().to_string(),
|
||||||
|
title: "t".to_string(),
|
||||||
|
media_count: 1,
|
||||||
|
render_version: 1,
|
||||||
|
tz_offset_minutes: -240,
|
||||||
|
voice: None,
|
||||||
|
generated_at: utc(2025, month, day, 7, 0, 0),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
(id, mp4)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reel_ids(conn: &Arc<Mutex<diesel::SqliteConnection>>) -> Vec<i32> {
|
||||||
|
let mut c = conn.lock().unwrap();
|
||||||
|
schema::precomputed_reels::table
|
||||||
|
.select(schema::precomputed_reels::id)
|
||||||
|
.order(schema::precomputed_reels::id.asc())
|
||||||
|
.load::<i32>(&mut *c)
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn utc_backfill_invalidates_exactly_the_moved_days_reels() {
|
||||||
|
let (tmp, library, conn, exif_dao, _face_dao) = setup();
|
||||||
|
let reel_dao = reel_dao_for(&conn);
|
||||||
|
let reels_dir = TempDir::new().unwrap();
|
||||||
|
|
||||||
|
// Manual wall 2017-06-12 02:00 in NY. The legacy value, read as an
|
||||||
|
// instant in NY by the old /memories SQL, showed 06-11 22:00; the
|
||||||
|
// true instant is 06-12 06:00Z = 06-12 02:00 EDT → the day moves
|
||||||
|
// 06-11 → 06-12.
|
||||||
|
let moved = insert_legacy(
|
||||||
|
&exif_dao,
|
||||||
|
1,
|
||||||
|
"moved.jpg",
|
||||||
|
Some(utc(2017, 6, 12, 2, 0, 0)),
|
||||||
|
Some("manual"),
|
||||||
|
);
|
||||||
|
// Unchanged: a timestamp-filename row's legacy value is already the instant.
|
||||||
|
write_image(tmp.path(), "1400000000.jpg", b"z");
|
||||||
|
insert_legacy(
|
||||||
|
&exif_dao,
|
||||||
|
1,
|
||||||
|
"1400000000.jpg",
|
||||||
|
Some(1_400_000_000),
|
||||||
|
Some("filename"),
|
||||||
|
);
|
||||||
|
|
||||||
|
let (_, mp4_11) = seed_reel(&reel_dao, reels_dir.path(), "day", "1", 6, 11);
|
||||||
|
let (_, mp4_12) = seed_reel(&reel_dao, reels_dir.path(), "day", "all", 6, 12);
|
||||||
|
let (other_lib, _) = seed_reel(&reel_dao, reels_dir.path(), "day", "2", 6, 12);
|
||||||
|
let (other_day, mp4_0301) = seed_reel(&reel_dao, reels_dir.path(), "day", "1", 3, 1);
|
||||||
|
let (week, _) = seed_reel(&reel_dao, reels_dir.path(), "week", "1", 6, 12);
|
||||||
|
|
||||||
|
let out = backfill_utc_dates_capped(&ctx(), &library, &exif_dao, &reel_dao, NY, 500);
|
||||||
|
assert_eq!(out.converted, 2);
|
||||||
|
assert_eq!(out.day_changed, 1, "{out:?}");
|
||||||
|
assert_eq!(out.reels_invalidated, 2, "{out:?}");
|
||||||
|
assert_eq!(row(&conn, moved).date_utc, Some(utc(2017, 6, 12, 6, 0, 0)));
|
||||||
|
|
||||||
|
// Exactly the moved days' day reels for this library (+ "all") are gone.
|
||||||
|
assert_eq!(reel_ids(&conn), vec![other_lib, other_day, week]);
|
||||||
|
assert!(
|
||||||
|
!mp4_11.exists() && !mp4_12.exists(),
|
||||||
|
"stale renders unlinked"
|
||||||
|
);
|
||||||
|
assert!(mp4_0301.exists());
|
||||||
|
|
||||||
|
// With the ledger row gone the nightly pre-gen's freshness gate
|
||||||
|
// (`latest_for` must match the new cache key) can't skip → it
|
||||||
|
// regenerates on its normal schedule; no new scheduler involved.
|
||||||
|
let latest = reel_dao
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.latest_for(&ctx(), "day", "all")
|
||||||
|
.unwrap();
|
||||||
|
assert!(latest.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn utc_backfill_utc_midnight_straddle_marks_nothing() {
|
||||||
|
let (_tmp, library, conn, exif_dao, _face_dao) = setup();
|
||||||
|
let reel_dao = reel_dao_for(&conn);
|
||||||
|
let reels_dir = TempDir::new().unwrap();
|
||||||
|
|
||||||
|
// Manual wall 2017-06-11 23:30 in NY. Legacy instant 06-11 23:30Z
|
||||||
|
// (NY 19:30); true instant 06-12 03:30Z (NY 23:30). The UTC date
|
||||||
|
// flips 06-11 → 06-12, but the display-zone day stays 06-11.
|
||||||
|
let straddle = insert_legacy(
|
||||||
|
&exif_dao,
|
||||||
|
1,
|
||||||
|
"late.jpg",
|
||||||
|
Some(utc(2017, 6, 11, 23, 30, 0)),
|
||||||
|
Some("manual"),
|
||||||
|
);
|
||||||
|
seed_reel(&reel_dao, reels_dir.path(), "day", "1", 6, 11);
|
||||||
|
seed_reel(&reel_dao, reels_dir.path(), "day", "all", 6, 12);
|
||||||
|
|
||||||
|
let out = backfill_utc_dates_capped(&ctx(), &library, &exif_dao, &reel_dao, NY, 500);
|
||||||
|
assert_eq!(out.converted, 1);
|
||||||
|
assert_eq!(out.day_changed, 0);
|
||||||
|
assert_eq!(out.reels_invalidated, 0);
|
||||||
|
assert_eq!(
|
||||||
|
row(&conn, straddle).date_utc,
|
||||||
|
Some(utc(2017, 6, 12, 3, 30, 0))
|
||||||
|
);
|
||||||
|
assert_eq!(reel_ids(&conn).len(), 2);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -43,6 +43,17 @@ pub struct DuplicateRow {
|
|||||||
pub duplicate_decided_at: Option<i64>,
|
pub duplicate_decided_at: Option<i64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// One row the true-instant backfill drain (ADR-0001, ticket #118) still
|
||||||
|
/// has to convert: on the legacy convention (`date_utc IS NULL`) with a
|
||||||
|
/// legacy `date_taken` to compare the converted day against.
|
||||||
|
#[derive(Debug, Clone, Queryable)]
|
||||||
|
pub struct UtcBackfillCandidate {
|
||||||
|
pub id: i32,
|
||||||
|
pub rel_path: String,
|
||||||
|
pub date_taken: i64,
|
||||||
|
pub date_taken_source: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
pub mod calendar_dao;
|
pub mod calendar_dao;
|
||||||
pub mod daily_summary_dao;
|
pub mod daily_summary_dao;
|
||||||
pub mod insight_generation_job_dao;
|
pub mod insight_generation_job_dao;
|
||||||
@@ -513,6 +524,46 @@ pub trait ExifDao: Sync + Send {
|
|||||||
date_taken_tz_source: Option<&str>,
|
date_taken_tz_source: Option<&str>,
|
||||||
) -> Result<(), DbError>;
|
) -> Result<(), DbError>;
|
||||||
|
|
||||||
|
/// Rows the true-instant drain (ADR-0001, ticket #118) still has to
|
||||||
|
/// convert: `date_utc IS NULL AND date_taken IS NOT NULL` in
|
||||||
|
/// `library_id`, `id > after_id`, ordered by id ASC. Rows with both
|
||||||
|
/// NULL belong to `get_rows_needing_date_backfill` (whose writer fills
|
||||||
|
/// the new triple too). The `after_id` cursor lets the drain page past
|
||||||
|
/// rows it can't convert this tick (missing files) without spending
|
||||||
|
/// its per-tick cap on them. Backed by `idx_image_exif_utc_backfill`.
|
||||||
|
fn get_rows_needing_utc_backfill(
|
||||||
|
&mut self,
|
||||||
|
context: &opentelemetry::Context,
|
||||||
|
library_id: i32,
|
||||||
|
after_id: i32,
|
||||||
|
limit: i64,
|
||||||
|
) -> Result<Vec<UtcBackfillCandidate>, DbError>;
|
||||||
|
|
||||||
|
/// Persist a converted row's new-contract triple (`date_utc`,
|
||||||
|
/// `camera_offset_minutes`, `date_taken_tz_source`) by primary key.
|
||||||
|
/// Never touches the frozen legacy `date_taken` / `date_taken_source`
|
||||||
|
/// pair. Guarded on `date_utc IS NULL` so a manual edit or ingest
|
||||||
|
/// that raced the drain wins. Returns whether a row was written.
|
||||||
|
fn backfill_date_utc(
|
||||||
|
&mut self,
|
||||||
|
context: &opentelemetry::Context,
|
||||||
|
id: i32,
|
||||||
|
date_utc: i64,
|
||||||
|
camera_offset_minutes: Option<i32>,
|
||||||
|
date_taken_tz_source: &str,
|
||||||
|
) -> Result<bool, DbError>;
|
||||||
|
|
||||||
|
/// Progress surface for the true-instant drain: rows in `library_id`
|
||||||
|
/// not yet on the new contract (`date_utc IS NULL`). Counts every such
|
||||||
|
/// row — including NULL-date rows the legacy drain resolves and rows
|
||||||
|
/// whose file is missing (until the missing-file scan retires them) —
|
||||||
|
/// because cutover (#120) waits on exactly this set reaching zero.
|
||||||
|
fn count_rows_needing_utc_backfill(
|
||||||
|
&mut self,
|
||||||
|
context: &opentelemetry::Context,
|
||||||
|
library_id: i32,
|
||||||
|
) -> Result<i64, DbError>;
|
||||||
|
|
||||||
/// Find image_exif rows needing a CLIP embedding for semantic search:
|
/// Find image_exif rows needing a CLIP embedding for semantic search:
|
||||||
/// `clip_embedding IS NULL AND content_hash IS NOT NULL`, ordered by id
|
/// `clip_embedding IS NULL AND content_hash IS NOT NULL`, ordered by id
|
||||||
/// ASC, limited. Hash-less rows wait for `backfill_unhashed_backlog` to
|
/// ASC, limited. Hash-less rows wait for `backfill_unhashed_backlog` to
|
||||||
@@ -1501,6 +1552,95 @@ impl ExifDao for SqliteExifDao {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn get_rows_needing_utc_backfill(
|
||||||
|
&mut self,
|
||||||
|
context: &opentelemetry::Context,
|
||||||
|
library_id_val: i32,
|
||||||
|
after_id: i32,
|
||||||
|
limit: i64,
|
||||||
|
) -> Result<Vec<UtcBackfillCandidate>, DbError> {
|
||||||
|
trace_db_call(context, "query", "get_rows_needing_utc_backfill", |_span| {
|
||||||
|
use schema::image_exif::dsl::*;
|
||||||
|
|
||||||
|
let mut connection = self.connection.lock().expect("Unable to get ExifDao");
|
||||||
|
|
||||||
|
// Partial index `(library_id, id) WHERE date_utc IS NULL`
|
||||||
|
// serves the filter + order directly.
|
||||||
|
image_exif
|
||||||
|
.filter(library_id.eq(library_id_val))
|
||||||
|
.filter(date_utc.is_null())
|
||||||
|
.filter(date_taken.is_not_null())
|
||||||
|
.filter(id.gt(after_id))
|
||||||
|
.select((
|
||||||
|
id,
|
||||||
|
rel_path,
|
||||||
|
date_taken.assume_not_null(),
|
||||||
|
date_taken_source,
|
||||||
|
))
|
||||||
|
.order(id.asc())
|
||||||
|
.limit(limit)
|
||||||
|
.load::<UtcBackfillCandidate>(connection.deref_mut())
|
||||||
|
.map_err(|e| anyhow::anyhow!("Query error: {}", e))
|
||||||
|
})
|
||||||
|
.map_err(|e| DbError::log(DbErrorKind::QueryError, e))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn backfill_date_utc(
|
||||||
|
&mut self,
|
||||||
|
context: &opentelemetry::Context,
|
||||||
|
id_val: i32,
|
||||||
|
date_utc_val: i64,
|
||||||
|
camera_offset_minutes_val: Option<i32>,
|
||||||
|
date_taken_tz_source_val: &str,
|
||||||
|
) -> Result<bool, DbError> {
|
||||||
|
trace_db_call(context, "update", "backfill_date_utc", |_span| {
|
||||||
|
use schema::image_exif::dsl::*;
|
||||||
|
|
||||||
|
let mut connection = self.connection.lock().expect("Unable to get ExifDao");
|
||||||
|
|
||||||
|
// New-contract triple only — the legacy pair is frozen
|
||||||
|
// (ADR-0001).
|
||||||
|
diesel::update(image_exif.find(id_val).filter(date_utc.is_null()))
|
||||||
|
.set((
|
||||||
|
date_utc.eq(Some(date_utc_val)),
|
||||||
|
camera_offset_minutes.eq(camera_offset_minutes_val),
|
||||||
|
date_taken_tz_source.eq(Some(date_taken_tz_source_val)),
|
||||||
|
))
|
||||||
|
.execute(connection.deref_mut())
|
||||||
|
.map(|rows| rows > 0)
|
||||||
|
.map_err(|e| anyhow::anyhow!("diesel update failed (id={}): {}", id_val, e))
|
||||||
|
})
|
||||||
|
.map_err(|e| {
|
||||||
|
log::warn!("backfill_date_utc: {}", e);
|
||||||
|
DbError::new(DbErrorKind::UpdateError)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn count_rows_needing_utc_backfill(
|
||||||
|
&mut self,
|
||||||
|
context: &opentelemetry::Context,
|
||||||
|
library_id_val: i32,
|
||||||
|
) -> Result<i64, DbError> {
|
||||||
|
trace_db_call(
|
||||||
|
context,
|
||||||
|
"query",
|
||||||
|
"count_rows_needing_utc_backfill",
|
||||||
|
|_span| {
|
||||||
|
use schema::image_exif::dsl::*;
|
||||||
|
|
||||||
|
let mut connection = self.connection.lock().expect("Unable to get ExifDao");
|
||||||
|
|
||||||
|
image_exif
|
||||||
|
.filter(library_id.eq(library_id_val))
|
||||||
|
.filter(date_utc.is_null())
|
||||||
|
.count()
|
||||||
|
.get_result::<i64>(connection.deref_mut())
|
||||||
|
.map_err(|e| anyhow::anyhow!("Query error: {}", e))
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.map_err(|e| DbError::log(DbErrorKind::QueryError, e))
|
||||||
|
}
|
||||||
|
|
||||||
fn list_clip_unencoded_candidates(
|
fn list_clip_unencoded_candidates(
|
||||||
&mut self,
|
&mut self,
|
||||||
context: &opentelemetry::Context,
|
context: &opentelemetry::Context,
|
||||||
|
|||||||
@@ -58,6 +58,24 @@ pub trait PrecomputedReelDao: Sync + Send {
|
|||||||
/// to protect files a ledger row still points at.
|
/// to protect files a ledger row still points at.
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
fn all_cache_keys(&mut self, context: &opentelemetry::Context) -> Result<Vec<String>, DbError>;
|
fn all_cache_keys(&mut self, context: &opentelemetry::Context) -> Result<Vec<String>, DbError>;
|
||||||
|
|
||||||
|
/// Every reel row for `span` whose `library_key` is one of
|
||||||
|
/// `library_keys`. Used by the true-instant date drain (ticket #118)
|
||||||
|
/// to find the day reels a re-dated photo invalidates.
|
||||||
|
fn list_for_span(
|
||||||
|
&mut self,
|
||||||
|
context: &opentelemetry::Context,
|
||||||
|
span: &str,
|
||||||
|
library_keys: &[String],
|
||||||
|
) -> Result<Vec<PrecomputedReel>, DbError>;
|
||||||
|
|
||||||
|
/// Delete the given reel rows (stale-marking: the nightly pre-gen
|
||||||
|
/// finds no current row and regenerates). Returns rows deleted.
|
||||||
|
fn delete_ids(
|
||||||
|
&mut self,
|
||||||
|
context: &opentelemetry::Context,
|
||||||
|
ids: &[i32],
|
||||||
|
) -> Result<usize, DbError>;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct SqlitePrecomputedReelDao {
|
pub struct SqlitePrecomputedReelDao {
|
||||||
@@ -219,6 +237,49 @@ impl PrecomputedReelDao for SqlitePrecomputedReelDao {
|
|||||||
})
|
})
|
||||||
.map_err(|e| DbError::log(DbErrorKind::QueryError, e))
|
.map_err(|e| DbError::log(DbErrorKind::QueryError, e))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn list_for_span(
|
||||||
|
&mut self,
|
||||||
|
context: &opentelemetry::Context,
|
||||||
|
span: &str,
|
||||||
|
library_keys: &[String],
|
||||||
|
) -> Result<Vec<PrecomputedReel>, DbError> {
|
||||||
|
trace_db_call(context, "query", "list_for_span", |_span| {
|
||||||
|
use schema::precomputed_reels::dsl;
|
||||||
|
|
||||||
|
let mut connection = self
|
||||||
|
.connection
|
||||||
|
.lock()
|
||||||
|
.expect("Unable to lock PrecomputedReelDao");
|
||||||
|
|
||||||
|
dsl::precomputed_reels
|
||||||
|
.filter(dsl::span.eq(span))
|
||||||
|
.filter(dsl::library_key.eq_any(library_keys))
|
||||||
|
.load::<PrecomputedReel>(connection.deref_mut())
|
||||||
|
.map_err(|e| anyhow::anyhow!("Failed to list reels for span: {}", e))
|
||||||
|
})
|
||||||
|
.map_err(|e| DbError::log(DbErrorKind::QueryError, e))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn delete_ids(
|
||||||
|
&mut self,
|
||||||
|
context: &opentelemetry::Context,
|
||||||
|
ids: &[i32],
|
||||||
|
) -> Result<usize, DbError> {
|
||||||
|
trace_db_call(context, "delete", "delete_ids", |_span| {
|
||||||
|
use schema::precomputed_reels::dsl;
|
||||||
|
|
||||||
|
let mut connection = self
|
||||||
|
.connection
|
||||||
|
.lock()
|
||||||
|
.expect("Unable to lock PrecomputedReelDao");
|
||||||
|
|
||||||
|
diesel::delete(dsl::precomputed_reels.filter(dsl::id.eq_any(ids)))
|
||||||
|
.execute(connection.deref_mut())
|
||||||
|
.map_err(|e| anyhow::anyhow!("Failed to delete reels: {}", e))
|
||||||
|
})
|
||||||
|
.map_err(|e| DbError::log(DbErrorKind::UpdateError, e))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -351,7 +351,7 @@ const EXIFTOOL_EVIDENCE_TAGS: &[&str] = &[
|
|||||||
/// Cache the "exiftool exists on PATH" check across the process lifetime so
|
/// Cache the "exiftool exists on PATH" check across the process lifetime so
|
||||||
/// the per-tick backfill doesn't fork a doomed subprocess every iteration on
|
/// the per-tick backfill doesn't fork a doomed subprocess every iteration on
|
||||||
/// deploys without exiftool installed.
|
/// deploys without exiftool installed.
|
||||||
fn exiftool_available() -> bool {
|
pub fn exiftool_available() -> bool {
|
||||||
static AVAIL: OnceLock<bool> = OnceLock::new();
|
static AVAIL: OnceLock<bool> = OnceLock::new();
|
||||||
*AVAIL.get_or_init(|| {
|
*AVAIL.get_or_init(|| {
|
||||||
let ok = Command::new("exiftool")
|
let ok = Command::new("exiftool")
|
||||||
|
|||||||
@@ -1756,6 +1756,35 @@ mod tests {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn get_rows_needing_utc_backfill(
|
||||||
|
&mut self,
|
||||||
|
_context: &opentelemetry::Context,
|
||||||
|
_library_id: i32,
|
||||||
|
_after_id: i32,
|
||||||
|
_limit: i64,
|
||||||
|
) -> Result<Vec<crate::database::UtcBackfillCandidate>, DbError> {
|
||||||
|
Ok(Vec::new())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn backfill_date_utc(
|
||||||
|
&mut self,
|
||||||
|
_context: &opentelemetry::Context,
|
||||||
|
_id: i32,
|
||||||
|
_date_utc: i64,
|
||||||
|
_camera_offset_minutes: Option<i32>,
|
||||||
|
_date_taken_tz_source: &str,
|
||||||
|
) -> Result<bool, DbError> {
|
||||||
|
Ok(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn count_rows_needing_utc_backfill(
|
||||||
|
&mut self,
|
||||||
|
_context: &opentelemetry::Context,
|
||||||
|
_library_id: i32,
|
||||||
|
) -> Result<i64, DbError> {
|
||||||
|
Ok(0)
|
||||||
|
}
|
||||||
|
|
||||||
fn set_manual_date_taken(
|
fn set_manual_date_taken(
|
||||||
&mut self,
|
&mut self,
|
||||||
_context: &opentelemetry::Context,
|
_context: &opentelemetry::Context,
|
||||||
|
|||||||
@@ -460,6 +460,10 @@ pub struct LibraryStatus {
|
|||||||
#[serde(flatten)]
|
#[serde(flatten)]
|
||||||
pub library: Library,
|
pub library: Library,
|
||||||
pub health: LibraryHealth,
|
pub health: LibraryHealth,
|
||||||
|
/// Rows in this library not yet converted to the true-instant date
|
||||||
|
/// contract (`date_utc IS NULL`, ADR-0001 / ticket #118). Drains to 0
|
||||||
|
/// over watcher ticks; `None` when the count query failed.
|
||||||
|
pub date_backfill_remaining: Option<i64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(serde::Serialize)]
|
#[derive(serde::Serialize)]
|
||||||
@@ -488,6 +492,12 @@ pub async fn list_libraries(_claims: Claims, app_state: Data<AppState>) -> impl
|
|||||||
.library_health
|
.library_health
|
||||||
.read()
|
.read()
|
||||||
.unwrap_or_else(|e| e.into_inner());
|
.unwrap_or_else(|e| e.into_inner());
|
||||||
|
let context = opentelemetry::Context::new();
|
||||||
|
let mut exif_dao = app_state
|
||||||
|
.insight_generator
|
||||||
|
.exif_dao()
|
||||||
|
.lock()
|
||||||
|
.expect("Unable to lock ExifDao");
|
||||||
let libraries = live_guard
|
let libraries = live_guard
|
||||||
.iter()
|
.iter()
|
||||||
.map(|lib| LibraryStatus {
|
.map(|lib| LibraryStatus {
|
||||||
@@ -496,6 +506,11 @@ pub async fn list_libraries(_claims: Claims, app_state: Data<AppState>) -> impl
|
|||||||
.get(&lib.id)
|
.get(&lib.id)
|
||||||
.cloned()
|
.cloned()
|
||||||
.unwrap_or(LibraryHealth::Online),
|
.unwrap_or(LibraryHealth::Online),
|
||||||
|
// Live count off the partial index — cheap, and exact even for
|
||||||
|
// a stale library the watcher isn't refreshing.
|
||||||
|
date_backfill_remaining: exif_dao
|
||||||
|
.count_rows_needing_utc_backfill(&context, lib.id)
|
||||||
|
.ok(),
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
HttpResponse::Ok().json(LibrariesResponse {
|
HttpResponse::Ok().json(LibrariesResponse {
|
||||||
|
|||||||
@@ -151,6 +151,12 @@ fn main() -> std::io::Result<()> {
|
|||||||
.registry
|
.registry
|
||||||
.register(Box::new(thumbnails::VIDEO_GAUGE.clone()))
|
.register(Box::new(thumbnails::VIDEO_GAUGE.clone()))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
// True-instant date drain progress (#118). Refreshed per watcher
|
||||||
|
// tick; see `backfill::refresh_utc_backfill_progress`.
|
||||||
|
prometheus
|
||||||
|
.registry
|
||||||
|
.register(Box::new(backfill::DATE_BACKFILL_REMAINING.clone()))
|
||||||
|
.unwrap();
|
||||||
// HLS readiness gauges. Updated by the watcher every full-scan
|
// HLS readiness gauges. Updated by the watcher every full-scan
|
||||||
// tick and on every `/hls/stats` request. See `hls_stats`.
|
// tick and on every `/hls/stats` request. See `hls_stats`.
|
||||||
prometheus
|
prometheus
|
||||||
|
|||||||
+44
-1
@@ -26,7 +26,10 @@ use log::{debug, error, info, warn};
|
|||||||
use crate::backfill;
|
use crate::backfill;
|
||||||
use crate::content_hash;
|
use crate::content_hash;
|
||||||
use crate::database::models::InsertImageExif;
|
use crate::database::models::InsertImageExif;
|
||||||
use crate::database::{ExifDao, PreviewDao, SqliteExifDao, SqlitePreviewDao};
|
use crate::database::{
|
||||||
|
ExifDao, PrecomputedReelDao, PreviewDao, SqliteExifDao, SqlitePrecomputedReelDao,
|
||||||
|
SqlitePreviewDao, SqliteUserAiPrefsDao, UserAiPrefsDao,
|
||||||
|
};
|
||||||
use crate::date_resolver;
|
use crate::date_resolver;
|
||||||
use crate::exif;
|
use crate::exif;
|
||||||
use crate::face_watch;
|
use crate::face_watch;
|
||||||
@@ -336,6 +339,17 @@ pub fn watch_files(
|
|||||||
Box::new(SqliteTagDao::default()) as Box<dyn tags::TagDao>
|
Box::new(SqliteTagDao::default()) as Box<dyn tags::TagDao>
|
||||||
));
|
));
|
||||||
|
|
||||||
|
// True-instant date drain (#118): day-reel invalidation + the
|
||||||
|
// display zone the day comparison runs in.
|
||||||
|
let precomputed_reel_dao = Arc::new(Mutex::new(
|
||||||
|
Box::new(SqlitePrecomputedReelDao::new()) as Box<dyn PrecomputedReelDao>
|
||||||
|
));
|
||||||
|
let user_ai_prefs_dao: Mutex<Box<dyn UserAiPrefsDao>> =
|
||||||
|
Mutex::new(Box::new(SqliteUserAiPrefsDao::new()));
|
||||||
|
// Last-known per-library remaining count behind the
|
||||||
|
// `imageserver_date_backfill_remaining` gauge.
|
||||||
|
let mut utc_backfill_remaining: HashMap<i32, i64> = HashMap::new();
|
||||||
|
|
||||||
let mut last_quick_scan = SystemTime::now();
|
let mut last_quick_scan = SystemTime::now();
|
||||||
// Initialize to UNIX_EPOCH so the *first* tick is treated as a
|
// Initialize to UNIX_EPOCH so the *first* tick is treated as a
|
||||||
// full scan. That replaces the legacy startup ScanDirectoryMessage
|
// full scan. That replaces the legacy startup ScanDirectoryMessage
|
||||||
@@ -494,6 +508,35 @@ pub fn watch_files(
|
|||||||
{
|
{
|
||||||
let context = opentelemetry::Context::new();
|
let context = opentelemetry::Context::new();
|
||||||
backfill::backfill_missing_date_taken(&context, lib, &exif_dao);
|
backfill::backfill_missing_date_taken(&context, lib, &exif_dao);
|
||||||
|
|
||||||
|
// True-instant drain (ADR-0001, #118): convert
|
||||||
|
// legacy mixed-convention rows, invalidating day
|
||||||
|
// reels whose day moved. Inside the online gate, so a
|
||||||
|
// stale library is skipped whole and resumes next
|
||||||
|
// tick. The display zone is re-read per tick so a
|
||||||
|
// prefs change applies without restart.
|
||||||
|
let display_tz = {
|
||||||
|
let mut prefs = user_ai_prefs_dao.lock().expect("prefs dao poisoned");
|
||||||
|
match prefs.get_prefs(&context) {
|
||||||
|
Ok(Some(p)) => {
|
||||||
|
crate::camera_tz::display_zone_for(p.display_tz.as_deref())
|
||||||
|
}
|
||||||
|
_ => crate::camera_tz::display_zone_for(None),
|
||||||
|
}
|
||||||
|
};
|
||||||
|
backfill::backfill_utc_dates(
|
||||||
|
&context,
|
||||||
|
lib,
|
||||||
|
&exif_dao,
|
||||||
|
&precomputed_reel_dao,
|
||||||
|
&display_tz,
|
||||||
|
);
|
||||||
|
backfill::refresh_utc_backfill_progress(
|
||||||
|
&context,
|
||||||
|
lib,
|
||||||
|
&exif_dao,
|
||||||
|
&mut utc_backfill_remaining,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
if is_full_scan {
|
if is_full_scan {
|
||||||
|
|||||||
Reference in new issue
Block a user