Compare commits
5
Commits
b7b59c1ba2
...
fcaf8487c2
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fcaf8487c2 | ||
|
|
98db2a48db | ||
|
|
ab16bc7da6 | ||
|
|
89fc24ceab | ||
|
|
3aa6ce314c |
No files matched your search
@@ -437,6 +437,28 @@ a stronger tool comes online (exiftool install, new filename regex),
|
|||||||
re-resolve out-of-band rather than re-introducing the steady-state
|
re-resolve out-of-band rather than re-introducing the steady-state
|
||||||
eligibility.
|
eligibility.
|
||||||
|
|
||||||
|
**True-instant drain (`backfill_utc_dates`, ticket #118).** Runs right
|
||||||
|
after `backfill_missing_date_taken` inside the per-library online gate,
|
||||||
|
sharing its `DATE_BACKFILL_MAX_PER_TICK` cap. Converts legacy
|
||||||
|
mixed-convention rows (`date_utc IS NULL AND date_taken IS NOT NULL`,
|
||||||
|
partial index `idx_image_exif_utc_backfill`) by re-running
|
||||||
|
`resolve_dates_batch` with empty priors (one exiftool subprocess per
|
||||||
|
batch) and writing **only** the new-contract triple — the legacy pair
|
||||||
|
stays frozen. Legacy `manual` rows convert via
|
||||||
|
`camera_tz::manual_date_resolution` instead of being re-resolved from
|
||||||
|
the file. Missing-file rows are paged past without spending the cap, and
|
||||||
|
a per-library resume cursor (carried across ticks by the watcher, wraps
|
||||||
|
at end of table) keeps unconvertible rows from stalling the rows behind
|
||||||
|
them; EXIF/exiftool-legacy rows wait when exiftool is absent (they'd
|
||||||
|
otherwise fall to fs_time). When a converted row's display-zone day differs from
|
||||||
|
the day its legacy value rendered on, the `day`-span `precomputed_reels`
|
||||||
|
rows covering either month-day (this library + `all`) are deleted and
|
||||||
|
unlinked; the nightly pre-gen regenerates. Daily conversation summaries
|
||||||
|
are SMS-derived and unaffected. Progress: Prometheus
|
||||||
|
`imageserver_date_backfill_remaining` (per tick) and
|
||||||
|
`date_backfill_remaining` per library on `GET /libraries` (live count of
|
||||||
|
`date_utc IS NULL`).
|
||||||
|
|
||||||
`/memories` is a single SQL query against this column
|
`/memories` is a single SQL query against this column
|
||||||
(`get_memories_in_window` in `src/database/mod.rs`), using
|
(`get_memories_in_window` in `src/database/mod.rs`), using
|
||||||
`strftime('%m-%d' | '%W' | '%m', date_taken, 'unixepoch', tz)` for
|
`strftime('%m-%d' | '%W' | '%m', date_taken, 'unixepoch', tz)` for
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
DROP INDEX IF EXISTS idx_image_exif_utc_backfill;
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
-- Partial index backing the true-instant backfill drain (ADR-0001, ticket
|
||||||
|
-- #118). The drain selects rows still on the legacy convention
|
||||||
|
-- (`date_utc IS NULL`) per library in id order, and the progress surface
|
||||||
|
-- (Prometheus `imageserver_date_backfill_remaining` + `/libraries`) counts
|
||||||
|
-- the same set. Mirrors `idx_image_exif_date_backfill`: the index shrinks
|
||||||
|
-- to nothing as the drain converges, so it costs ~0 after cutover.
|
||||||
|
CREATE INDEX idx_image_exif_utc_backfill
|
||||||
|
ON image_exif (library_id, id)
|
||||||
|
WHERE date_utc IS NULL;
|
||||||
+968
-7
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,76 @@
|
|||||||
|
//! Pre-cutover audit utility for the filename date class (ticket #117).
|
||||||
|
//!
|
||||||
|
//! One-shot, operator-run: lists every file matching a filename-class
|
||||||
|
//! pattern across the configured libraries — the file's current stored
|
||||||
|
//! date + source and the date / source / offset-source the re-based
|
||||||
|
//! resolver would assign it. Strictly read-only: it changes nothing.
|
||||||
|
//!
|
||||||
|
//! This is the human spot-check that gates the production backfill
|
||||||
|
//! drain: review the report, fix any mis-tagged pattern or
|
||||||
|
//! misclassified file, then start the drain.
|
||||||
|
//!
|
||||||
|
//! Usage: `cargo run --bin filename_audit` (reads DATABASE_URL and
|
||||||
|
//! EXCLUDED_DIRS from the environment, like the server).
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
use diesel::Connection;
|
||||||
|
use diesel::sqlite::SqliteConnection;
|
||||||
|
use image_api::filename_audit::{format_report, run_audit};
|
||||||
|
use image_api::libraries::load_all;
|
||||||
|
|
||||||
|
fn main() -> anyhow::Result<()> {
|
||||||
|
env_logger::init();
|
||||||
|
dotenv::dotenv().ok();
|
||||||
|
|
||||||
|
let database_url = dotenv::var("DATABASE_URL")
|
||||||
|
.map_err(|_| anyhow::anyhow!("DATABASE_URL is not set (check .env)"))?;
|
||||||
|
let base_path = dotenv::var("BASE_PATH").ok();
|
||||||
|
|
||||||
|
let mut conn =
|
||||||
|
SqliteConnection::establish(&database_url).map_err(|e| anyhow::anyhow!(e.to_string()))?;
|
||||||
|
|
||||||
|
// Migrations are already run by the server; the audit only SELECTs.
|
||||||
|
// A fresh database without the date-class columns would fail the
|
||||||
|
// select below with a clear SQL error — that is the expected signal.
|
||||||
|
|
||||||
|
let global_excludes: Vec<String> = dotenv::var("EXCLUDED_DIRS")
|
||||||
|
.unwrap_or_default()
|
||||||
|
.split(',')
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.map(String::from)
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
let libraries = load_all(&mut conn);
|
||||||
|
if libraries.is_empty() {
|
||||||
|
anyhow::bail!(
|
||||||
|
"no rows in the `libraries` table — is DATABASE_URL pointing at the live database?"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// BASE_PATH is informational for the operator (the libraries table
|
||||||
|
// carries the authoritative roots); warn when it differs from every
|
||||||
|
// known root so a stale env var doesn't mislead the review.
|
||||||
|
if let Some(base) = &base_path
|
||||||
|
&& let base = PathBuf::from(base)
|
||||||
|
&& !libraries
|
||||||
|
.iter()
|
||||||
|
.any(|l| PathBuf::from(&l.root_path) == base)
|
||||||
|
{
|
||||||
|
eprintln!(
|
||||||
|
"note: BASE_PATH ({}) does not match any library root; auditing the library table",
|
||||||
|
base.display()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
println!("Filename date class audit (read-only)");
|
||||||
|
println!("======================================");
|
||||||
|
println!("Libraries: {}", libraries.len());
|
||||||
|
println!();
|
||||||
|
|
||||||
|
let (rows, skipped) = run_audit(&mut conn, libraries, &global_excludes);
|
||||||
|
print!("{}", format_report(&rows, &skipped));
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
@@ -43,6 +43,17 @@ pub struct DuplicateRow {
|
|||||||
pub duplicate_decided_at: Option<i64>,
|
pub duplicate_decided_at: Option<i64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// One row the true-instant backfill drain (ADR-0001, ticket #118) still
|
||||||
|
/// has to convert: on the legacy convention (`date_utc IS NULL`) with a
|
||||||
|
/// legacy `date_taken` to compare the converted day against.
|
||||||
|
#[derive(Debug, Clone, Queryable)]
|
||||||
|
pub struct UtcBackfillCandidate {
|
||||||
|
pub id: i32,
|
||||||
|
pub rel_path: String,
|
||||||
|
pub date_taken: i64,
|
||||||
|
pub date_taken_source: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
pub mod calendar_dao;
|
pub mod calendar_dao;
|
||||||
pub mod daily_summary_dao;
|
pub mod daily_summary_dao;
|
||||||
pub mod insight_generation_job_dao;
|
pub mod insight_generation_job_dao;
|
||||||
@@ -513,6 +524,46 @@ pub trait ExifDao: Sync + Send {
|
|||||||
date_taken_tz_source: Option<&str>,
|
date_taken_tz_source: Option<&str>,
|
||||||
) -> Result<(), DbError>;
|
) -> Result<(), DbError>;
|
||||||
|
|
||||||
|
/// Rows the true-instant drain (ADR-0001, ticket #118) still has to
|
||||||
|
/// convert: `date_utc IS NULL AND date_taken IS NOT NULL` in
|
||||||
|
/// `library_id`, `id > after_id`, ordered by id ASC. Rows with both
|
||||||
|
/// NULL belong to `get_rows_needing_date_backfill` (whose writer fills
|
||||||
|
/// the new triple too). The `after_id` cursor lets the drain page past
|
||||||
|
/// rows it can't convert this tick (missing files) without spending
|
||||||
|
/// its per-tick cap on them. Backed by `idx_image_exif_utc_backfill`.
|
||||||
|
fn get_rows_needing_utc_backfill(
|
||||||
|
&mut self,
|
||||||
|
context: &opentelemetry::Context,
|
||||||
|
library_id: i32,
|
||||||
|
after_id: i32,
|
||||||
|
limit: i64,
|
||||||
|
) -> Result<Vec<UtcBackfillCandidate>, DbError>;
|
||||||
|
|
||||||
|
/// Persist a converted row's new-contract triple (`date_utc`,
|
||||||
|
/// `camera_offset_minutes`, `date_taken_tz_source`) by primary key.
|
||||||
|
/// Never touches the frozen legacy `date_taken` / `date_taken_source`
|
||||||
|
/// pair. Guarded on `date_utc IS NULL` so a manual edit or ingest
|
||||||
|
/// that raced the drain wins. Returns whether a row was written.
|
||||||
|
fn backfill_date_utc(
|
||||||
|
&mut self,
|
||||||
|
context: &opentelemetry::Context,
|
||||||
|
id: i32,
|
||||||
|
date_utc: i64,
|
||||||
|
camera_offset_minutes: Option<i32>,
|
||||||
|
date_taken_tz_source: &str,
|
||||||
|
) -> Result<bool, DbError>;
|
||||||
|
|
||||||
|
/// Progress surface for the true-instant drain: rows in `library_id`
|
||||||
|
/// not yet on the new contract (`date_utc IS NULL`). Counts every such
|
||||||
|
/// row — including NULL-date rows the legacy drain resolves and rows
|
||||||
|
/// whose file is missing (until the missing-file scan retires them) —
|
||||||
|
/// because cutover (#120) waits on exactly this set reaching zero.
|
||||||
|
fn count_rows_needing_utc_backfill(
|
||||||
|
&mut self,
|
||||||
|
context: &opentelemetry::Context,
|
||||||
|
library_id: i32,
|
||||||
|
) -> Result<i64, DbError>;
|
||||||
|
|
||||||
/// Find image_exif rows needing a CLIP embedding for semantic search:
|
/// Find image_exif rows needing a CLIP embedding for semantic search:
|
||||||
/// `clip_embedding IS NULL AND content_hash IS NOT NULL`, ordered by id
|
/// `clip_embedding IS NULL AND content_hash IS NOT NULL`, ordered by id
|
||||||
/// ASC, limited. Hash-less rows wait for `backfill_unhashed_backlog` to
|
/// ASC, limited. Hash-less rows wait for `backfill_unhashed_backlog` to
|
||||||
@@ -1501,6 +1552,95 @@ impl ExifDao for SqliteExifDao {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn get_rows_needing_utc_backfill(
|
||||||
|
&mut self,
|
||||||
|
context: &opentelemetry::Context,
|
||||||
|
library_id_val: i32,
|
||||||
|
after_id: i32,
|
||||||
|
limit: i64,
|
||||||
|
) -> Result<Vec<UtcBackfillCandidate>, DbError> {
|
||||||
|
trace_db_call(context, "query", "get_rows_needing_utc_backfill", |_span| {
|
||||||
|
use schema::image_exif::dsl::*;
|
||||||
|
|
||||||
|
let mut connection = self.connection.lock().expect("Unable to get ExifDao");
|
||||||
|
|
||||||
|
// Partial index `(library_id, id) WHERE date_utc IS NULL`
|
||||||
|
// serves the filter + order directly.
|
||||||
|
image_exif
|
||||||
|
.filter(library_id.eq(library_id_val))
|
||||||
|
.filter(date_utc.is_null())
|
||||||
|
.filter(date_taken.is_not_null())
|
||||||
|
.filter(id.gt(after_id))
|
||||||
|
.select((
|
||||||
|
id,
|
||||||
|
rel_path,
|
||||||
|
date_taken.assume_not_null(),
|
||||||
|
date_taken_source,
|
||||||
|
))
|
||||||
|
.order(id.asc())
|
||||||
|
.limit(limit)
|
||||||
|
.load::<UtcBackfillCandidate>(connection.deref_mut())
|
||||||
|
.map_err(|e| anyhow::anyhow!("Query error: {}", e))
|
||||||
|
})
|
||||||
|
.map_err(|e| DbError::log(DbErrorKind::QueryError, e))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn backfill_date_utc(
|
||||||
|
&mut self,
|
||||||
|
context: &opentelemetry::Context,
|
||||||
|
id_val: i32,
|
||||||
|
date_utc_val: i64,
|
||||||
|
camera_offset_minutes_val: Option<i32>,
|
||||||
|
date_taken_tz_source_val: &str,
|
||||||
|
) -> Result<bool, DbError> {
|
||||||
|
trace_db_call(context, "update", "backfill_date_utc", |_span| {
|
||||||
|
use schema::image_exif::dsl::*;
|
||||||
|
|
||||||
|
let mut connection = self.connection.lock().expect("Unable to get ExifDao");
|
||||||
|
|
||||||
|
// New-contract triple only — the legacy pair is frozen
|
||||||
|
// (ADR-0001).
|
||||||
|
diesel::update(image_exif.find(id_val).filter(date_utc.is_null()))
|
||||||
|
.set((
|
||||||
|
date_utc.eq(Some(date_utc_val)),
|
||||||
|
camera_offset_minutes.eq(camera_offset_minutes_val),
|
||||||
|
date_taken_tz_source.eq(Some(date_taken_tz_source_val)),
|
||||||
|
))
|
||||||
|
.execute(connection.deref_mut())
|
||||||
|
.map(|rows| rows > 0)
|
||||||
|
.map_err(|e| anyhow::anyhow!("diesel update failed (id={}): {}", id_val, e))
|
||||||
|
})
|
||||||
|
.map_err(|e| {
|
||||||
|
log::warn!("backfill_date_utc: {}", e);
|
||||||
|
DbError::new(DbErrorKind::UpdateError)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn count_rows_needing_utc_backfill(
|
||||||
|
&mut self,
|
||||||
|
context: &opentelemetry::Context,
|
||||||
|
library_id_val: i32,
|
||||||
|
) -> Result<i64, DbError> {
|
||||||
|
trace_db_call(
|
||||||
|
context,
|
||||||
|
"query",
|
||||||
|
"count_rows_needing_utc_backfill",
|
||||||
|
|_span| {
|
||||||
|
use schema::image_exif::dsl::*;
|
||||||
|
|
||||||
|
let mut connection = self.connection.lock().expect("Unable to get ExifDao");
|
||||||
|
|
||||||
|
image_exif
|
||||||
|
.filter(library_id.eq(library_id_val))
|
||||||
|
.filter(date_utc.is_null())
|
||||||
|
.count()
|
||||||
|
.get_result::<i64>(connection.deref_mut())
|
||||||
|
.map_err(|e| anyhow::anyhow!("Query error: {}", e))
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.map_err(|e| DbError::log(DbErrorKind::QueryError, e))
|
||||||
|
}
|
||||||
|
|
||||||
fn list_clip_unencoded_candidates(
|
fn list_clip_unencoded_candidates(
|
||||||
&mut self,
|
&mut self,
|
||||||
context: &opentelemetry::Context,
|
context: &opentelemetry::Context,
|
||||||
|
|||||||
@@ -58,6 +58,24 @@ pub trait PrecomputedReelDao: Sync + Send {
|
|||||||
/// to protect files a ledger row still points at.
|
/// to protect files a ledger row still points at.
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
fn all_cache_keys(&mut self, context: &opentelemetry::Context) -> Result<Vec<String>, DbError>;
|
fn all_cache_keys(&mut self, context: &opentelemetry::Context) -> Result<Vec<String>, DbError>;
|
||||||
|
|
||||||
|
/// Every reel row for `span` whose `library_key` is one of
|
||||||
|
/// `library_keys`. Used by the true-instant date drain (ticket #118)
|
||||||
|
/// to find the day reels a re-dated photo invalidates.
|
||||||
|
fn list_for_span(
|
||||||
|
&mut self,
|
||||||
|
context: &opentelemetry::Context,
|
||||||
|
span: &str,
|
||||||
|
library_keys: &[String],
|
||||||
|
) -> Result<Vec<PrecomputedReel>, DbError>;
|
||||||
|
|
||||||
|
/// Delete the given reel rows (stale-marking: the nightly pre-gen
|
||||||
|
/// finds no current row and regenerates). Returns rows deleted.
|
||||||
|
fn delete_ids(
|
||||||
|
&mut self,
|
||||||
|
context: &opentelemetry::Context,
|
||||||
|
ids: &[i32],
|
||||||
|
) -> Result<usize, DbError>;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct SqlitePrecomputedReelDao {
|
pub struct SqlitePrecomputedReelDao {
|
||||||
@@ -219,6 +237,49 @@ impl PrecomputedReelDao for SqlitePrecomputedReelDao {
|
|||||||
})
|
})
|
||||||
.map_err(|e| DbError::log(DbErrorKind::QueryError, e))
|
.map_err(|e| DbError::log(DbErrorKind::QueryError, e))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn list_for_span(
|
||||||
|
&mut self,
|
||||||
|
context: &opentelemetry::Context,
|
||||||
|
span: &str,
|
||||||
|
library_keys: &[String],
|
||||||
|
) -> Result<Vec<PrecomputedReel>, DbError> {
|
||||||
|
trace_db_call(context, "query", "list_for_span", |_span| {
|
||||||
|
use schema::precomputed_reels::dsl;
|
||||||
|
|
||||||
|
let mut connection = self
|
||||||
|
.connection
|
||||||
|
.lock()
|
||||||
|
.expect("Unable to lock PrecomputedReelDao");
|
||||||
|
|
||||||
|
dsl::precomputed_reels
|
||||||
|
.filter(dsl::span.eq(span))
|
||||||
|
.filter(dsl::library_key.eq_any(library_keys))
|
||||||
|
.load::<PrecomputedReel>(connection.deref_mut())
|
||||||
|
.map_err(|e| anyhow::anyhow!("Failed to list reels for span: {}", e))
|
||||||
|
})
|
||||||
|
.map_err(|e| DbError::log(DbErrorKind::QueryError, e))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn delete_ids(
|
||||||
|
&mut self,
|
||||||
|
context: &opentelemetry::Context,
|
||||||
|
ids: &[i32],
|
||||||
|
) -> Result<usize, DbError> {
|
||||||
|
trace_db_call(context, "delete", "delete_ids", |_span| {
|
||||||
|
use schema::precomputed_reels::dsl;
|
||||||
|
|
||||||
|
let mut connection = self
|
||||||
|
.connection
|
||||||
|
.lock()
|
||||||
|
.expect("Unable to lock PrecomputedReelDao");
|
||||||
|
|
||||||
|
diesel::delete(dsl::precomputed_reels.filter(dsl::id.eq_any(ids)))
|
||||||
|
.execute(connection.deref_mut())
|
||||||
|
.map_err(|e| anyhow::anyhow!("Failed to delete reels: {}", e))
|
||||||
|
})
|
||||||
|
.map_err(|e| DbError::log(DbErrorKind::UpdateError, e))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -351,7 +351,7 @@ const EXIFTOOL_EVIDENCE_TAGS: &[&str] = &[
|
|||||||
/// Cache the "exiftool exists on PATH" check across the process lifetime so
|
/// Cache the "exiftool exists on PATH" check across the process lifetime so
|
||||||
/// the per-tick backfill doesn't fork a doomed subprocess every iteration on
|
/// the per-tick backfill doesn't fork a doomed subprocess every iteration on
|
||||||
/// deploys without exiftool installed.
|
/// deploys without exiftool installed.
|
||||||
fn exiftool_available() -> bool {
|
pub fn exiftool_available() -> bool {
|
||||||
static AVAIL: OnceLock<bool> = OnceLock::new();
|
static AVAIL: OnceLock<bool> = OnceLock::new();
|
||||||
*AVAIL.get_or_init(|| {
|
*AVAIL.get_or_init(|| {
|
||||||
let ok = Command::new("exiftool")
|
let ok = Command::new("exiftool")
|
||||||
|
|||||||
@@ -0,0 +1,570 @@
|
|||||||
|
//! Pre-cutover audit utility for the filename date class (ticket #117).
|
||||||
|
//!
|
||||||
|
//! A one-shot, operator-run report listing every file that matches a
|
||||||
|
//! filename-class pattern: the file's current stored date + source and
|
||||||
|
//! the date / source / offset-source the re-based resolver would assign
|
||||||
|
//! it. It reads the live database and walks the libraries (honoring the
|
||||||
|
//! library exclusions and the online gate — offline libraries are
|
||||||
|
//! reported as skipped, never processed). It changes nothing: strictly
|
||||||
|
//! read-only, no writes to any table.
|
||||||
|
//!
|
||||||
|
//! Purpose: the filename class is a documented inversion of EXIF-wins
|
||||||
|
//! and depends on the operator's memory of "what these filenames mean".
|
||||||
|
//! This report is the human spot-check that gates the production
|
||||||
|
//! backfill drain — the operator reviews it, fixes any mis-tagged
|
||||||
|
//! pattern or misclassified file, and only then starts the drain.
|
||||||
|
//!
|
||||||
|
//! The proposed values mirror the drain's inputs: the resolver runs
|
||||||
|
//! with empty priors, exactly as the per-tick drain calls
|
||||||
|
//! `resolve_dates_batch` today. In-class files never touch exiftool, so
|
||||||
|
//! the audit spawns no subprocesses at all.
|
||||||
|
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
use diesel::prelude::*;
|
||||||
|
use diesel::sqlite::SqliteConnection;
|
||||||
|
|
||||||
|
use crate::database::schema::image_exif;
|
||||||
|
use crate::date_resolver::ResolvedDate;
|
||||||
|
use crate::libraries::{Library, LibraryHealth, probe_online};
|
||||||
|
use crate::memories::{PathExcluder, match_filename_date};
|
||||||
|
|
||||||
|
/// The stored date contract for one `image_exif` row: the legacy pair
|
||||||
|
/// plus the new-contract audit value (NULL until the drain converts it).
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub struct StoredDate {
|
||||||
|
pub date_taken: Option<i64>,
|
||||||
|
pub date_taken_source: Option<String>,
|
||||||
|
pub date_taken_tz_source: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One report row: a filename-class file, what the database has for it
|
||||||
|
/// now, and what the re-based resolver would assign.
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub struct AuditRow {
|
||||||
|
pub library_id: i32,
|
||||||
|
pub library_name: String,
|
||||||
|
/// Root-relative path, slash-separated.
|
||||||
|
pub rel_path: String,
|
||||||
|
/// Which filename-class pattern matched (`screenshot`, `dash`, …).
|
||||||
|
pub pattern: &'static str,
|
||||||
|
/// The row's stored contract; `None` when the file is not indexed
|
||||||
|
/// yet (no `image_exif` row).
|
||||||
|
pub current: Option<StoredDate>,
|
||||||
|
/// What the re-based resolver would assign (legacy value, source,
|
||||||
|
/// true instant, offset, offset-source).
|
||||||
|
pub proposed: ResolvedDate,
|
||||||
|
/// True when a stored row exists and its legacy `date_taken`
|
||||||
|
/// disagrees with the proposed legacy value — the rows the operator
|
||||||
|
/// needs to spot-check hardest.
|
||||||
|
pub changed: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A library the audit did not process, with the reason. Disabled
|
||||||
|
/// (operator kill switch) and offline (probe failed) are reported,
|
||||||
|
/// never partially processed.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub struct SkippedLibrary {
|
||||||
|
pub library_id: i32,
|
||||||
|
pub name: String,
|
||||||
|
pub reason: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One `image_exif` row's date contract, as loaded for the audit.
|
||||||
|
#[derive(diesel::Queryable)]
|
||||||
|
struct StoredDateRow {
|
||||||
|
rel_path: String,
|
||||||
|
date_taken: Option<i64>,
|
||||||
|
date_taken_source: Option<String>,
|
||||||
|
date_taken_tz_source: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Load `rel_path -> stored date fields` for one library. Read-only
|
||||||
|
/// (SELECTs only; `&mut` is diesel's connection contract, not a write).
|
||||||
|
pub fn stored_dates_for_library(
|
||||||
|
conn: &mut SqliteConnection,
|
||||||
|
library_id: i32,
|
||||||
|
) -> Result<HashMap<String, StoredDate>, diesel::result::Error> {
|
||||||
|
let rows: Vec<StoredDateRow> = image_exif::table
|
||||||
|
.filter(image_exif::library_id.eq(library_id))
|
||||||
|
.select((
|
||||||
|
image_exif::rel_path,
|
||||||
|
image_exif::date_taken,
|
||||||
|
image_exif::date_taken_source,
|
||||||
|
image_exif::date_taken_tz_source,
|
||||||
|
))
|
||||||
|
.load(conn)?;
|
||||||
|
Ok(rows
|
||||||
|
.into_iter()
|
||||||
|
.map(
|
||||||
|
|StoredDateRow {
|
||||||
|
rel_path,
|
||||||
|
date_taken,
|
||||||
|
date_taken_source,
|
||||||
|
date_taken_tz_source,
|
||||||
|
}| {
|
||||||
|
(
|
||||||
|
rel_path,
|
||||||
|
StoredDate {
|
||||||
|
date_taken,
|
||||||
|
date_taken_source,
|
||||||
|
date_taken_tz_source,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Audit one online library: walk its root (honoring the effective
|
||||||
|
/// excludes), match the filename class, and compute the proposed
|
||||||
|
/// resolution for every in-class file in a single resolver pass. Rows
|
||||||
|
/// are stable-sorted by `rel_path`. `stored` comes from
|
||||||
|
/// [`stored_dates_for_library`].
|
||||||
|
pub fn audit_library(
|
||||||
|
library: &Library,
|
||||||
|
global_excludes: &[String],
|
||||||
|
stored: &HashMap<String, StoredDate>,
|
||||||
|
) -> Vec<AuditRow> {
|
||||||
|
let root = PathBuf::from(&library.root_path);
|
||||||
|
let excluder = PathExcluder::new(&root, &library.effective_excluded_dirs(global_excludes));
|
||||||
|
|
||||||
|
let mut matched: Vec<(String, PathBuf, &'static str)> = Vec::new();
|
||||||
|
// Depth 0 (the root itself) is always allowed so a pathological
|
||||||
|
// exclude matching the root can't silently yield an empty report —
|
||||||
|
// same guard as file_scan::walk_library_files.
|
||||||
|
for entry in walkdir::WalkDir::new(&root)
|
||||||
|
.into_iter()
|
||||||
|
.filter_entry(|e| e.depth() == 0 || !excluder.is_excluded(e.path()))
|
||||||
|
.flatten()
|
||||||
|
{
|
||||||
|
if !entry.file_type().is_file() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Some(rel) = entry
|
||||||
|
.path()
|
||||||
|
.strip_prefix(&root)
|
||||||
|
.ok()
|
||||||
|
.map(|p| p.to_string_lossy().replace('\\', "/"))
|
||||||
|
else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let Some(fm) = match_filename_date(entry.file_name().to_str().unwrap_or("")) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
matched.push((rel, entry.path().to_path_buf(), fm.pattern));
|
||||||
|
}
|
||||||
|
matched.sort_by(|a, b| a.0.cmp(&b.0));
|
||||||
|
|
||||||
|
// One resolver pass for the whole library. In-class files resolve
|
||||||
|
// from the name alone, so this spawns no exiftool subprocess.
|
||||||
|
let paths: Vec<PathBuf> = matched.iter().map(|(_, p, _)| p.clone()).collect();
|
||||||
|
let resolved = crate::date_resolver::resolve_dates_batch(&paths, &HashMap::new());
|
||||||
|
|
||||||
|
matched
|
||||||
|
.into_iter()
|
||||||
|
.map(|(rel_path, abs, pattern)| {
|
||||||
|
// An in-class name always resolves (the class never fails to
|
||||||
|
// date a file it previously dated); `None` would be a bug in
|
||||||
|
// the resolver, not a missing file.
|
||||||
|
let proposed = resolved
|
||||||
|
.get(&abs)
|
||||||
|
.copied()
|
||||||
|
.expect("in-class filename must resolve without exiftool");
|
||||||
|
let current = stored.get(&rel_path).cloned();
|
||||||
|
let changed = current
|
||||||
|
.as_ref()
|
||||||
|
.map(|c| c.date_taken != Some(proposed.timestamp))
|
||||||
|
.unwrap_or(false);
|
||||||
|
AuditRow {
|
||||||
|
library_id: library.id,
|
||||||
|
library_name: library.name.clone(),
|
||||||
|
rel_path,
|
||||||
|
pattern,
|
||||||
|
current,
|
||||||
|
proposed,
|
||||||
|
changed,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Run the audit over every configured library: disabled libraries are
|
||||||
|
/// reported as skipped; online libraries are walked; probe-failed
|
||||||
|
/// libraries are reported as skipped (never partially processed). The
|
||||||
|
/// result is stable-sorted by library id, then `rel_path`, so two runs
|
||||||
|
/// diff cleanly.
|
||||||
|
pub fn run_audit(
|
||||||
|
conn: &mut SqliteConnection,
|
||||||
|
libraries: Vec<Library>,
|
||||||
|
global_excludes: &[String],
|
||||||
|
) -> (Vec<AuditRow>, Vec<SkippedLibrary>) {
|
||||||
|
let mut rows: Vec<AuditRow> = Vec::new();
|
||||||
|
let mut skipped: Vec<SkippedLibrary> = Vec::new();
|
||||||
|
|
||||||
|
for lib in libraries {
|
||||||
|
if !lib.enabled {
|
||||||
|
skipped.push(SkippedLibrary {
|
||||||
|
library_id: lib.id,
|
||||||
|
name: lib.name.clone(),
|
||||||
|
reason: "disabled (enabled=false)".to_string(),
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// Same online gate as the watcher: probe with prior knowledge
|
||||||
|
// (had_data) from the image_exif row count.
|
||||||
|
let had_data: i64 = image_exif::table
|
||||||
|
.filter(image_exif::library_id.eq(lib.id))
|
||||||
|
.count()
|
||||||
|
.get_result(conn)
|
||||||
|
.unwrap_or(0);
|
||||||
|
match probe_online(&lib, had_data > 0) {
|
||||||
|
LibraryHealth::Online => {
|
||||||
|
let stored = stored_dates_for_library(conn, lib.id).unwrap_or_default();
|
||||||
|
rows.extend(audit_library(&lib, global_excludes, &stored));
|
||||||
|
}
|
||||||
|
LibraryHealth::Stale { reason, .. } => {
|
||||||
|
skipped.push(SkippedLibrary {
|
||||||
|
library_id: lib.id,
|
||||||
|
name: lib.name.clone(),
|
||||||
|
reason: format!("offline ({reason})"),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
rows.sort_by(|a, b| {
|
||||||
|
(a.library_id, a.rel_path.as_str()).cmp(&(b.library_id, b.rel_path.as_str()))
|
||||||
|
});
|
||||||
|
(rows, skipped)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn format_epoch_utc(ts: Option<i64>) -> String {
|
||||||
|
match ts.and_then(|t| chrono::DateTime::<chrono::Utc>::from_timestamp(t, 0)) {
|
||||||
|
Some(dt) => dt.format("%Y-%m-%d %H:%M:%S").to_string(),
|
||||||
|
None => "-".to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Render the report. One entry per in-class file — three lines each:
|
||||||
|
/// header, `current:`, `proposed:` — stable-sorted, with a `*` on rows
|
||||||
|
/// whose stored legacy date would change; then the skipped libraries
|
||||||
|
/// and a summary. Deterministic: two runs diff cleanly.
|
||||||
|
pub fn format_report(rows: &[AuditRow], skipped: &[SkippedLibrary]) -> String {
|
||||||
|
let mut out = String::new();
|
||||||
|
for row in rows {
|
||||||
|
let current = match &row.current {
|
||||||
|
Some(c) => format!(
|
||||||
|
"{} [{}]",
|
||||||
|
format_epoch_utc(c.date_taken),
|
||||||
|
c.date_taken_source.as_deref().unwrap_or("-")
|
||||||
|
),
|
||||||
|
None => "unindexed".to_string(),
|
||||||
|
};
|
||||||
|
let proposed_tz = row
|
||||||
|
.proposed
|
||||||
|
.tz_source
|
||||||
|
.map(|t| t.as_str().to_string())
|
||||||
|
.unwrap_or_else(|| "-".into());
|
||||||
|
// The numeric offset (minutes ahead of UTC) is the value the
|
||||||
|
// operator actually spot-checks: a wall pattern resolved with
|
||||||
|
// empty evidence carries the server zone's offset for that date.
|
||||||
|
let proposed_offset = row
|
||||||
|
.proposed
|
||||||
|
.camera_offset_minutes
|
||||||
|
.map(|m| format!("{m:+}m"))
|
||||||
|
.unwrap_or_else(|| "-".into());
|
||||||
|
out.push_str(&format!(
|
||||||
|
"{}\t{:<19}\t{:<20}\tlib={} {}\n current: {}\n proposed: {} [{}] tz_source={} offset={}\n",
|
||||||
|
if row.changed { "*" } else { " " },
|
||||||
|
row.pattern,
|
||||||
|
row.rel_path,
|
||||||
|
row.library_id,
|
||||||
|
row.library_name,
|
||||||
|
current,
|
||||||
|
format_epoch_utc(Some(row.proposed.timestamp)),
|
||||||
|
row.proposed.source.as_str(),
|
||||||
|
proposed_tz,
|
||||||
|
proposed_offset,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if !skipped.is_empty() {
|
||||||
|
out.push_str("\nskipped libraries:\n");
|
||||||
|
for s in skipped {
|
||||||
|
out.push_str(&format!(
|
||||||
|
" lib={} {} — {}\n",
|
||||||
|
s.library_id, s.name, s.reason
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.push_str(&format!(
|
||||||
|
"\n{} file(s) in the filename class, {} would change, {} library(ies) skipped\n",
|
||||||
|
rows.len(),
|
||||||
|
rows.iter().filter(|r| r.changed).count(),
|
||||||
|
skipped.len()
|
||||||
|
));
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use diesel::Connection;
|
||||||
|
use diesel_migrations::{EmbeddedMigrations, MigrationHarness, embed_migrations};
|
||||||
|
use std::fs;
|
||||||
|
use tempfile::tempdir;
|
||||||
|
|
||||||
|
const DB_MIGRATIONS: EmbeddedMigrations = embed_migrations!();
|
||||||
|
|
||||||
|
fn migrated_conn() -> SqliteConnection {
|
||||||
|
let mut conn = SqliteConnection::establish(":memory:").expect("in-memory sqlite");
|
||||||
|
conn.run_pending_migrations(DB_MIGRATIONS)
|
||||||
|
.expect("migrations");
|
||||||
|
conn
|
||||||
|
}
|
||||||
|
|
||||||
|
fn seed_row(
|
||||||
|
conn: &mut SqliteConnection,
|
||||||
|
library_id: i32,
|
||||||
|
rel_path: &str,
|
||||||
|
date_taken: Option<i64>,
|
||||||
|
source: Option<&str>,
|
||||||
|
) {
|
||||||
|
diesel::insert_into(image_exif::table)
|
||||||
|
.values((
|
||||||
|
image_exif::library_id.eq(library_id),
|
||||||
|
image_exif::rel_path.eq(rel_path),
|
||||||
|
image_exif::date_taken.eq(date_taken),
|
||||||
|
image_exif::date_taken_source.eq(source),
|
||||||
|
image_exif::created_time.eq(1_700_000_000),
|
||||||
|
image_exif::last_modified.eq(1_700_000_000),
|
||||||
|
))
|
||||||
|
.execute(conn)
|
||||||
|
.expect("seed row");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn library(id: i32, name: &str, root: &std::path::Path, enabled: bool) -> Library {
|
||||||
|
Library {
|
||||||
|
id,
|
||||||
|
name: name.to_string(),
|
||||||
|
root_path: root.to_string_lossy().to_string(),
|
||||||
|
enabled,
|
||||||
|
excluded_dirs: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fixture library seeded with mixed conventions:
|
||||||
|
/// - `Screenshot_2014-06-01-20-44-50.png` — in-class wall, stored row
|
||||||
|
/// says exif/2021 (the inversion the operator must spot-check).
|
||||||
|
/// - `att_1422489664680106.jpeg` — in-class unix timestamp, stored
|
||||||
|
/// row already agrees.
|
||||||
|
/// - `2015-01-09_02-15-15.jpg` — in-class wall, not indexed yet.
|
||||||
|
/// - `plain_photo.jpg` — out of class, must not appear.
|
||||||
|
/// - `@eaDir/2016-02-02_03-03-03.jpg` — excluded dir, must not appear.
|
||||||
|
fn fixture_library(root: &std::path::Path) {
|
||||||
|
for path in [
|
||||||
|
"Screenshot_2014-06-01-20-44-50.png",
|
||||||
|
"att_1422489664680106.jpeg",
|
||||||
|
"2015-01-09_02-15-15.jpg",
|
||||||
|
"plain_photo.jpg",
|
||||||
|
"@eaDir/2016-02-02_03-03-03.jpg",
|
||||||
|
] {
|
||||||
|
let full = root.join(path);
|
||||||
|
if let Some(parent) = full.parent() {
|
||||||
|
fs::create_dir_all(parent).unwrap();
|
||||||
|
}
|
||||||
|
fs::write(&full, b"fixture").unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn audit_shows_inversion_timestamp_and_exclusions() {
|
||||||
|
let tmp = tempdir().unwrap();
|
||||||
|
fixture_library(tmp.path());
|
||||||
|
let mut conn = migrated_conn();
|
||||||
|
|
||||||
|
// 2021-01-15 12:00:00 UTC — a stale EXIF date the filename
|
||||||
|
// outranks. att_1422489664680106 -> first 10 digits =
|
||||||
|
// 1422489664 = 2015-01-29 07:21:04 UTC.
|
||||||
|
let stale_exif: i64 = 1_610_742_400;
|
||||||
|
seed_row(
|
||||||
|
&mut conn,
|
||||||
|
1,
|
||||||
|
"Screenshot_2014-06-01-20-44-50.png",
|
||||||
|
Some(stale_exif),
|
||||||
|
Some("exif"),
|
||||||
|
);
|
||||||
|
seed_row(
|
||||||
|
&mut conn,
|
||||||
|
1,
|
||||||
|
"att_1422489664680106.jpeg",
|
||||||
|
Some(1422489664),
|
||||||
|
Some("filename"),
|
||||||
|
);
|
||||||
|
|
||||||
|
let lib = library(1, "photos", tmp.path(), true);
|
||||||
|
let stored = stored_dates_for_library(&mut conn, 1).unwrap();
|
||||||
|
// Global excludes as the EXCLUDED_DIRS env var would provide in
|
||||||
|
// production (junk dirs like @eaDir never enter the class walk).
|
||||||
|
let rows = audit_library(&lib, &["@eaDir".to_string()], &stored);
|
||||||
|
|
||||||
|
// Exactly the three in-class files; out-of-class and excluded
|
||||||
|
// paths are absent.
|
||||||
|
let rels: Vec<&str> = rows.iter().map(|r| r.rel_path.as_str()).collect();
|
||||||
|
assert_eq!(
|
||||||
|
rels,
|
||||||
|
vec![
|
||||||
|
"2015-01-09_02-15-15.jpg",
|
||||||
|
"Screenshot_2014-06-01-20-44-50.png",
|
||||||
|
"att_1422489664680106.jpeg"
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
// The inversion: stored exif/2021, proposed filename/2014 wall
|
||||||
|
// (wall-as-UTC legacy), tz_source assumed_server (empty priors,
|
||||||
|
// mirroring the drain).
|
||||||
|
let inv = rows
|
||||||
|
.iter()
|
||||||
|
.find(|r| r.rel_path == "Screenshot_2014-06-01-20-44-50.png")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(inv.pattern, "screenshot");
|
||||||
|
assert_eq!(inv.current.as_ref().unwrap().date_taken, Some(stale_exif));
|
||||||
|
assert_eq!(
|
||||||
|
inv.current.as_ref().unwrap().date_taken_source.as_deref(),
|
||||||
|
Some("exif")
|
||||||
|
);
|
||||||
|
assert_eq!(inv.proposed.source.as_str(), "filename");
|
||||||
|
assert_eq!(
|
||||||
|
inv.proposed.timestamp,
|
||||||
|
1401655490 // 2014-06-01 20:44:50 as UTC (wall-as-UTC legacy)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
inv.proposed.tz_source.map(|t| t.as_str()),
|
||||||
|
Some("assumed_server")
|
||||||
|
);
|
||||||
|
// Server zone America/New_York is EDT (UTC-4) in June.
|
||||||
|
assert_eq!(inv.proposed.camera_offset_minutes, Some(-240));
|
||||||
|
assert!(inv.changed, "stored exif date must flag as changed");
|
||||||
|
|
||||||
|
// The unix timestamp: utc_direct, no offset, and the stored row
|
||||||
|
// already agrees -> not flagged.
|
||||||
|
let ts = rows
|
||||||
|
.iter()
|
||||||
|
.find(|r| r.rel_path == "att_1422489664680106.jpeg")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(ts.pattern, "unix_timestamp");
|
||||||
|
assert_eq!(ts.proposed.source.as_str(), "filename");
|
||||||
|
assert_eq!(ts.proposed.timestamp, 1422489664);
|
||||||
|
assert_eq!(ts.proposed.date_utc, Some(1422489664));
|
||||||
|
assert_eq!(ts.proposed.camera_offset_minutes, None);
|
||||||
|
assert_eq!(
|
||||||
|
ts.proposed.tz_source.map(|t| t.as_str()),
|
||||||
|
Some("utc_direct")
|
||||||
|
);
|
||||||
|
assert!(!ts.changed);
|
||||||
|
|
||||||
|
// Unindexed in-class file: current is None, not flagged.
|
||||||
|
let unidx = rows
|
||||||
|
.iter()
|
||||||
|
.find(|r| r.rel_path == "2015-01-09_02-15-15.jpg")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(unidx.pattern, "dash");
|
||||||
|
assert!(unidx.current.is_none());
|
||||||
|
assert!(!unidx.changed);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn run_audit_skips_disabled_and_offline_libraries() {
|
||||||
|
let tmp = tempdir().unwrap();
|
||||||
|
let online_root = tmp.path().join("online");
|
||||||
|
let missing_root = tmp.path().join("missing");
|
||||||
|
fs::create_dir_all(&online_root).unwrap();
|
||||||
|
fixture_library(&online_root);
|
||||||
|
// missing_root is never created -> probe fails.
|
||||||
|
|
||||||
|
let mut conn = migrated_conn();
|
||||||
|
// Give the offline library a row so had_data=true (the strict
|
||||||
|
// probe: empty dir with rows is suspicious).
|
||||||
|
seed_row(&mut conn, 2, "whatever.jpg", Some(1), Some("exif"));
|
||||||
|
|
||||||
|
let libs = vec![
|
||||||
|
library(2, "offline", &missing_root, true),
|
||||||
|
library(1, "online", &online_root, true),
|
||||||
|
library(3, "disabled", &online_root, false),
|
||||||
|
];
|
||||||
|
let (rows, skipped) = run_audit(&mut conn, libs, &[]);
|
||||||
|
|
||||||
|
let skipped_ids: Vec<i32> = skipped.iter().map(|s| s.library_id).collect();
|
||||||
|
assert_eq!(skipped_ids, vec![2, 3]);
|
||||||
|
assert!(skipped[0].reason.starts_with("offline"));
|
||||||
|
assert_eq!(skipped[1].reason, "disabled (enabled=false)");
|
||||||
|
|
||||||
|
// Only the online library was processed.
|
||||||
|
let libs_seen: Vec<i32> = rows.iter().map(|r| r.library_id).collect();
|
||||||
|
assert!(libs_seen.iter().all(|id| *id == 1));
|
||||||
|
assert!(!rows.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn run_audit_sorts_by_library_then_path() {
|
||||||
|
let tmp = tempdir().unwrap();
|
||||||
|
let a = tmp.path().join("a");
|
||||||
|
let b = tmp.path().join("b");
|
||||||
|
fs::create_dir_all(&a).unwrap();
|
||||||
|
fs::create_dir_all(&b).unwrap();
|
||||||
|
fs::write(a.join("2015-02-02_02-02-02.jpg"), b"").unwrap();
|
||||||
|
fs::write(a.join("Screenshot_2014-06-01-20-44-50.png"), b"").unwrap();
|
||||||
|
fs::write(b.join("1401638400.jpeg"), b"").unwrap();
|
||||||
|
// Intentionally pass libraries out of order.
|
||||||
|
let libs = vec![library(2, "b", &b, true), library(1, "a", &a, true)];
|
||||||
|
let mut conn = migrated_conn();
|
||||||
|
let (rows, skipped) = run_audit(&mut conn, libs, &[]);
|
||||||
|
assert!(skipped.is_empty());
|
||||||
|
let got: Vec<(i32, String)> = rows
|
||||||
|
.iter()
|
||||||
|
.map(|r| (r.library_id, r.rel_path.clone()))
|
||||||
|
.collect();
|
||||||
|
assert_eq!(
|
||||||
|
got,
|
||||||
|
vec![
|
||||||
|
(1, "2015-02-02_02-02-02.jpg".to_string()),
|
||||||
|
(1, "Screenshot_2014-06-01-20-44-50.png".to_string()),
|
||||||
|
(2, "1401638400.jpeg".to_string()),
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn format_report_is_deterministic_and_marks_changed() {
|
||||||
|
let tmp = tempdir().unwrap();
|
||||||
|
fixture_library(tmp.path());
|
||||||
|
let mut conn = migrated_conn();
|
||||||
|
seed_row(
|
||||||
|
&mut conn,
|
||||||
|
1,
|
||||||
|
"Screenshot_2014-06-01-20-44-50.png",
|
||||||
|
Some(1_610_742_400),
|
||||||
|
Some("exif"),
|
||||||
|
);
|
||||||
|
let lib = library(1, "photos", tmp.path(), true);
|
||||||
|
let (rows, skipped) = run_audit(&mut conn, vec![lib], &["@eaDir".to_string()]);
|
||||||
|
let first = format_report(&rows, &skipped);
|
||||||
|
let second = format_report(&rows, &skipped);
|
||||||
|
assert_eq!(first, second);
|
||||||
|
assert!(first.contains("*\t"), "changed row must be flagged");
|
||||||
|
assert!(first.contains("screenshot"));
|
||||||
|
assert!(first.contains("utc_direct"));
|
||||||
|
// Numeric offset is printed: the wall row carries the server-zone
|
||||||
|
// offset for that date, timestamp rows have none.
|
||||||
|
assert!(first.contains("offset=-240m"));
|
||||||
|
assert!(first.contains("offset=-\n"));
|
||||||
|
// The stored source renders without debug quotes (P1 from
|
||||||
|
// review): `2021-... [exif]`, never `["exif"]`.
|
||||||
|
assert!(first.contains("[exif]"));
|
||||||
|
assert!(!first.contains("\"exif\""));
|
||||||
|
assert!(first.contains("3 file(s) in the filename class, 1 would change"));
|
||||||
|
// Excluded path must not appear anywhere in the report.
|
||||||
|
assert!(!first.contains("2016-02-02_03-03-03.jpg"));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1756,6 +1756,35 @@ mod tests {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn get_rows_needing_utc_backfill(
|
||||||
|
&mut self,
|
||||||
|
_context: &opentelemetry::Context,
|
||||||
|
_library_id: i32,
|
||||||
|
_after_id: i32,
|
||||||
|
_limit: i64,
|
||||||
|
) -> Result<Vec<crate::database::UtcBackfillCandidate>, DbError> {
|
||||||
|
Ok(Vec::new())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn backfill_date_utc(
|
||||||
|
&mut self,
|
||||||
|
_context: &opentelemetry::Context,
|
||||||
|
_id: i32,
|
||||||
|
_date_utc: i64,
|
||||||
|
_camera_offset_minutes: Option<i32>,
|
||||||
|
_date_taken_tz_source: &str,
|
||||||
|
) -> Result<bool, DbError> {
|
||||||
|
Ok(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn count_rows_needing_utc_backfill(
|
||||||
|
&mut self,
|
||||||
|
_context: &opentelemetry::Context,
|
||||||
|
_library_id: i32,
|
||||||
|
) -> Result<i64, DbError> {
|
||||||
|
Ok(0)
|
||||||
|
}
|
||||||
|
|
||||||
fn set_manual_date_taken(
|
fn set_manual_date_taken(
|
||||||
&mut self,
|
&mut self,
|
||||||
_context: &opentelemetry::Context,
|
_context: &opentelemetry::Context,
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ pub mod face_watch;
|
|||||||
pub mod faces;
|
pub mod faces;
|
||||||
pub mod file_scan;
|
pub mod file_scan;
|
||||||
pub mod file_types;
|
pub mod file_types;
|
||||||
|
pub mod filename_audit;
|
||||||
pub mod files;
|
pub mod files;
|
||||||
pub mod geo;
|
pub mod geo;
|
||||||
pub mod libraries;
|
pub mod libraries;
|
||||||
|
|||||||
@@ -460,6 +460,10 @@ pub struct LibraryStatus {
|
|||||||
#[serde(flatten)]
|
#[serde(flatten)]
|
||||||
pub library: Library,
|
pub library: Library,
|
||||||
pub health: LibraryHealth,
|
pub health: LibraryHealth,
|
||||||
|
/// Rows in this library not yet converted to the true-instant date
|
||||||
|
/// contract (`date_utc IS NULL`, ADR-0001 / ticket #118). Drains to 0
|
||||||
|
/// over watcher ticks; `None` when the count query failed.
|
||||||
|
pub date_backfill_remaining: Option<i64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(serde::Serialize)]
|
#[derive(serde::Serialize)]
|
||||||
@@ -488,6 +492,12 @@ pub async fn list_libraries(_claims: Claims, app_state: Data<AppState>) -> impl
|
|||||||
.library_health
|
.library_health
|
||||||
.read()
|
.read()
|
||||||
.unwrap_or_else(|e| e.into_inner());
|
.unwrap_or_else(|e| e.into_inner());
|
||||||
|
let context = opentelemetry::Context::new();
|
||||||
|
let mut exif_dao = app_state
|
||||||
|
.insight_generator
|
||||||
|
.exif_dao()
|
||||||
|
.lock()
|
||||||
|
.expect("Unable to lock ExifDao");
|
||||||
let libraries = live_guard
|
let libraries = live_guard
|
||||||
.iter()
|
.iter()
|
||||||
.map(|lib| LibraryStatus {
|
.map(|lib| LibraryStatus {
|
||||||
@@ -496,6 +506,11 @@ pub async fn list_libraries(_claims: Claims, app_state: Data<AppState>) -> impl
|
|||||||
.get(&lib.id)
|
.get(&lib.id)
|
||||||
.cloned()
|
.cloned()
|
||||||
.unwrap_or(LibraryHealth::Online),
|
.unwrap_or(LibraryHealth::Online),
|
||||||
|
// Live count off the partial index — cheap, and exact even for
|
||||||
|
// a stale library the watcher isn't refreshing.
|
||||||
|
date_backfill_remaining: exif_dao
|
||||||
|
.count_rows_needing_utc_backfill(&context, lib.id)
|
||||||
|
.ok(),
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
HttpResponse::Ok().json(LibrariesResponse {
|
HttpResponse::Ok().json(LibrariesResponse {
|
||||||
|
|||||||
@@ -151,6 +151,12 @@ fn main() -> std::io::Result<()> {
|
|||||||
.registry
|
.registry
|
||||||
.register(Box::new(thumbnails::VIDEO_GAUGE.clone()))
|
.register(Box::new(thumbnails::VIDEO_GAUGE.clone()))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
// True-instant date drain progress (#118). Refreshed per watcher
|
||||||
|
// tick; see `backfill::refresh_utc_backfill_progress`.
|
||||||
|
prometheus
|
||||||
|
.registry
|
||||||
|
.register(Box::new(backfill::DATE_BACKFILL_REMAINING.clone()))
|
||||||
|
.unwrap();
|
||||||
// HLS readiness gauges. Updated by the watcher every full-scan
|
// HLS readiness gauges. Updated by the watcher every full-scan
|
||||||
// tick and on every `/hls/stats` request. See `hls_stats`.
|
// tick and on every `/hls/stats` request. See `hls_stats`.
|
||||||
prometheus
|
prometheus
|
||||||
|
|||||||
+88
-14
@@ -146,7 +146,7 @@ pub struct MemoriesResponse {
|
|||||||
/// class**: the filename is the original date of record and outranks
|
/// class**: the filename is the original date of record and outranks
|
||||||
/// EXIF / container dates (ADR-0001). Files whose name matches nothing
|
/// EXIF / container dates (ADR-0001). Files whose name matches nothing
|
||||||
/// are out of the class and keep the global EXIF-wins waterfall.
|
/// are out of the class and keep the global EXIF-wins waterfall.
|
||||||
#[derive(Copy, Clone, Debug, PartialEq)]
|
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
||||||
pub enum FilenameDate {
|
pub enum FilenameDate {
|
||||||
/// Wall clock the device displayed (screenshot / dash / compact).
|
/// Wall clock the device displayed (screenshot / dash / compact).
|
||||||
Wall(NaiveDateTime),
|
Wall(NaiveDateTime),
|
||||||
@@ -174,12 +174,26 @@ impl FilenameDate {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn extract_date_from_filename(filename: &str) -> Option<FilenameDate> {
|
/// A filename that matched one of the filename-class patterns (#116):
|
||||||
|
/// which pattern fired and the date it carries.
|
||||||
|
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub struct FilenameMatch {
|
||||||
|
/// Stable pattern name for audit / debug surfaces: `screenshot`,
|
||||||
|
/// `screenshot_compact`, `dash`, `dash_compact`, `compact`,
|
||||||
|
/// `unix_timestamp`.
|
||||||
|
pub pattern: &'static str,
|
||||||
|
pub date: FilenameDate,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Match a filename against the filename-class patterns, reporting which
|
||||||
|
/// pattern fired. This is the canonical matcher; `extract_date_from_filename`
|
||||||
|
/// is the date-only projection of it.
|
||||||
|
pub fn match_filename_date(filename: &str) -> Option<FilenameMatch> {
|
||||||
// The wall-clock patterns (screenshot / dash / compact) carry the
|
// The wall-clock patterns (screenshot / dash / compact) carry the
|
||||||
// device's own clock; the unix-timestamp pattern carries a true
|
// device's own clock; the unix-timestamp pattern carries a true
|
||||||
// instant. `legacy_datetime()` reproduces the pre-migration
|
// instant. `FilenameDate::legacy_datetime()` reproduces the
|
||||||
// interpretation of each (naive-as-UTC / epoch) for the legacy
|
// pre-migration interpretation of each (naive-as-UTC / epoch) for the
|
||||||
// column and its consumers.
|
// legacy column and its consumers.
|
||||||
let build_wall_from_ymd_capture = |captures: ®ex::Captures| -> Option<NaiveDateTime> {
|
let build_wall_from_ymd_capture = |captures: ®ex::Captures| -> Option<NaiveDateTime> {
|
||||||
let year = captures.get(1)?.as_str().parse::<i32>().ok()?;
|
let year = captures.get(1)?.as_str().parse::<i32>().ok()?;
|
||||||
let month = captures.get(2)?.as_str().parse::<u32>().ok()?;
|
let month = captures.get(2)?.as_str().parse::<u32>().ok()?;
|
||||||
@@ -197,7 +211,10 @@ pub fn extract_date_from_filename(filename: &str) -> Option<FilenameDate> {
|
|||||||
.captures(filename)
|
.captures(filename)
|
||||||
.and_then(|c| build_wall_from_ymd_capture(&c))
|
.and_then(|c| build_wall_from_ymd_capture(&c))
|
||||||
{
|
{
|
||||||
return Some(FilenameDate::Wall(naive));
|
return Some(FilenameMatch {
|
||||||
|
pattern: "screenshot",
|
||||||
|
date: FilenameDate::Wall(naive),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// Screenshot format: Screenshot_20140601[_-]204450.png
|
// Screenshot format: Screenshot_20140601[_-]204450.png
|
||||||
@@ -206,7 +223,10 @@ pub fn extract_date_from_filename(filename: &str) -> Option<FilenameDate> {
|
|||||||
.captures(filename)
|
.captures(filename)
|
||||||
.and_then(|c| build_wall_from_ymd_capture(&c))
|
.and_then(|c| build_wall_from_ymd_capture(&c))
|
||||||
{
|
{
|
||||||
return Some(FilenameDate::Wall(naive));
|
return Some(FilenameMatch {
|
||||||
|
pattern: "screenshot_compact",
|
||||||
|
date: FilenameDate::Wall(naive),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. Dash format: 2015-01-09_02-15-15.jpg
|
// 2. Dash format: 2015-01-09_02-15-15.jpg
|
||||||
@@ -215,7 +235,10 @@ pub fn extract_date_from_filename(filename: &str) -> Option<FilenameDate> {
|
|||||||
.captures(filename)
|
.captures(filename)
|
||||||
.and_then(|c| build_wall_from_ymd_capture(&c))
|
.and_then(|c| build_wall_from_ymd_capture(&c))
|
||||||
{
|
{
|
||||||
return Some(FilenameDate::Wall(naive));
|
return Some(FilenameMatch {
|
||||||
|
pattern: "dash",
|
||||||
|
date: FilenameDate::Wall(naive),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// Dash with compact time format: 2015-01-09-021515.jpg
|
// Dash with compact time format: 2015-01-09-021515.jpg
|
||||||
@@ -224,7 +247,10 @@ pub fn extract_date_from_filename(filename: &str) -> Option<FilenameDate> {
|
|||||||
.captures(filename)
|
.captures(filename)
|
||||||
.and_then(|c| build_wall_from_ymd_capture(&c))
|
.and_then(|c| build_wall_from_ymd_capture(&c))
|
||||||
{
|
{
|
||||||
return Some(FilenameDate::Wall(naive));
|
return Some(FilenameMatch {
|
||||||
|
pattern: "dash_compact",
|
||||||
|
date: FilenameDate::Wall(naive),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3. Compact format: 20140927101712.jpg
|
// 3. Compact format: 20140927101712.jpg
|
||||||
@@ -233,7 +259,10 @@ pub fn extract_date_from_filename(filename: &str) -> Option<FilenameDate> {
|
|||||||
.captures(filename)
|
.captures(filename)
|
||||||
.and_then(|c| build_wall_from_ymd_capture(&c))
|
.and_then(|c| build_wall_from_ymd_capture(&c))
|
||||||
{
|
{
|
||||||
return Some(FilenameDate::Wall(naive));
|
return Some(FilenameMatch {
|
||||||
|
pattern: "compact",
|
||||||
|
date: FilenameDate::Wall(naive),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// 4. Timestamp format: 1401638400.jpeg, att_1422489664680106.jpeg, att_142248967186928.jpeg
|
// 4. Timestamp format: 1401638400.jpeg, att_1422489664680106.jpeg, att_142248967186928.jpeg
|
||||||
@@ -281,7 +310,10 @@ pub fn extract_date_from_filename(filename: &str) -> Option<FilenameDate> {
|
|||||||
.and_then(DateTime::<Utc>::from_timestamp_millis)
|
.and_then(DateTime::<Utc>::from_timestamp_millis)
|
||||||
.and_then(plausible_filename_date)
|
.and_then(plausible_filename_date)
|
||||||
{
|
{
|
||||||
return Some(FilenameDate::Utc(date_time));
|
return Some(FilenameMatch {
|
||||||
|
pattern: "unix_timestamp",
|
||||||
|
date: FilenameDate::Utc(date_time),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// For 14-16 digits, treat first 10 digits as seconds to avoid far future dates
|
// For 14-16 digits, treat first 10 digits as seconds to avoid far future dates
|
||||||
@@ -293,7 +325,10 @@ pub fn extract_date_from_filename(filename: &str) -> Option<FilenameDate> {
|
|||||||
.and_then(|ts| DateTime::<Utc>::from_timestamp(ts, 0))
|
.and_then(|ts| DateTime::<Utc>::from_timestamp(ts, 0))
|
||||||
.and_then(plausible_filename_date)
|
.and_then(plausible_filename_date)
|
||||||
{
|
{
|
||||||
return Some(FilenameDate::Utc(date_time));
|
return Some(FilenameMatch {
|
||||||
|
pattern: "unix_timestamp",
|
||||||
|
date: FilenameDate::Utc(date_time),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// Exactly 10 digits - seconds since epoch
|
// Exactly 10 digits - seconds since epoch
|
||||||
@@ -311,7 +346,10 @@ pub fn extract_date_from_filename(filename: &str) -> Option<FilenameDate> {
|
|||||||
if is_snapchat && date_time.timestamp() < SNAPCHAT_LAUNCH_TS {
|
if is_snapchat && date_time.timestamp() < SNAPCHAT_LAUNCH_TS {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
return Some(FilenameDate::Utc(date_time));
|
return Some(FilenameMatch {
|
||||||
|
pattern: "unix_timestamp",
|
||||||
|
date: FilenameDate::Utc(date_time),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// 11-12 digits: try as milliseconds (might be partial millisecond timestamp)
|
// 11-12 digits: try as milliseconds (might be partial millisecond timestamp)
|
||||||
@@ -322,13 +360,23 @@ pub fn extract_date_from_filename(filename: &str) -> Option<FilenameDate> {
|
|||||||
.and_then(DateTime::<Utc>::from_timestamp_millis)
|
.and_then(DateTime::<Utc>::from_timestamp_millis)
|
||||||
.and_then(plausible_filename_date)
|
.and_then(plausible_filename_date)
|
||||||
{
|
{
|
||||||
return Some(FilenameDate::Utc(date_time));
|
return Some(FilenameMatch {
|
||||||
|
pattern: "unix_timestamp",
|
||||||
|
date: FilenameDate::Utc(date_time),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Date-only projection of [`match_filename_date`]: the class match
|
||||||
|
/// without the pattern name. Resolver and consumer call sites use this;
|
||||||
|
/// the audit utility uses the full match.
|
||||||
|
pub fn extract_date_from_filename(filename: &str) -> Option<FilenameDate> {
|
||||||
|
match_filename_date(filename).map(|m| m.date)
|
||||||
|
}
|
||||||
|
|
||||||
/// Sanity gate for filename-derived timestamps. Real photo capture dates
|
/// Sanity gate for filename-derived timestamps. Real photo capture dates
|
||||||
/// live in a narrow window; values outside it are almost always sequential
|
/// live in a narrow window; values outside it are almost always sequential
|
||||||
/// scan IDs (`000227580005.jpg` → 1970) or arbitrary numeric suffixes
|
/// scan IDs (`000227580005.jpg` → 1970) or arbitrary numeric suffixes
|
||||||
@@ -782,6 +830,32 @@ mod tests {
|
|||||||
assert_eq!(date_time.timestamp(), 1383929602);
|
assert_eq!(date_time.timestamp(), 1383929602);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Pattern-name seam (ticket #117): the audit utility prints which
|
||||||
|
// pattern fired, so every class pattern must report a stable name.
|
||||||
|
#[test]
|
||||||
|
fn test_match_filename_date_reports_pattern_names() {
|
||||||
|
let cases: &[(&str, &str)] = &[
|
||||||
|
("Screenshot_2014-06-01-20-44-50.png", "screenshot"),
|
||||||
|
("Screenshot_20140601_204450.png", "screenshot_compact"),
|
||||||
|
("2015-01-09_02-15-15.jpg", "dash"),
|
||||||
|
("2015-01-09-021515.jpg", "dash_compact"),
|
||||||
|
("20140927101712.jpg", "compact"),
|
||||||
|
("att_1422489664680106.jpeg", "unix_timestamp"),
|
||||||
|
("1401638400.jpeg", "unix_timestamp"),
|
||||||
|
];
|
||||||
|
for (name, pattern) in cases {
|
||||||
|
let match_ =
|
||||||
|
match_filename_date(name).unwrap_or_else(|| panic!("no match for {}", name));
|
||||||
|
assert_eq!(match_.pattern, *pattern, "pattern name for {}", name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_match_filename_date_out_of_class_is_none() {
|
||||||
|
assert_eq!(match_filename_date("IMG_12345.jpg"), None);
|
||||||
|
assert_eq!(match_filename_date("000227580005.jpg"), None);
|
||||||
|
}
|
||||||
|
|
||||||
// The obsolete `test_memory_date_priority_*` tests covered the old
|
// The obsolete `test_memory_date_priority_*` tests covered the old
|
||||||
// request-time waterfall in `get_memory_date_with_priority`. Their
|
// request-time waterfall in `get_memory_date_with_priority`. Their
|
||||||
// replacement lives in `crate::date_resolver::tests` (resolver
|
// replacement lives in `crate::date_resolver::tests` (resolver
|
||||||
|
|||||||
+48
-1
@@ -26,7 +26,10 @@ use log::{debug, error, info, warn};
|
|||||||
use crate::backfill;
|
use crate::backfill;
|
||||||
use crate::content_hash;
|
use crate::content_hash;
|
||||||
use crate::database::models::InsertImageExif;
|
use crate::database::models::InsertImageExif;
|
||||||
use crate::database::{ExifDao, PreviewDao, SqliteExifDao, SqlitePreviewDao};
|
use crate::database::{
|
||||||
|
ExifDao, PrecomputedReelDao, PreviewDao, SqliteExifDao, SqlitePrecomputedReelDao,
|
||||||
|
SqlitePreviewDao, SqliteUserAiPrefsDao, UserAiPrefsDao,
|
||||||
|
};
|
||||||
use crate::date_resolver;
|
use crate::date_resolver;
|
||||||
use crate::exif;
|
use crate::exif;
|
||||||
use crate::face_watch;
|
use crate::face_watch;
|
||||||
@@ -336,6 +339,20 @@ pub fn watch_files(
|
|||||||
Box::new(SqliteTagDao::default()) as Box<dyn tags::TagDao>
|
Box::new(SqliteTagDao::default()) as Box<dyn tags::TagDao>
|
||||||
));
|
));
|
||||||
|
|
||||||
|
// True-instant date drain (#118): day-reel invalidation + the
|
||||||
|
// display zone the day comparison runs in.
|
||||||
|
let precomputed_reel_dao = Arc::new(Mutex::new(
|
||||||
|
Box::new(SqlitePrecomputedReelDao::new()) as Box<dyn PrecomputedReelDao>
|
||||||
|
));
|
||||||
|
let user_ai_prefs_dao: Mutex<Box<dyn UserAiPrefsDao>> =
|
||||||
|
Mutex::new(Box::new(SqliteUserAiPrefsDao::new()));
|
||||||
|
// Last-known per-library remaining count behind the
|
||||||
|
// `imageserver_date_backfill_remaining` gauge.
|
||||||
|
let mut utc_backfill_remaining: HashMap<i32, i64> = HashMap::new();
|
||||||
|
// Per-library resume cursor for the true-instant drain; in-memory,
|
||||||
|
// so a restart resumes from 0 (harmless — the drain is idempotent).
|
||||||
|
let mut utc_backfill_cursors: HashMap<i32, i32> = HashMap::new();
|
||||||
|
|
||||||
let mut last_quick_scan = SystemTime::now();
|
let mut last_quick_scan = SystemTime::now();
|
||||||
// Initialize to UNIX_EPOCH so the *first* tick is treated as a
|
// Initialize to UNIX_EPOCH so the *first* tick is treated as a
|
||||||
// full scan. That replaces the legacy startup ScanDirectoryMessage
|
// full scan. That replaces the legacy startup ScanDirectoryMessage
|
||||||
@@ -494,6 +511,36 @@ pub fn watch_files(
|
|||||||
{
|
{
|
||||||
let context = opentelemetry::Context::new();
|
let context = opentelemetry::Context::new();
|
||||||
backfill::backfill_missing_date_taken(&context, lib, &exif_dao);
|
backfill::backfill_missing_date_taken(&context, lib, &exif_dao);
|
||||||
|
|
||||||
|
// True-instant drain (ADR-0001, #118): convert
|
||||||
|
// legacy mixed-convention rows, invalidating day
|
||||||
|
// reels whose day moved. Inside the online gate, so a
|
||||||
|
// stale library is skipped whole and resumes next
|
||||||
|
// tick. The display zone is re-read per tick so a
|
||||||
|
// prefs change applies without restart.
|
||||||
|
let display_tz = {
|
||||||
|
let mut prefs = user_ai_prefs_dao.lock().expect("prefs dao poisoned");
|
||||||
|
match prefs.get_prefs(&context) {
|
||||||
|
Ok(Some(p)) => {
|
||||||
|
crate::camera_tz::display_zone_for(p.display_tz.as_deref())
|
||||||
|
}
|
||||||
|
_ => crate::camera_tz::display_zone_for(None),
|
||||||
|
}
|
||||||
|
};
|
||||||
|
backfill::backfill_utc_dates(
|
||||||
|
&context,
|
||||||
|
lib,
|
||||||
|
&exif_dao,
|
||||||
|
&precomputed_reel_dao,
|
||||||
|
&display_tz,
|
||||||
|
utc_backfill_cursors.entry(lib.id).or_insert(0),
|
||||||
|
);
|
||||||
|
backfill::refresh_utc_backfill_progress(
|
||||||
|
&context,
|
||||||
|
lib,
|
||||||
|
&exif_dao,
|
||||||
|
&mut utc_backfill_remaining,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
if is_full_scan {
|
if is_full_scan {
|
||||||
|
|||||||
Reference in new issue
Block a user