From e6ae1727cc71f2438dd28986799bd0251edaca60 Mon Sep 17 00:00:00 2001 From: Cameron Cordes Date: Fri, 16 Oct 2026 23:47:22 -0400 Subject: [PATCH] fix(dates): also clear implausible true instants with a good legacy date (#160) The first #160 migration cleared rows whose legacy date_taken_source was 'filename'. But the true-instant drain wrote only the new-contract triple, so rows with a correct legacy date (e.g. an exiftool container date) could still carry a filename-derived implausible year in date_utc. Prod has three: VID_ videos dated 2095 / 2229 / 8580 whose containers say 2016 / 2017. Clear just the triple wherever date_utc is outside [1995, 2028) UTC; backfill_utc_dates re-resolves it with the gated matcher, and the frozen legacy pair is left alone. On prod those three are the only rows with date_utc out of range. Co-Authored-By: Claude Opus 5.5 --- .../down.sql | 3 + .../up.sql | 17 ++++ src/filename_audit.rs | 86 +++++++++++++++++-- 3 files changed, 100 insertions(+), 6 deletions(-) create mode 100644 migrations/2026-10-17-000000_reresolve_implausible_utc_dates/down.sql create mode 100644 migrations/2026-10-17-000000_reresolve_implausible_utc_dates/up.sql diff --git a/migrations/2026-10-17-000000_reresolve_implausible_utc_dates/down.sql b/migrations/2026-10-17-000000_reresolve_implausible_utc_dates/down.sql new file mode 100644 index 0000000..af123be --- /dev/null +++ b/migrations/2026-10-17-000000_reresolve_implausible_utc_dates/down.sql @@ -0,0 +1,3 @@ +-- Data fix only: the cleared instants were wrong and are re-resolved by the +-- drain, so there is nothing to restore. +SELECT 1; diff --git a/migrations/2026-10-17-000000_reresolve_implausible_utc_dates/up.sql b/migrations/2026-10-17-000000_reresolve_implausible_utc_dates/up.sql new file mode 100644 index 0000000..a746679 --- /dev/null +++ b/migrations/2026-10-17-000000_reresolve_implausible_utc_dates/up.sql @@ -0,0 +1,17 @@ +-- #160 follow-up. The first fix (2026-10-16 migration) cleared rows whose +-- legacy `date_taken_source` was 'filename'. But the true-instant drain +-- (#118) re-ran the resolver over rows with a correct legacy date (e.g. an +-- exiftool container date) and wrote only the new-contract triple, so +-- some rows carry a filename-derived implausible year in `date_utc` while +-- their legacy pair says 'exiftool'. On prod: three VID_ videos +-- (years 2095, 2229, 8580) whose containers say 2016/2017. +-- +-- Clear just the triple on any row whose true instant is outside +-- [1995-01-01, 2028-01-01) UTC. `date_utc IS NULL` puts it back in +-- `backfill_utc_dates`, which re-resolves it with the gated matcher; the +-- legacy pair is left alone (it's frozen for rollback, and here correct). +UPDATE image_exif +SET date_utc = NULL, + camera_offset_minutes = NULL, + date_taken_tz_source = NULL +WHERE date_utc < 788918400 OR date_utc >= 1830297600; diff --git a/src/filename_audit.rs b/src/filename_audit.rs index 883e263..aa90d7f 100644 --- a/src/filename_audit.rs +++ b/src/filename_audit.rs @@ -495,18 +495,92 @@ mod tests { assert_eq!(rels, vec!["Screenshot_2014-06-01-20-44-50.png"]); } - #[test] - fn migration_clears_implausible_filename_dates_for_re_resolution() { - // Migrate up to just before the #160 data fix, seed rows as the - // old matcher left them, then apply it. - const FIX: &str = "reresolve_implausible_filename_dates"; + /// A connection migrated up to (not including) the migration whose + /// name contains `name`, for seeding the state a data fix repairs. + fn conn_before_migration(name: &str) -> SqliteConnection { let mut conn = SqliteConnection::establish(":memory:").expect("in-memory sqlite"); for m in conn.pending_migrations(DB_MIGRATIONS).expect("pending") { - if m.name().to_string().contains(FIX) { + if m.name().to_string().contains(name) { break; } conn.run_migration(&*m).expect("migration"); } + conn + } + + #[test] + fn migration_clears_implausible_true_instants_but_keeps_legacy_dates() { + // The prod shape the first fix missed: a correct exiftool legacy + // date, with a filename-derived 2229 instant written by the drain. + let mut conn = conn_before_migration("reresolve_implausible_utc_dates"); + seed_row( + &mut conn, + 1, + "VID_22290524_080551_818.mp4", + Some(1_500_069_534), + Some("exiftool"), + ); + seed_row(&mut conn, 1, "beach.jpg", Some(1_500_069_534), Some("exif")); + let set_utc = |conn: &mut SqliteConnection, path: &str, utc: i64| { + diesel::update(image_exif::table.filter(image_exif::rel_path.eq(path))) + .set(( + image_exif::date_utc.eq(utc), + image_exif::camera_offset_minutes.eq(-300), + image_exif::date_taken_tz_source.eq("assumed_server"), + )) + .execute(conn) + .unwrap(); + }; + set_utc(&mut conn, "VID_22290524_080551_818.mp4", 8_185_669_551); + set_utc(&mut conn, "beach.jpg", 1_500_069_534); + + conn.run_pending_migrations(DB_MIGRATIONS) + .expect("fix migration"); + + type Row = ( + String, + Option, + Option, + Option, + Option, + ); + let rows: Vec = image_exif::table + .select(( + image_exif::rel_path, + image_exif::date_taken, + image_exif::date_taken_source, + image_exif::date_utc, + image_exif::camera_offset_minutes, + )) + .order(image_exif::rel_path) + .load(&mut conn) + .unwrap(); + assert_eq!( + rows, + vec![ + ( + "VID_22290524_080551_818.mp4".into(), + Some(1_500_069_534), + Some("exiftool".into()), + None, + None + ), + ( + "beach.jpg".into(), + Some(1_500_069_534), + Some("exif".into()), + Some(1_500_069_534), + Some(-300) + ), + ] + ); + } + + #[test] + fn migration_clears_implausible_filename_dates_for_re_resolution() { + // Migrate up to just before the #160 data fix, seed rows as the + // old matcher left them, then apply it. + let mut conn = conn_before_migration("reresolve_implausible_filename_dates"); // 2229-05-24 08:05:51 and 1053-05-19 10:15:27 (wall-as-UTC), a // real 2014 filename date, and an fs_time row that isn't ours to // touch even though its date is odd.