fix(dates): also clear implausible true instants with a good legacy date (#160)
The first #160 migration cleared rows whose legacy date_taken_source was 'filename'. But the true-instant drain wrote only the new-contract triple, so rows with a correct legacy date (e.g. an exiftool container date) could still carry a filename-derived implausible year in date_utc. Prod has three: VID_<counter> videos dated 2095 / 2229 / 8580 whose containers say 2016 / 2017. Clear just the triple wherever date_utc is outside [1995, 2028) UTC; backfill_utc_dates re-resolves it with the gated matcher, and the frozen legacy pair is left alone. On prod those three are the only rows with date_utc out of range. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
200abf9a10
commit
e6ae1727cc
3 files changed
+100
-6
No files matched your search
@@ -0,0 +1,3 @@
|
||||
-- Data fix only: the cleared instants were wrong and are re-resolved by the
|
||||
-- drain, so there is nothing to restore.
|
||||
SELECT 1;
|
||||
@@ -0,0 +1,17 @@
|
||||
-- #160 follow-up. The first fix (2026-10-16 migration) cleared rows whose
|
||||
-- legacy `date_taken_source` was 'filename'. But the true-instant drain
|
||||
-- (#118) re-ran the resolver over rows with a correct legacy date (e.g. an
|
||||
-- exiftool container date) and wrote only the new-contract triple, so
|
||||
-- some rows carry a filename-derived implausible year in `date_utc` while
|
||||
-- their legacy pair says 'exiftool'. On prod: three VID_<counter> videos
|
||||
-- (years 2095, 2229, 8580) whose containers say 2016/2017.
|
||||
--
|
||||
-- Clear just the triple on any row whose true instant is outside
|
||||
-- [1995-01-01, 2028-01-01) UTC. `date_utc IS NULL` puts it back in
|
||||
-- `backfill_utc_dates`, which re-resolves it with the gated matcher; the
|
||||
-- legacy pair is left alone (it's frozen for rollback, and here correct).
|
||||
UPDATE image_exif
|
||||
SET date_utc = NULL,
|
||||
camera_offset_minutes = NULL,
|
||||
date_taken_tz_source = NULL
|
||||
WHERE date_utc < 788918400 OR date_utc >= 1830297600;
|
||||
+80
-6
@@ -495,18 +495,92 @@ mod tests {
|
||||
assert_eq!(rels, vec!["Screenshot_2014-06-01-20-44-50.png"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migration_clears_implausible_filename_dates_for_re_resolution() {
|
||||
// Migrate up to just before the #160 data fix, seed rows as the
|
||||
// old matcher left them, then apply it.
|
||||
const FIX: &str = "reresolve_implausible_filename_dates";
|
||||
/// A connection migrated up to (not including) the migration whose
|
||||
/// name contains `name`, for seeding the state a data fix repairs.
|
||||
fn conn_before_migration(name: &str) -> SqliteConnection {
|
||||
let mut conn = SqliteConnection::establish(":memory:").expect("in-memory sqlite");
|
||||
for m in conn.pending_migrations(DB_MIGRATIONS).expect("pending") {
|
||||
if m.name().to_string().contains(FIX) {
|
||||
if m.name().to_string().contains(name) {
|
||||
break;
|
||||
}
|
||||
conn.run_migration(&*m).expect("migration");
|
||||
}
|
||||
conn
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migration_clears_implausible_true_instants_but_keeps_legacy_dates() {
|
||||
// The prod shape the first fix missed: a correct exiftool legacy
|
||||
// date, with a filename-derived 2229 instant written by the drain.
|
||||
let mut conn = conn_before_migration("reresolve_implausible_utc_dates");
|
||||
seed_row(
|
||||
&mut conn,
|
||||
1,
|
||||
"VID_22290524_080551_818.mp4",
|
||||
Some(1_500_069_534),
|
||||
Some("exiftool"),
|
||||
);
|
||||
seed_row(&mut conn, 1, "beach.jpg", Some(1_500_069_534), Some("exif"));
|
||||
let set_utc = |conn: &mut SqliteConnection, path: &str, utc: i64| {
|
||||
diesel::update(image_exif::table.filter(image_exif::rel_path.eq(path)))
|
||||
.set((
|
||||
image_exif::date_utc.eq(utc),
|
||||
image_exif::camera_offset_minutes.eq(-300),
|
||||
image_exif::date_taken_tz_source.eq("assumed_server"),
|
||||
))
|
||||
.execute(conn)
|
||||
.unwrap();
|
||||
};
|
||||
set_utc(&mut conn, "VID_22290524_080551_818.mp4", 8_185_669_551);
|
||||
set_utc(&mut conn, "beach.jpg", 1_500_069_534);
|
||||
|
||||
conn.run_pending_migrations(DB_MIGRATIONS)
|
||||
.expect("fix migration");
|
||||
|
||||
type Row = (
|
||||
String,
|
||||
Option<i64>,
|
||||
Option<String>,
|
||||
Option<i64>,
|
||||
Option<i32>,
|
||||
);
|
||||
let rows: Vec<Row> = image_exif::table
|
||||
.select((
|
||||
image_exif::rel_path,
|
||||
image_exif::date_taken,
|
||||
image_exif::date_taken_source,
|
||||
image_exif::date_utc,
|
||||
image_exif::camera_offset_minutes,
|
||||
))
|
||||
.order(image_exif::rel_path)
|
||||
.load(&mut conn)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
rows,
|
||||
vec![
|
||||
(
|
||||
"VID_22290524_080551_818.mp4".into(),
|
||||
Some(1_500_069_534),
|
||||
Some("exiftool".into()),
|
||||
None,
|
||||
None
|
||||
),
|
||||
(
|
||||
"beach.jpg".into(),
|
||||
Some(1_500_069_534),
|
||||
Some("exif".into()),
|
||||
Some(1_500_069_534),
|
||||
Some(-300)
|
||||
),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migration_clears_implausible_filename_dates_for_re_resolution() {
|
||||
// Migrate up to just before the #160 data fix, seed rows as the
|
||||
// old matcher left them, then apply it.
|
||||
let mut conn = conn_before_migration("reresolve_implausible_filename_dates");
|
||||
// 2229-05-24 08:05:51 and 1053-05-19 10:15:27 (wall-as-UTC), a
|
||||
// real 2014 filename date, and an fs_time row that isn't ours to
|
||||
// touch even though its date is odd.
|
||||
|
||||
Reference in new issue
Block a user