fix(dates): also clear implausible true instants with a good legacy date (#160)
The first #160 migration cleared rows whose legacy date_taken_source was 'filename'. But the true-instant drain wrote only the new-contract triple, so rows with a correct legacy date (e.g. an exiftool container date) could still carry a filename-derived implausible year in date_utc. Prod has three: VID_<counter> videos dated 2095 / 2229 / 8580 whose containers say 2016 / 2017. Clear just the triple wherever date_utc is outside [1995, 2028) UTC; backfill_utc_dates re-resolves it with the gated matcher, and the frozen legacy pair is left alone. On prod those three are the only rows with date_utc out of range. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
200abf9a10
commit
e6ae1727cc
3 files changed
+100
-6
No files matched your search
@@ -0,0 +1,3 @@
|
|||||||
|
-- Data fix only: the cleared instants were wrong and are re-resolved by the
|
||||||
|
-- drain, so there is nothing to restore.
|
||||||
|
SELECT 1;
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
-- #160 follow-up. The first fix (2026-10-16 migration) cleared rows whose
|
||||||
|
-- legacy `date_taken_source` was 'filename'. But the true-instant drain
|
||||||
|
-- (#118) re-ran the resolver over rows with a correct legacy date (e.g. an
|
||||||
|
-- exiftool container date) and wrote only the new-contract triple, so
|
||||||
|
-- some rows carry a filename-derived implausible year in `date_utc` while
|
||||||
|
-- their legacy pair says 'exiftool'. On prod: three VID_<counter> videos
|
||||||
|
-- (years 2095, 2229, 8580) whose containers say 2016/2017.
|
||||||
|
--
|
||||||
|
-- Clear just the triple on any row whose true instant is outside
|
||||||
|
-- [1995-01-01, 2028-01-01) UTC. `date_utc IS NULL` puts it back in
|
||||||
|
-- `backfill_utc_dates`, which re-resolves it with the gated matcher; the
|
||||||
|
-- legacy pair is left alone (it's frozen for rollback, and here correct).
|
||||||
|
UPDATE image_exif
|
||||||
|
SET date_utc = NULL,
|
||||||
|
camera_offset_minutes = NULL,
|
||||||
|
date_taken_tz_source = NULL
|
||||||
|
WHERE date_utc < 788918400 OR date_utc >= 1830297600;
|
||||||
+80
-6
@@ -495,18 +495,92 @@ mod tests {
|
|||||||
assert_eq!(rels, vec!["Screenshot_2014-06-01-20-44-50.png"]);
|
assert_eq!(rels, vec!["Screenshot_2014-06-01-20-44-50.png"]);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
/// A connection migrated up to (not including) the migration whose
|
||||||
fn migration_clears_implausible_filename_dates_for_re_resolution() {
|
/// name contains `name`, for seeding the state a data fix repairs.
|
||||||
// Migrate up to just before the #160 data fix, seed rows as the
|
fn conn_before_migration(name: &str) -> SqliteConnection {
|
||||||
// old matcher left them, then apply it.
|
|
||||||
const FIX: &str = "reresolve_implausible_filename_dates";
|
|
||||||
let mut conn = SqliteConnection::establish(":memory:").expect("in-memory sqlite");
|
let mut conn = SqliteConnection::establish(":memory:").expect("in-memory sqlite");
|
||||||
for m in conn.pending_migrations(DB_MIGRATIONS).expect("pending") {
|
for m in conn.pending_migrations(DB_MIGRATIONS).expect("pending") {
|
||||||
if m.name().to_string().contains(FIX) {
|
if m.name().to_string().contains(name) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
conn.run_migration(&*m).expect("migration");
|
conn.run_migration(&*m).expect("migration");
|
||||||
}
|
}
|
||||||
|
conn
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn migration_clears_implausible_true_instants_but_keeps_legacy_dates() {
|
||||||
|
// The prod shape the first fix missed: a correct exiftool legacy
|
||||||
|
// date, with a filename-derived 2229 instant written by the drain.
|
||||||
|
let mut conn = conn_before_migration("reresolve_implausible_utc_dates");
|
||||||
|
seed_row(
|
||||||
|
&mut conn,
|
||||||
|
1,
|
||||||
|
"VID_22290524_080551_818.mp4",
|
||||||
|
Some(1_500_069_534),
|
||||||
|
Some("exiftool"),
|
||||||
|
);
|
||||||
|
seed_row(&mut conn, 1, "beach.jpg", Some(1_500_069_534), Some("exif"));
|
||||||
|
let set_utc = |conn: &mut SqliteConnection, path: &str, utc: i64| {
|
||||||
|
diesel::update(image_exif::table.filter(image_exif::rel_path.eq(path)))
|
||||||
|
.set((
|
||||||
|
image_exif::date_utc.eq(utc),
|
||||||
|
image_exif::camera_offset_minutes.eq(-300),
|
||||||
|
image_exif::date_taken_tz_source.eq("assumed_server"),
|
||||||
|
))
|
||||||
|
.execute(conn)
|
||||||
|
.unwrap();
|
||||||
|
};
|
||||||
|
set_utc(&mut conn, "VID_22290524_080551_818.mp4", 8_185_669_551);
|
||||||
|
set_utc(&mut conn, "beach.jpg", 1_500_069_534);
|
||||||
|
|
||||||
|
conn.run_pending_migrations(DB_MIGRATIONS)
|
||||||
|
.expect("fix migration");
|
||||||
|
|
||||||
|
type Row = (
|
||||||
|
String,
|
||||||
|
Option<i64>,
|
||||||
|
Option<String>,
|
||||||
|
Option<i64>,
|
||||||
|
Option<i32>,
|
||||||
|
);
|
||||||
|
let rows: Vec<Row> = image_exif::table
|
||||||
|
.select((
|
||||||
|
image_exif::rel_path,
|
||||||
|
image_exif::date_taken,
|
||||||
|
image_exif::date_taken_source,
|
||||||
|
image_exif::date_utc,
|
||||||
|
image_exif::camera_offset_minutes,
|
||||||
|
))
|
||||||
|
.order(image_exif::rel_path)
|
||||||
|
.load(&mut conn)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
rows,
|
||||||
|
vec![
|
||||||
|
(
|
||||||
|
"VID_22290524_080551_818.mp4".into(),
|
||||||
|
Some(1_500_069_534),
|
||||||
|
Some("exiftool".into()),
|
||||||
|
None,
|
||||||
|
None
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"beach.jpg".into(),
|
||||||
|
Some(1_500_069_534),
|
||||||
|
Some("exif".into()),
|
||||||
|
Some(1_500_069_534),
|
||||||
|
Some(-300)
|
||||||
|
),
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn migration_clears_implausible_filename_dates_for_re_resolution() {
|
||||||
|
// Migrate up to just before the #160 data fix, seed rows as the
|
||||||
|
// old matcher left them, then apply it.
|
||||||
|
let mut conn = conn_before_migration("reresolve_implausible_filename_dates");
|
||||||
// 2229-05-24 08:05:51 and 1053-05-19 10:15:27 (wall-as-UTC), a
|
// 2229-05-24 08:05:51 and 1053-05-19 10:15:27 (wall-as-UTC), a
|
||||||
// real 2014 filename date, and an fs_time row that isn't ours to
|
// real 2014 filename date, and an fs_time row that isn't ours to
|
||||||
// touch even though its date is odd.
|
// touch even though its date is odd.
|
||||||
|
|||||||
Reference in new issue
Block a user