From 36e4af9fa61d8933fd92957b4b60577758c30395 Mon Sep 17 00:00:00 2001 From: Cameron Cordes Date: Fri, 16 Oct 2026 23:06:23 -0400 Subject: [PATCH] fix(dates): year-gate the filename wall-clock patterns (#160) match_filename_date only gated the unix-timestamp branch to years 1995..=now+1; the five wall patterns returned any NaiveDateTime. Counters and IDs shaped like YYYYMMDD_HHMMSS (an app's VID__, Facebook's <8 digits>__o.jpg) became dates of record in years from 267 to 9985: 67 rows on prod, all via screenshot_compact. Both branches now share plausible_filename_year; a wall miss falls through to the next pattern and then the rest of the waterfall. The filename audit no longer proposes these dates. A data migration clears the five date columns on filename-sourced rows dated outside [1995, 2028) UTC, so backfill_missing_date_taken re-resolves them from EXIF / exiftool / fs time on the next ticks. Co-Authored-By: Claude Opus 5.5 --- .../down.sql | 3 + .../up.sql | 23 ++++ src/filename_audit.rs | 94 +++++++++++++++ src/memories.rs | 107 +++++++++++++++--- 4 files changed, 211 insertions(+), 16 deletions(-) create mode 100644 migrations/2026-10-16-000000_reresolve_implausible_filename_dates/down.sql create mode 100644 migrations/2026-10-16-000000_reresolve_implausible_filename_dates/up.sql diff --git a/migrations/2026-10-16-000000_reresolve_implausible_filename_dates/down.sql b/migrations/2026-10-16-000000_reresolve_implausible_filename_dates/down.sql new file mode 100644 index 0000000..e9f5403 --- /dev/null +++ b/migrations/2026-10-16-000000_reresolve_implausible_filename_dates/down.sql @@ -0,0 +1,3 @@ +-- Data fix only: the cleared dates were wrong and are re-resolved by the +-- drain, so there is nothing to restore. +SELECT 1; diff --git a/migrations/2026-10-16-000000_reresolve_implausible_filename_dates/up.sql b/migrations/2026-10-16-000000_reresolve_implausible_filename_dates/up.sql new file mode 100644 index 0000000..8b0a0ac --- /dev/null +++ b/migrations/2026-10-16-000000_reresolve_implausible_filename_dates/up.sql @@ -0,0 +1,23 @@ +-- #160: the filename wall-clock patterns had no year gate, so counters and +-- IDs shaped like YYYYMMDD_HHMMSS (`VID_22290524_080551.mp4`, Facebook's +-- `10530519_101527…_o.jpg`) became dates of record in years like 1053 and +-- 2229. The matcher now rejects years outside 1995..=now+1; this clears +-- the rows it already dated so they're re-resolved by the fixed waterfall. +-- +-- Both the legacy pair and the new-contract triple are cleared: +-- `date_taken IS NULL` puts the row back in `backfill_missing_date_taken`, +-- which rewrites all five columns (EXIF / exiftool / fs_time now that the +-- filename no longer matches). Only filename-sourced rows dated outside +-- [1995-01-01, 2028-01-01) UTC are touched; a real date re-resolves to +-- itself anyway. +UPDATE image_exif +SET date_taken = NULL, + date_taken_source = NULL, + date_utc = NULL, + camera_offset_minutes = NULL, + date_taken_tz_source = NULL +WHERE date_taken_source = 'filename' + AND ( + date_taken < 788918400 OR date_taken >= 1830297600 + OR date_utc < 788918400 OR date_utc >= 1830297600 + ); diff --git a/src/filename_audit.rs b/src/filename_audit.rs index c51547e..883e263 100644 --- a/src/filename_audit.rs +++ b/src/filename_audit.rs @@ -474,6 +474,100 @@ mod tests { assert!(!unidx.changed); } + #[test] + fn audit_leaves_out_counter_and_id_names_with_implausible_years() { + // #160: these matched screenshot_compact as years 2229 / 8580 / + // 1053 and the audit proposed them as dates of record. + let tmp = tempdir().unwrap(); + for path in [ + "VID_22290524_080551_818.mp4", + "VID_185800302_235056_842.mp4", + "10530519_10152700326093436_5479865808523667420_o.jpg", + "Screenshot_2014-06-01-20-44-50.png", + ] { + fs::write(tmp.path().join(path), b"fixture").unwrap(); + } + let mut conn = migrated_conn(); + let lib = library(1, "photos", tmp.path(), true); + let stored = stored_dates_for_library(&mut conn, 1).unwrap(); + let rows = audit_library(&lib, &[], &stored); + let rels: Vec<&str> = rows.iter().map(|r| r.rel_path.as_str()).collect(); + assert_eq!(rels, vec!["Screenshot_2014-06-01-20-44-50.png"]); + } + + #[test] + fn migration_clears_implausible_filename_dates_for_re_resolution() { + // Migrate up to just before the #160 data fix, seed rows as the + // old matcher left them, then apply it. + const FIX: &str = "reresolve_implausible_filename_dates"; + let mut conn = SqliteConnection::establish(":memory:").expect("in-memory sqlite"); + for m in conn.pending_migrations(DB_MIGRATIONS).expect("pending") { + if m.name().to_string().contains(FIX) { + break; + } + conn.run_migration(&*m).expect("migration"); + } + // 2229-05-24 08:05:51 and 1053-05-19 10:15:27 (wall-as-UTC), a + // real 2014 filename date, and an fs_time row that isn't ours to + // touch even though its date is odd. + seed_row( + &mut conn, + 1, + "VID_22290524_080551_818.mp4", + Some(8_185_651_551), + Some("filename"), + ); + seed_row( + &mut conn, + 1, + "10530519_1015270_o.jpg", + Some(-28_925_732_673), + Some("filename"), + ); + seed_row( + &mut conn, + 1, + "Screenshot_2014-06-01-20-44-50.png", + Some(1_401_655_490), + Some("filename"), + ); + seed_row(&mut conn, 1, "old_scan.jpg", Some(100), Some("fs_time")); + diesel::update(image_exif::table) + .set(( + image_exif::date_utc.eq(image_exif::date_taken), + image_exif::date_taken_tz_source.eq("assumed_server"), + )) + .execute(&mut conn) + .unwrap(); + + conn.run_pending_migrations(DB_MIGRATIONS) + .expect("fix migration"); + + type Dates = ( + String, + Option, + Option, + Option, + Option, + ); + let rows: Vec = image_exif::table + .select(( + image_exif::rel_path, + image_exif::date_taken, + image_exif::date_taken_source, + image_exif::date_utc, + image_exif::date_taken_tz_source, + )) + .order(image_exif::rel_path) + .load(&mut conn) + .unwrap(); + let cleared = |r: &Dates| r.1.is_none() && r.2.is_none() && r.3.is_none() && r.4.is_none(); + for r in &rows { + let expect_cleared = r.0.starts_with("VID_") || r.0.starts_with("1053"); + assert_eq!(cleared(r), expect_cleared, "{:?}", r); + } + } + #[test] fn run_audit_skips_disabled_and_offline_libraries() { let tmp = tempdir().unwrap(); diff --git a/src/memories.rs b/src/memories.rs index 3f1c3eb..e1c4875 100644 --- a/src/memories.rs +++ b/src/memories.rs @@ -328,12 +328,19 @@ pub fn match_filename_date(filename: &str) -> Option { NaiveDate::from_ymd_opt(year, month, day)?.and_hms_opt(hour, min, sec) }; + // Eight digits, a separator and six more also match counters and IDs + // (`VID_22290524_080551.mp4`, Facebook's `10530519_101527…_o.jpg`), so + // the wall patterns get the same year gate as the timestamp branch; a + // miss falls through to the next pattern (#160). + let build_wall = |captures: regex::Captures| -> Option { + build_wall_from_ymd_capture(&captures).filter(|naive| plausible_filename_year(naive.year())) + }; // 1. Screenshot format: Screenshot_2014-06-01-20-44-50.png if let Some(naive) = regex::Regex::new(r"(\d{4})-(\d{2})-(\d{2})-(\d{2})-(\d{2})-(\d{2})") .ok()? .captures(filename) - .and_then(|c| build_wall_from_ymd_capture(&c)) + .and_then(build_wall) { return Some(FilenameMatch { pattern: "screenshot", @@ -345,7 +352,7 @@ pub fn match_filename_date(filename: &str) -> Option { if let Some(naive) = regex::Regex::new(r"(\d{4})(\d{2})(\d{2})[_-](\d{2})(\d{2})(\d{2})") .ok()? .captures(filename) - .and_then(|c| build_wall_from_ymd_capture(&c)) + .and_then(build_wall) { return Some(FilenameMatch { pattern: "screenshot_compact", @@ -357,7 +364,7 @@ pub fn match_filename_date(filename: &str) -> Option { if let Some(naive) = regex::Regex::new(r"(\d{4})-(\d{2})-(\d{2})_(\d{2})-(\d{2})-(\d{2})") .ok()? .captures(filename) - .and_then(|c| build_wall_from_ymd_capture(&c)) + .and_then(build_wall) { return Some(FilenameMatch { pattern: "dash", @@ -369,7 +376,7 @@ pub fn match_filename_date(filename: &str) -> Option { if let Some(naive) = regex::Regex::new(r"(\d{4})-(\d{2})-(\d{2})-(\d{2})(\d{2})(\d{2})") .ok()? .captures(filename) - .and_then(|c| build_wall_from_ymd_capture(&c)) + .and_then(build_wall) { return Some(FilenameMatch { pattern: "dash_compact", @@ -381,7 +388,7 @@ pub fn match_filename_date(filename: &str) -> Option { if let Some(naive) = regex::Regex::new(r"(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})") .ok()? .captures(filename) - .and_then(|c| build_wall_from_ymd_capture(&c)) + .and_then(build_wall) { return Some(FilenameMatch { pattern: "compact", @@ -509,17 +516,15 @@ pub fn extract_date_from_filename(filename: &str) -> Option { /// `fs_time`, which is a much better proxy for content age than a fake /// epoch date. fn plausible_filename_date(dt: DateTime) -> Option> { - use chrono::Datelike; - let year = dt.year(); - // 1995 predates digital photography for most users; allowing one year - // past `now` covers clock-skew on freshly-taken shots without letting - // 2037 timestamps through. - let max_year = Utc::now().year() + 1; - if (1995..=max_year).contains(&year) { - Some(dt) - } else { - None - } + plausible_filename_year(dt.year()).then_some(dt) +} + +/// The year window both filename branches (wall clock and unix timestamp) +/// accept. 1995 predates digital photography for most users; allowing one +/// year past `now` covers clock-skew on freshly-taken shots without letting +/// 2037 timestamps through. +fn plausible_filename_year(year: i32) -> bool { + (1995..=Utc::now().year() + 1).contains(&year) } /// Default lookback for `/memories`. The original 15-year cap pre-dated @@ -994,6 +999,76 @@ mod tests { } } + #[test] + fn test_wall_patterns_reject_implausible_years() { + // One per wall pattern, too far out and too far back (#160). + let cases: &[&str] = &[ + "Screenshot_2229-05-24-08-05-51.png", + "Screenshot_1053-05-19-10-15-27.png", + "VID_22290524_080551_818.mp4", + "IMG_10530519-101527.jpg", + "2229-05-24_08-05-51.jpg", + "1053-05-19_10-15-27.jpg", + "2229-05-24-080551.jpg", + "1053-05-19-101527.jpg", + "22290524080551.jpg", + "10530519101527.jpg", + ]; + // A miss may still fall through to a later pattern (a 14-digit run + // can read as a unix timestamp); it just can't be a wall date. + for name in cases { + if let Some(m) = match_filename_date(name) { + assert!( + !matches!(m.date, FilenameDate::Wall(_)), + "{} matched wall pattern {}", + name, + m.pattern + ); + } + } + assert_eq!( + match_filename_date("Screenshot_2229-05-24-08-05-51.png"), + None + ); + } + + #[test] + fn test_wall_patterns_reject_prod_counter_and_id_names() { + // Shapes from the prod audit on #160: an app's VID_ and + // Facebook download IDs, both caught by screenshot_compact. + for name in [ + "VID_20950505_044049.mp4", + "VID_185800302_235056_842.mp4", + "VID_121920715_134958_293.mp4", + "10530519_10152700326093436_5479865808523667420_o.jpg", + "10460624_10202578362324975_1067700837005629791_o.jpg", + ] { + assert_eq!(match_filename_date(name), None, "{} should not match", name); + } + } + + #[test] + fn test_wall_pattern_miss_falls_through_to_next_pattern() { + // screenshot_compact sees year 2229 first; compact still finds 2014. + let m = + match_filename_date("VID_22290524_080551_20140927101712.jpg").expect("compact match"); + assert_eq!(m.pattern, "compact"); + let FilenameDate::Wall(naive) = m.date else { + panic!("compact is a wall pattern"); + }; + assert_eq!(naive.year(), 2014); + } + + #[test] + fn test_wall_patterns_keep_plausible_edges() { + assert!(match_filename_date("Screenshot_1995-01-01-00-00-00.png").is_some()); + let next_year = Utc::now().year() + 1; + let name = format!("Screenshot_{next_year}-01-01-00-00-00.png"); + assert!(match_filename_date(&name).is_some()); + let name = format!("Screenshot_{}-01-01-00-00-00.png", next_year + 1); + assert_eq!(match_filename_date(&name), None); + } + #[test] fn test_match_filename_date_out_of_class_is_none() { assert_eq!(match_filename_date("IMG_12345.jpg"), None);