Merge pull request 'fix(dates): year-gate the filename wall-clock patterns (#160)' (#176) from fix/filename-wall-year-gate into master

Reviewed-on: #176

migrations/2026-10-16-000000_reresolve_implausible_filename_dates/down.sql

@@ -0,0 +1,3 @@
+-- Data fix only: the cleared dates were wrong and are re-resolved by the
+-- drain, so there is nothing to restore.
+SELECT 1;

migrations/2026-10-16-000000_reresolve_implausible_filename_dates/up.sql

@@ -0,0 +1,23 @@
+-- #160: the filename wall-clock patterns had no year gate, so counters and
+-- IDs shaped like YYYYMMDD_HHMMSS (`VID_22290524_080551.mp4`, Facebook's
+-- `10530519_101527…_o.jpg`) became dates of record in years like 1053 and
+-- 2229. The matcher now rejects years outside 1995..=now+1; this clears
+-- the rows it already dated so they're re-resolved by the fixed waterfall.
+--
+-- Both the legacy pair and the new-contract triple are cleared:
+-- `date_taken IS NULL` puts the row back in `backfill_missing_date_taken`,
+-- which rewrites all five columns (EXIF / exiftool / fs_time now that the
+-- filename no longer matches). Only filename-sourced rows dated outside
+-- [1995-01-01, 2028-01-01) UTC are touched; a real date re-resolves to
+-- itself anyway.
+UPDATE image_exif
+SET date_taken = NULL,
+    date_taken_source = NULL,
+    date_utc = NULL,
+    camera_offset_minutes = NULL,
+    date_taken_tz_source = NULL
+WHERE date_taken_source = 'filename'
+  AND (
+    date_taken < 788918400 OR date_taken >= 1830297600
+    OR date_utc < 788918400 OR date_utc >= 1830297600
+  );

src/filename_audit.rs

@@ -474,6 +474,100 @@ mod tests {
         assert!(!unidx.changed);
     }
 
+    #[test]
+    fn audit_leaves_out_counter_and_id_names_with_implausible_years() {
+        // #160: these matched screenshot_compact as years 2229 / 8580 /
+        // 1053 and the audit proposed them as dates of record.
+        let tmp = tempdir().unwrap();
+        for path in [
+            "VID_22290524_080551_818.mp4",
+            "VID_185800302_235056_842.mp4",
+            "10530519_10152700326093436_5479865808523667420_o.jpg",
+            "Screenshot_2014-06-01-20-44-50.png",
+        ] {
+            fs::write(tmp.path().join(path), b"fixture").unwrap();
+        }
+        let mut conn = migrated_conn();
+        let lib = library(1, "photos", tmp.path(), true);
+        let stored = stored_dates_for_library(&mut conn, 1).unwrap();
+        let rows = audit_library(&lib, &[], &stored);
+        let rels: Vec<&str> = rows.iter().map(|r| r.rel_path.as_str()).collect();
+        assert_eq!(rels, vec!["Screenshot_2014-06-01-20-44-50.png"]);
+    }
+
+    #[test]
+    fn migration_clears_implausible_filename_dates_for_re_resolution() {
+        // Migrate up to just before the #160 data fix, seed rows as the
+        // old matcher left them, then apply it.
+        const FIX: &str = "reresolve_implausible_filename_dates";
+        let mut conn = SqliteConnection::establish(":memory:").expect("in-memory sqlite");
+        for m in conn.pending_migrations(DB_MIGRATIONS).expect("pending") {
+            if m.name().to_string().contains(FIX) {
+                break;
+            }
+            conn.run_migration(&*m).expect("migration");
+        }
+        // 2229-05-24 08:05:51 and 1053-05-19 10:15:27 (wall-as-UTC), a
+        // real 2014 filename date, and an fs_time row that isn't ours to
+        // touch even though its date is odd.
+        seed_row(
+            &mut conn,
+            1,
+            "VID_22290524_080551_818.mp4",
+            Some(8_185_651_551),
+            Some("filename"),
+        );
+        seed_row(
+            &mut conn,
+            1,
+            "10530519_1015270_o.jpg",
+            Some(-28_925_732_673),
+            Some("filename"),
+        );
+        seed_row(
+            &mut conn,
+            1,
+            "Screenshot_2014-06-01-20-44-50.png",
+            Some(1_401_655_490),
+            Some("filename"),
+        );
+        seed_row(&mut conn, 1, "old_scan.jpg", Some(100), Some("fs_time"));
+        diesel::update(image_exif::table)
+            .set((
+                image_exif::date_utc.eq(image_exif::date_taken),
+                image_exif::date_taken_tz_source.eq("assumed_server"),
+            ))
+            .execute(&mut conn)
+            .unwrap();
+
+        conn.run_pending_migrations(DB_MIGRATIONS)
+            .expect("fix migration");
+
+        type Dates = (
+            String,
+            Option<i64>,
+            Option<String>,
+            Option<i64>,
+            Option<String>,
+        );
+        let rows: Vec<Dates> = image_exif::table
+            .select((
+                image_exif::rel_path,
+                image_exif::date_taken,
+                image_exif::date_taken_source,
+                image_exif::date_utc,
+                image_exif::date_taken_tz_source,
+            ))
+            .order(image_exif::rel_path)
+            .load(&mut conn)
+            .unwrap();
+        let cleared = |r: &Dates| r.1.is_none() && r.2.is_none() && r.3.is_none() && r.4.is_none();
+        for r in &rows {
+            let expect_cleared = r.0.starts_with("VID_") || r.0.starts_with("1053");
+            assert_eq!(cleared(r), expect_cleared, "{:?}", r);
+        }
+    }
+
     #[test]
     fn run_audit_skips_disabled_and_offline_libraries() {
         let tmp = tempdir().unwrap();

src/memories.rs

@@ -328,12 +328,19 @@ pub fn match_filename_date(filename: &str) -> Option<FilenameMatch> {
 
         NaiveDate::from_ymd_opt(year, month, day)?.and_hms_opt(hour, min, sec)
     };
+    // Eight digits, a separator and six more also match counters and IDs
+    // (`VID_22290524_080551.mp4`, Facebook's `10530519_101527…_o.jpg`), so
+    // the wall patterns get the same year gate as the timestamp branch; a
+    // miss falls through to the next pattern (#160).
+    let build_wall = |captures: regex::Captures| -> Option<NaiveDateTime> {
+        build_wall_from_ymd_capture(&captures).filter(|naive| plausible_filename_year(naive.year()))
+    };
 
     // 1. Screenshot format: Screenshot_2014-06-01-20-44-50.png
     if let Some(naive) = regex::Regex::new(r"(\d{4})-(\d{2})-(\d{2})-(\d{2})-(\d{2})-(\d{2})")
         .ok()?
         .captures(filename)
-        .and_then(|c| build_wall_from_ymd_capture(&c))
+        .and_then(build_wall)
     {
         return Some(FilenameMatch {
             pattern: "screenshot",
@@ -345,7 +352,7 @@ pub fn match_filename_date(filename: &str) -> Option<FilenameMatch> {
     if let Some(naive) = regex::Regex::new(r"(\d{4})(\d{2})(\d{2})[_-](\d{2})(\d{2})(\d{2})")
         .ok()?
         .captures(filename)
-        .and_then(|c| build_wall_from_ymd_capture(&c))
+        .and_then(build_wall)
     {
         return Some(FilenameMatch {
             pattern: "screenshot_compact",
@@ -357,7 +364,7 @@ pub fn match_filename_date(filename: &str) -> Option<FilenameMatch> {
     if let Some(naive) = regex::Regex::new(r"(\d{4})-(\d{2})-(\d{2})_(\d{2})-(\d{2})-(\d{2})")
         .ok()?
         .captures(filename)
-        .and_then(|c| build_wall_from_ymd_capture(&c))
+        .and_then(build_wall)
     {
         return Some(FilenameMatch {
             pattern: "dash",
@@ -369,7 +376,7 @@ pub fn match_filename_date(filename: &str) -> Option<FilenameMatch> {
     if let Some(naive) = regex::Regex::new(r"(\d{4})-(\d{2})-(\d{2})-(\d{2})(\d{2})(\d{2})")
         .ok()?
         .captures(filename)
-        .and_then(|c| build_wall_from_ymd_capture(&c))
+        .and_then(build_wall)
     {
         return Some(FilenameMatch {
             pattern: "dash_compact",
@@ -381,7 +388,7 @@ pub fn match_filename_date(filename: &str) -> Option<FilenameMatch> {
     if let Some(naive) = regex::Regex::new(r"(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})")
         .ok()?
         .captures(filename)
-        .and_then(|c| build_wall_from_ymd_capture(&c))
+        .and_then(build_wall)
     {
         return Some(FilenameMatch {
             pattern: "compact",
@@ -509,17 +516,15 @@ pub fn extract_date_from_filename(filename: &str) -> Option<FilenameDate> {
 /// `fs_time`, which is a much better proxy for content age than a fake
 /// epoch date.
 fn plausible_filename_date(dt: DateTime<Utc>) -> Option<DateTime<Utc>> {
-    use chrono::Datelike;
+    plausible_filename_year(dt.year()).then_some(dt)
-    let year = dt.year();
+}
-    // 1995 predates digital photography for most users; allowing one year
+
-    // past `now` covers clock-skew on freshly-taken shots without letting
+/// The year window both filename branches (wall clock and unix timestamp)
-    // 2037 timestamps through.
+/// accept. 1995 predates digital photography for most users; allowing one
-    let max_year = Utc::now().year() + 1;
+/// year past `now` covers clock-skew on freshly-taken shots without letting
-    if (1995..=max_year).contains(&year) {
+/// 2037 timestamps through.
-        Some(dt)
+fn plausible_filename_year(year: i32) -> bool {
-    } else {
+    (1995..=Utc::now().year() + 1).contains(&year)
-        None
-    }
 }
 
 /// Default lookback for `/memories`. The original 15-year cap pre-dated
@@ -994,6 +999,76 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_wall_patterns_reject_implausible_years() {
+        // One per wall pattern, too far out and too far back (#160).
+        let cases: &[&str] = &[
+            "Screenshot_2229-05-24-08-05-51.png",
+            "Screenshot_1053-05-19-10-15-27.png",
+            "VID_22290524_080551_818.mp4",
+            "IMG_10530519-101527.jpg",
+            "2229-05-24_08-05-51.jpg",
+            "1053-05-19_10-15-27.jpg",
+            "2229-05-24-080551.jpg",
+            "1053-05-19-101527.jpg",
+            "22290524080551.jpg",
+            "10530519101527.jpg",
+        ];
+        // A miss may still fall through to a later pattern (a 14-digit run
+        // can read as a unix timestamp); it just can't be a wall date.
+        for name in cases {
+            if let Some(m) = match_filename_date(name) {
+                assert!(
+                    !matches!(m.date, FilenameDate::Wall(_)),
+                    "{} matched wall pattern {}",
+                    name,
+                    m.pattern
+                );
+            }
+        }
+        assert_eq!(
+            match_filename_date("Screenshot_2229-05-24-08-05-51.png"),
+            None
+        );
+    }
+
+    #[test]
+    fn test_wall_patterns_reject_prod_counter_and_id_names() {
+        // Shapes from the prod audit on #160: an app's VID_<counter> and
+        // Facebook download IDs, both caught by screenshot_compact.
+        for name in [
+            "VID_20950505_044049.mp4",
+            "VID_185800302_235056_842.mp4",
+            "VID_121920715_134958_293.mp4",
+            "10530519_10152700326093436_5479865808523667420_o.jpg",
+            "10460624_10202578362324975_1067700837005629791_o.jpg",
+        ] {
+            assert_eq!(match_filename_date(name), None, "{} should not match", name);
+        }
+    }
+
+    #[test]
+    fn test_wall_pattern_miss_falls_through_to_next_pattern() {
+        // screenshot_compact sees year 2229 first; compact still finds 2014.
+        let m =
+            match_filename_date("VID_22290524_080551_20140927101712.jpg").expect("compact match");
+        assert_eq!(m.pattern, "compact");
+        let FilenameDate::Wall(naive) = m.date else {
+            panic!("compact is a wall pattern");
+        };
+        assert_eq!(naive.year(), 2014);
+    }
+
+    #[test]
+    fn test_wall_patterns_keep_plausible_edges() {
+        assert!(match_filename_date("Screenshot_1995-01-01-00-00-00.png").is_some());
+        let next_year = Utc::now().year() + 1;
+        let name = format!("Screenshot_{next_year}-01-01-00-00-00.png");
+        assert!(match_filename_date(&name).is_some());
+        let name = format!("Screenshot_{}-01-01-00-00-00.png", next_year + 1);
+        assert_eq!(match_filename_date(&name), None);
+    }
+
     #[test]
     fn test_match_filename_date_out_of_class_is_none() {
         assert_eq!(match_filename_date("IMG_12345.jpg"), None);