fix(dates): year-gate the filename wall-clock patterns (#160)

match_filename_date only gated the unix-timestamp branch to years
1995..=now+1; the five wall patterns returned any NaiveDateTime. Counters
and IDs shaped like YYYYMMDD_HHMMSS (an app's VID_<counter>_<HHMMSS>,
Facebook's <8 digits>_<id>_o.jpg) became dates of record in years from
267 to 9985: 67 rows on prod, all via screenshot_compact.

Both branches now share plausible_filename_year; a wall miss falls
through to the next pattern and then the rest of the waterfall. The
filename audit no longer proposes these dates.

A data migration clears the five date columns on filename-sourced rows
dated outside [1995, 2028) UTC, so backfill_missing_date_taken
re-resolves them from EXIF / exiftool / fs time on the next ticks.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Cameron CordesandClaude Opus 5.5 committed 2026-10-16 23:06:23 -04:00
1 parent f1779358b5
commit 36e4af9fa6
4 files changed
+211 -16

No files matched your search

@@ -0,0 +1,3 @@
-- Data fix only: the cleared dates were wrong and are re-resolved by the
-- drain, so there is nothing to restore.
SELECT 1;
@@ -0,0 +1,23 @@
-- #160: the filename wall-clock patterns had no year gate, so counters and
-- IDs shaped like YYYYMMDD_HHMMSS (`VID_22290524_080551.mp4`, Facebook's
-- `10530519_101527…_o.jpg`) became dates of record in years like 1053 and
-- 2229. The matcher now rejects years outside 1995..=now+1; this clears
-- the rows it already dated so they're re-resolved by the fixed waterfall.
--
-- Both the legacy pair and the new-contract triple are cleared:
-- `date_taken IS NULL` puts the row back in `backfill_missing_date_taken`,
-- which rewrites all five columns (EXIF / exiftool / fs_time now that the
-- filename no longer matches). Only filename-sourced rows dated outside
-- [1995-01-01, 2028-01-01) UTC are touched; a real date re-resolves to
-- itself anyway.
UPDATE image_exif
SET date_taken = NULL,
date_taken_source = NULL,
date_utc = NULL,
camera_offset_minutes = NULL,
date_taken_tz_source = NULL
WHERE date_taken_source = 'filename'
AND (
date_taken < 788918400 OR date_taken >= 1830297600
OR date_utc < 788918400 OR date_utc >= 1830297600
);
+94
View File
@@ -474,6 +474,100 @@ mod tests {
assert!(!unidx.changed); assert!(!unidx.changed);
} }
#[test]
fn audit_leaves_out_counter_and_id_names_with_implausible_years() {
// #160: these matched screenshot_compact as years 2229 / 8580 /
// 1053 and the audit proposed them as dates of record.
let tmp = tempdir().unwrap();
for path in [
"VID_22290524_080551_818.mp4",
"VID_185800302_235056_842.mp4",
"10530519_10152700326093436_5479865808523667420_o.jpg",
"Screenshot_2014-06-01-20-44-50.png",
] {
fs::write(tmp.path().join(path), b"fixture").unwrap();
}
let mut conn = migrated_conn();
let lib = library(1, "photos", tmp.path(), true);
let stored = stored_dates_for_library(&mut conn, 1).unwrap();
let rows = audit_library(&lib, &[], &stored);
let rels: Vec<&str> = rows.iter().map(|r| r.rel_path.as_str()).collect();
assert_eq!(rels, vec!["Screenshot_2014-06-01-20-44-50.png"]);
}
#[test]
fn migration_clears_implausible_filename_dates_for_re_resolution() {
// Migrate up to just before the #160 data fix, seed rows as the
// old matcher left them, then apply it.
const FIX: &str = "reresolve_implausible_filename_dates";
let mut conn = SqliteConnection::establish(":memory:").expect("in-memory sqlite");
for m in conn.pending_migrations(DB_MIGRATIONS).expect("pending") {
if m.name().to_string().contains(FIX) {
break;
}
conn.run_migration(&*m).expect("migration");
}
// 2229-05-24 08:05:51 and 1053-05-19 10:15:27 (wall-as-UTC), a
// real 2014 filename date, and an fs_time row that isn't ours to
// touch even though its date is odd.
seed_row(
&mut conn,
1,
"VID_22290524_080551_818.mp4",
Some(8_185_651_551),
Some("filename"),
);
seed_row(
&mut conn,
1,
"10530519_1015270_o.jpg",
Some(-28_925_732_673),
Some("filename"),
);
seed_row(
&mut conn,
1,
"Screenshot_2014-06-01-20-44-50.png",
Some(1_401_655_490),
Some("filename"),
);
seed_row(&mut conn, 1, "old_scan.jpg", Some(100), Some("fs_time"));
diesel::update(image_exif::table)
.set((
image_exif::date_utc.eq(image_exif::date_taken),
image_exif::date_taken_tz_source.eq("assumed_server"),
))
.execute(&mut conn)
.unwrap();
conn.run_pending_migrations(DB_MIGRATIONS)
.expect("fix migration");
type Dates = (
String,
Option<i64>,
Option<String>,
Option<i64>,
Option<String>,
);
let rows: Vec<Dates> = image_exif::table
.select((
image_exif::rel_path,
image_exif::date_taken,
image_exif::date_taken_source,
image_exif::date_utc,
image_exif::date_taken_tz_source,
))
.order(image_exif::rel_path)
.load(&mut conn)
.unwrap();
let cleared = |r: &Dates| r.1.is_none() && r.2.is_none() && r.3.is_none() && r.4.is_none();
for r in &rows {
let expect_cleared = r.0.starts_with("VID_") || r.0.starts_with("1053");
assert_eq!(cleared(r), expect_cleared, "{:?}", r);
}
}
#[test] #[test]
fn run_audit_skips_disabled_and_offline_libraries() { fn run_audit_skips_disabled_and_offline_libraries() {
let tmp = tempdir().unwrap(); let tmp = tempdir().unwrap();
+91 -16
View File
@@ -328,12 +328,19 @@ pub fn match_filename_date(filename: &str) -> Option<FilenameMatch> {
NaiveDate::from_ymd_opt(year, month, day)?.and_hms_opt(hour, min, sec) NaiveDate::from_ymd_opt(year, month, day)?.and_hms_opt(hour, min, sec)
}; };
// Eight digits, a separator and six more also match counters and IDs
// (`VID_22290524_080551.mp4`, Facebook's `10530519_101527…_o.jpg`), so
// the wall patterns get the same year gate as the timestamp branch; a
// miss falls through to the next pattern (#160).
let build_wall = |captures: regex::Captures| -> Option<NaiveDateTime> {
build_wall_from_ymd_capture(&captures).filter(|naive| plausible_filename_year(naive.year()))
};
// 1. Screenshot format: Screenshot_2014-06-01-20-44-50.png // 1. Screenshot format: Screenshot_2014-06-01-20-44-50.png
if let Some(naive) = regex::Regex::new(r"(\d{4})-(\d{2})-(\d{2})-(\d{2})-(\d{2})-(\d{2})") if let Some(naive) = regex::Regex::new(r"(\d{4})-(\d{2})-(\d{2})-(\d{2})-(\d{2})-(\d{2})")
.ok()? .ok()?
.captures(filename) .captures(filename)
.and_then(|c| build_wall_from_ymd_capture(&c)) .and_then(build_wall)
{ {
return Some(FilenameMatch { return Some(FilenameMatch {
pattern: "screenshot", pattern: "screenshot",
@@ -345,7 +352,7 @@ pub fn match_filename_date(filename: &str) -> Option<FilenameMatch> {
if let Some(naive) = regex::Regex::new(r"(\d{4})(\d{2})(\d{2})[_-](\d{2})(\d{2})(\d{2})") if let Some(naive) = regex::Regex::new(r"(\d{4})(\d{2})(\d{2})[_-](\d{2})(\d{2})(\d{2})")
.ok()? .ok()?
.captures(filename) .captures(filename)
.and_then(|c| build_wall_from_ymd_capture(&c)) .and_then(build_wall)
{ {
return Some(FilenameMatch { return Some(FilenameMatch {
pattern: "screenshot_compact", pattern: "screenshot_compact",
@@ -357,7 +364,7 @@ pub fn match_filename_date(filename: &str) -> Option<FilenameMatch> {
if let Some(naive) = regex::Regex::new(r"(\d{4})-(\d{2})-(\d{2})_(\d{2})-(\d{2})-(\d{2})") if let Some(naive) = regex::Regex::new(r"(\d{4})-(\d{2})-(\d{2})_(\d{2})-(\d{2})-(\d{2})")
.ok()? .ok()?
.captures(filename) .captures(filename)
.and_then(|c| build_wall_from_ymd_capture(&c)) .and_then(build_wall)
{ {
return Some(FilenameMatch { return Some(FilenameMatch {
pattern: "dash", pattern: "dash",
@@ -369,7 +376,7 @@ pub fn match_filename_date(filename: &str) -> Option<FilenameMatch> {
if let Some(naive) = regex::Regex::new(r"(\d{4})-(\d{2})-(\d{2})-(\d{2})(\d{2})(\d{2})") if let Some(naive) = regex::Regex::new(r"(\d{4})-(\d{2})-(\d{2})-(\d{2})(\d{2})(\d{2})")
.ok()? .ok()?
.captures(filename) .captures(filename)
.and_then(|c| build_wall_from_ymd_capture(&c)) .and_then(build_wall)
{ {
return Some(FilenameMatch { return Some(FilenameMatch {
pattern: "dash_compact", pattern: "dash_compact",
@@ -381,7 +388,7 @@ pub fn match_filename_date(filename: &str) -> Option<FilenameMatch> {
if let Some(naive) = regex::Regex::new(r"(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})") if let Some(naive) = regex::Regex::new(r"(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})")
.ok()? .ok()?
.captures(filename) .captures(filename)
.and_then(|c| build_wall_from_ymd_capture(&c)) .and_then(build_wall)
{ {
return Some(FilenameMatch { return Some(FilenameMatch {
pattern: "compact", pattern: "compact",
@@ -509,17 +516,15 @@ pub fn extract_date_from_filename(filename: &str) -> Option<FilenameDate> {
/// `fs_time`, which is a much better proxy for content age than a fake /// `fs_time`, which is a much better proxy for content age than a fake
/// epoch date. /// epoch date.
fn plausible_filename_date(dt: DateTime<Utc>) -> Option<DateTime<Utc>> { fn plausible_filename_date(dt: DateTime<Utc>) -> Option<DateTime<Utc>> {
use chrono::Datelike; plausible_filename_year(dt.year()).then_some(dt)
let year = dt.year(); }
// 1995 predates digital photography for most users; allowing one year
// past `now` covers clock-skew on freshly-taken shots without letting /// The year window both filename branches (wall clock and unix timestamp)
// 2037 timestamps through. /// accept. 1995 predates digital photography for most users; allowing one
let max_year = Utc::now().year() + 1; /// year past `now` covers clock-skew on freshly-taken shots without letting
if (1995..=max_year).contains(&year) { /// 2037 timestamps through.
Some(dt) fn plausible_filename_year(year: i32) -> bool {
} else { (1995..=Utc::now().year() + 1).contains(&year)
None
}
} }
/// Default lookback for `/memories`. The original 15-year cap pre-dated /// Default lookback for `/memories`. The original 15-year cap pre-dated
@@ -994,6 +999,76 @@ mod tests {
} }
} }
#[test]
fn test_wall_patterns_reject_implausible_years() {
// One per wall pattern, too far out and too far back (#160).
let cases: &[&str] = &[
"Screenshot_2229-05-24-08-05-51.png",
"Screenshot_1053-05-19-10-15-27.png",
"VID_22290524_080551_818.mp4",
"IMG_10530519-101527.jpg",
"2229-05-24_08-05-51.jpg",
"1053-05-19_10-15-27.jpg",
"2229-05-24-080551.jpg",
"1053-05-19-101527.jpg",
"22290524080551.jpg",
"10530519101527.jpg",
];
// A miss may still fall through to a later pattern (a 14-digit run
// can read as a unix timestamp); it just can't be a wall date.
for name in cases {
if let Some(m) = match_filename_date(name) {
assert!(
!matches!(m.date, FilenameDate::Wall(_)),
"{} matched wall pattern {}",
name,
m.pattern
);
}
}
assert_eq!(
match_filename_date("Screenshot_2229-05-24-08-05-51.png"),
None
);
}
#[test]
fn test_wall_patterns_reject_prod_counter_and_id_names() {
// Shapes from the prod audit on #160: an app's VID_<counter> and
// Facebook download IDs, both caught by screenshot_compact.
for name in [
"VID_20950505_044049.mp4",
"VID_185800302_235056_842.mp4",
"VID_121920715_134958_293.mp4",
"10530519_10152700326093436_5479865808523667420_o.jpg",
"10460624_10202578362324975_1067700837005629791_o.jpg",
] {
assert_eq!(match_filename_date(name), None, "{} should not match", name);
}
}
#[test]
fn test_wall_pattern_miss_falls_through_to_next_pattern() {
// screenshot_compact sees year 2229 first; compact still finds 2014.
let m =
match_filename_date("VID_22290524_080551_20140927101712.jpg").expect("compact match");
assert_eq!(m.pattern, "compact");
let FilenameDate::Wall(naive) = m.date else {
panic!("compact is a wall pattern");
};
assert_eq!(naive.year(), 2014);
}
#[test]
fn test_wall_patterns_keep_plausible_edges() {
assert!(match_filename_date("Screenshot_1995-01-01-00-00-00.png").is_some());
let next_year = Utc::now().year() + 1;
let name = format!("Screenshot_{next_year}-01-01-00-00-00.png");
assert!(match_filename_date(&name).is_some());
let name = format!("Screenshot_{}-01-01-00-00-00.png", next_year + 1);
assert_eq!(match_filename_date(&name), None);
}
#[test] #[test]
fn test_match_filename_date_out_of_class_is_none() { fn test_match_filename_date_out_of_class_is_none() {
assert_eq!(match_filename_date("IMG_12345.jpg"), None); assert_eq!(match_filename_date("IMG_12345.jpg"), None);