fix(backfill): convert exiftool-legacy rows whose only dates are zero
backfill_utc_dates refused any fs_time re-resolution of a tag-dated row, assuming exiftool had failed on the file. Files whose date tags are all 0000:00:00 (e.g. uv-prod-long.mp4) were read fine and simply have no date, so they sat unconverted and re-ran through exiftool every tick. resolve_dates_batch_report now also returns the paths exiftool read without error (-Error surfaces ExifTool:Error in the JSON). The drain lets fs_time stand for an exiftool-legacy row only when exiftool read the file; read errors / missing entries and kamadak exif-legacy rows keep refusing. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
783fcb8dbe
commit
55d2e888d6
3 files changed
+222
-18
No files matched your search
+75
-7
@@ -48,7 +48,7 @@
|
||||
//! In-class files invert that rule deliberately (ADR-0001): a renamed
|
||||
//! screenshot or a chat-export timestamp is the only honest date left.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, Stdio};
|
||||
@@ -259,7 +259,30 @@ pub fn resolve_dates_batch(
|
||||
paths: &[PathBuf],
|
||||
prior_exif_priors: &HashMap<PathBuf, ExifPrior>,
|
||||
) -> HashMap<PathBuf, ResolvedDate> {
|
||||
resolve_dates_batch_report(paths, prior_exif_priors).resolved
|
||||
}
|
||||
|
||||
/// [`resolve_dates_batch`]'s result plus which files exiftool actually read.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct BatchResolution {
|
||||
pub resolved: HashMap<PathBuf, ResolvedDate>,
|
||||
/// Paths exiftool returned an error-free entry for, whether or not it
|
||||
/// held a usable date. A path in here that still resolved to `fs_time`
|
||||
/// genuinely has no date tag (e.g. a muxer that wrote all-zero
|
||||
/// `CreateDate`s); a path missing from it was never read — exiftool
|
||||
/// absent, crashed, or reported an error for that file.
|
||||
pub exiftool_read: HashSet<PathBuf>,
|
||||
}
|
||||
|
||||
/// [`resolve_dates_batch`], also reporting which files exiftool read, so
|
||||
/// a caller can tell "no date tag in the file" from "exiftool didn't get
|
||||
/// a look at it" when the waterfall lands on `fs_time`.
|
||||
pub fn resolve_dates_batch_report(
|
||||
paths: &[PathBuf],
|
||||
prior_exif_priors: &HashMap<PathBuf, ExifPrior>,
|
||||
) -> BatchResolution {
|
||||
let mut out: HashMap<PathBuf, ResolvedDate> = HashMap::new();
|
||||
let mut exiftool_read: HashSet<PathBuf> = HashSet::new();
|
||||
let mut needs_exiftool: Vec<&Path> = Vec::with_capacity(paths.len());
|
||||
|
||||
for path in paths {
|
||||
@@ -282,7 +305,10 @@ pub fn resolve_dates_batch(
|
||||
// One subprocess for the whole batch; each entry is already fully
|
||||
// resolved (legacy + new-contract fields).
|
||||
for (path, resolved) in exiftool_dates_batch(&needs_exiftool) {
|
||||
out.insert(path, resolved);
|
||||
if let Some(resolved) = resolved {
|
||||
out.insert(path.clone(), resolved);
|
||||
}
|
||||
exiftool_read.insert(path);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -310,7 +336,10 @@ pub fn resolve_dates_batch(
|
||||
}
|
||||
}
|
||||
|
||||
out
|
||||
BatchResolution {
|
||||
resolved: out,
|
||||
exiftool_read,
|
||||
}
|
||||
}
|
||||
|
||||
/// Tag priority for exiftool extraction. First parseable value wins.
|
||||
@@ -384,9 +413,12 @@ pub fn exiftool_available() -> bool {
|
||||
/// Append the exiftool arguments shared by the single-file and batch
|
||||
/// invocations: raw date strings (no `-d %s` — the resolver interprets
|
||||
/// them itself) with group prefixes for classification, plus the offset
|
||||
/// / GPS evidence tags for the cascade.
|
||||
/// / GPS evidence tags for the cascade. `-Error` puts exiftool's per-file
|
||||
/// read error (`ExifTool:Error`, e.g. "File format error") into the JSON
|
||||
/// — under `-q` it otherwise only reaches stderr, and the file's entry
|
||||
/// looks like a clean read that found no tags.
|
||||
fn exiftool_common_args(cmd: &mut Command) {
|
||||
cmd.arg("-j").arg("-q").arg("-G1");
|
||||
cmd.arg("-j").arg("-q").arg("-G1").arg("-Error");
|
||||
for tag in EXIFTOOL_DATE_TAGS {
|
||||
cmd.arg(format!("-{}", tag));
|
||||
}
|
||||
@@ -430,7 +462,12 @@ fn exiftool_date_single(path: &Path) -> Option<ResolvedDate> {
|
||||
/// libraries with very long path components. Resolution is the same pure
|
||||
/// function as the single-file path, so the drain and ingest produce
|
||||
/// identical results for the same file.
|
||||
fn exiftool_dates_batch(paths: &[&Path]) -> HashMap<PathBuf, ResolvedDate> {
|
||||
///
|
||||
/// Keyed by every file exiftool read without error, `None` when the read
|
||||
/// found no usable date. Files it never reported (crash, partial output)
|
||||
/// or reported an error for are absent, unless the entry still carried a
|
||||
/// usable date.
|
||||
fn exiftool_dates_batch(paths: &[&Path]) -> HashMap<PathBuf, Option<ResolvedDate>> {
|
||||
let mut out = HashMap::new();
|
||||
if paths.is_empty() || !exiftool_available() {
|
||||
return out;
|
||||
@@ -481,7 +518,8 @@ fn exiftool_dates_batch(paths: &[&Path]) -> HashMap<PathBuf, ResolvedDate> {
|
||||
);
|
||||
}
|
||||
for file in parse_exiftool_payload(&output.stdout) {
|
||||
if let Some(resolved) = resolve_exiftool_payload(&file.date_tags, &file.evidence) {
|
||||
let resolved = resolve_exiftool_payload(&file.date_tags, &file.evidence);
|
||||
if resolved.is_some() || !file.read_error {
|
||||
out.insert(PathBuf::from(file.source_file), resolved);
|
||||
}
|
||||
}
|
||||
@@ -810,6 +848,8 @@ struct ExiftoolFileData {
|
||||
source_file: String,
|
||||
date_tags: Vec<RawDateTag>,
|
||||
evidence: ExiftoolEvidence,
|
||||
/// exiftool reported `ExifTool:Error` for this file (couldn't read it).
|
||||
read_error: bool,
|
||||
}
|
||||
|
||||
/// Split the exiftool `-j -G1` output into per-file date tags + evidence.
|
||||
@@ -841,11 +881,16 @@ fn parse_exiftool_payload(stdout: &[u8]) -> Vec<ExiftoolFileData> {
|
||||
source_file: source_file.to_string(),
|
||||
date_tags: Vec::new(),
|
||||
evidence: ExiftoolEvidence::default(),
|
||||
read_error: false,
|
||||
};
|
||||
for (key, value) in obj {
|
||||
let Some((group, tag)) = key.split_once(':') else {
|
||||
continue;
|
||||
};
|
||||
if group == "ExifTool" && tag == "Error" {
|
||||
data.read_error = true;
|
||||
continue;
|
||||
}
|
||||
let Some(v) = value.as_str() else {
|
||||
continue;
|
||||
};
|
||||
@@ -962,6 +1007,29 @@ mod tests {
|
||||
assert!((lon - (-74.006)).abs() < 1e-9);
|
||||
}
|
||||
|
||||
/// A clean read whose date tags are all exiftool's zero placeholder is
|
||||
/// "no usable date", not a read error; `ExifTool:Error` is the error
|
||||
/// flag and isn't mistaken for a date tag.
|
||||
#[test]
|
||||
fn parse_payload_flags_read_errors_not_zero_dates() {
|
||||
let json = br#"[{
|
||||
"SourceFile": "/lib/zero.mp4",
|
||||
"QuickTime:CreateDate": "0000:00:00 00:00:00",
|
||||
"Track1:TrackCreateDate": "0000:00:00 00:00:00"
|
||||
},
|
||||
{
|
||||
"SourceFile": "/lib/bad.jpg",
|
||||
"ExifTool:Error": "File format error"
|
||||
}]"#;
|
||||
let files = parse_exiftool_payload(json);
|
||||
assert_eq!(files.len(), 2);
|
||||
assert!(!files[0].read_error);
|
||||
assert_eq!(files[0].date_tags.len(), 2);
|
||||
assert!(resolve_exiftool_payload(&files[0].date_tags, &files[0].evidence).is_none());
|
||||
assert!(files[1].read_error);
|
||||
assert!(files[1].date_tags.is_empty());
|
||||
}
|
||||
|
||||
/// #163: exiftool's default print conversion renders hemisphere refs as
|
||||
/// words. Before the fix "West" / "South" never matched, so western /
|
||||
/// southern coordinates came out positive and the GPS offset lookup hit
|
||||
|
||||
Reference in new issue
Block a user