fix(clip): stop re-sending undecodable files every pass

A 422 from Apollo's /encode_image (bytes it can't decode — corrupt
files, non-images with an image extension) left clip_embedding NULL,
so the drain re-sent the file and logged the same "permanent failure
(will retry next pass)" on every scan.

Permanent rejections now record a clip_failures marker scoped to the
row's content_hash; list_clip_unencoded_candidates skips the row while
the hash still matches, so a replaced file gets another try with no
cleanup. Only 422 (new ClipError::Rejected) marks: other 4xx are
deployment problems (auth, routing) and would otherwise blacklist the
whole library.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Cameron CordesandClaude Opus 5.5 committed 2026-10-17 00:20:00 -04:00
1 parent a605ccd6f8
commit e75e60786d
9 files changed
+179 -25

No files matched your search

+88
View File
@@ -551,6 +551,17 @@ pub trait ExifDao: Sync + Send {
model_version: &str,
) -> Result<(), DbError>;
/// Record that Apollo permanently rejected these bytes, so
/// `list_clip_unencoded_candidates` stops offering the row until its
/// `content_hash` changes. Upserts — one marker per path.
fn mark_clip_failed(
&mut self,
library_id: i32,
rel_path: &str,
content_hash: &str,
error: &str,
) -> Result<(), DbError>;
/// Load every `(content_hash, clip_embedding)` pair from the live
/// image_exif rows for the given libraries, optionally filtered to a
/// single `model_version` (cosine sim across mixed geometries is
@@ -1519,10 +1530,21 @@ impl ExifDao for SqliteExifDao {
// (clip_embedding IS NULL AND content_hash IS NOT NULL)
// filter; the planner hits it directly. ORDER BY id ASC
// keeps drain progress monotone across ticks.
// Skip rows whose current bytes already failed permanently
// (`mark_clip_failed`); a changed hash makes them eligible.
let failed = schema::clip_failures::table
.filter(schema::clip_failures::library_id.eq(library_id))
.filter(schema::clip_failures::rel_path.eq(rel_path))
.filter(
schema::clip_failures::content_hash
.nullable()
.eq(content_hash),
);
image_exif
.filter(library_id.eq(library_id_val))
.filter(clip_embedding.is_null())
.filter(content_hash.is_not_null())
.filter(diesel::dsl::not(diesel::dsl::exists(failed)))
.select((rel_path, content_hash.assume_not_null()))
.order(id.asc())
.limit(limit)
@@ -1585,6 +1607,41 @@ impl ExifDao for SqliteExifDao {
})
}
fn mark_clip_failed(
&mut self,
library_id_val: i32,
rel_path_val: &str,
content_hash_val: &str,
error_val: &str,
) -> Result<(), DbError> {
trace_db_call("insert", "mark_clip_failed", |_span| {
use schema::clip_failures::dsl::*;
let mut connection = self.connection.lock().expect("Unable to get ExifDao");
let now = chrono::Utc::now().timestamp();
diesel::insert_into(clip_failures)
.values((
library_id.eq(library_id_val),
rel_path.eq(rel_path_val),
content_hash.eq(content_hash_val),
error.eq(error_val),
failed_at.eq(now),
))
.on_conflict((library_id, rel_path))
.do_update()
.set((
content_hash.eq(content_hash_val),
error.eq(error_val),
failed_at.eq(now),
))
.execute(connection.deref_mut())
.map(|_| ())
.map_err(|e| anyhow::anyhow!("Insert error: {}", e))
})
.map_err(|e| DbError::log(DbErrorKind::InsertError, e))
}
fn list_clip_index(
&mut self,
library_ids_val: &[i32],
@@ -2559,6 +2616,37 @@ mod exif_dao_tests {
SqliteExifDao::from_connection(conn)
}
#[test]
fn clip_failure_marker_skips_row_until_its_bytes_change() {
let mut dao = setup_two_libraries();
insert_row(&mut dao, 1, "bad.jpg", None);
insert_row(&mut dao, 1, "good.jpg", None);
insert_row(&mut dao, 2, "bad.jpg", None);
for (lib, rel) in [(1, "bad.jpg"), (1, "good.jpg"), (2, "bad.jpg")] {
dao.backfill_content_hash(lib, rel, "h1", 1).unwrap();
}
let rels = |dao: &mut SqliteExifDao, lib| -> Vec<String> {
dao.list_clip_unencoded_candidates(lib, 10)
.unwrap()
.into_iter()
.map(|(rel, _)| rel)
.collect()
};
dao.mark_clip_failed(1, "bad.jpg", "h1", "decode_failed")
.unwrap();
assert_eq!(rels(&mut dao, 1), vec!["good.jpg"]);
// Same path in another library is a different file.
assert_eq!(rels(&mut dao, 2), vec!["bad.jpg"]);
// Re-marking upserts rather than erroring on the primary key.
dao.mark_clip_failed(1, "bad.jpg", "h1", "decode_failed")
.unwrap();
// The file was replaced: new bytes get another try.
dao.backfill_content_hash(1, "bad.jpg", "h2", 1).unwrap();
assert_eq!(rels(&mut dao, 1), vec!["bad.jpg", "good.jpg"]);
}
#[test]
fn get_all_with_date_taken_union_returns_all_libraries() {
let mut dao = setup_two_libraries();