fix(clip): stop re-sending undecodable files every pass
A 422 from Apollo's /encode_image (bytes it can't decode — corrupt files, non-images with an image extension) left clip_embedding NULL, so the drain re-sent the file and logged the same "permanent failure (will retry next pass)" on every scan. Permanent rejections now record a clip_failures marker scoped to the row's content_hash; list_clip_unencoded_candidates skips the row while the hash still matches, so a replaced file gets another try with no cleanup. Only 422 (new ClipError::Rejected) marks: other 4xx are deployment problems (auth, routing) and would otherwise blacklist the whole library. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a605ccd6f8
commit
e75e60786d
9 files changed
+179
-25
No files matched your search
@@ -551,6 +551,17 @@ pub trait ExifDao: Sync + Send {
|
||||
model_version: &str,
|
||||
) -> Result<(), DbError>;
|
||||
|
||||
/// Record that Apollo permanently rejected these bytes, so
|
||||
/// `list_clip_unencoded_candidates` stops offering the row until its
|
||||
/// `content_hash` changes. Upserts — one marker per path.
|
||||
fn mark_clip_failed(
|
||||
&mut self,
|
||||
library_id: i32,
|
||||
rel_path: &str,
|
||||
content_hash: &str,
|
||||
error: &str,
|
||||
) -> Result<(), DbError>;
|
||||
|
||||
/// Load every `(content_hash, clip_embedding)` pair from the live
|
||||
/// image_exif rows for the given libraries, optionally filtered to a
|
||||
/// single `model_version` (cosine sim across mixed geometries is
|
||||
@@ -1519,10 +1530,21 @@ impl ExifDao for SqliteExifDao {
|
||||
// (clip_embedding IS NULL AND content_hash IS NOT NULL)
|
||||
// filter; the planner hits it directly. ORDER BY id ASC
|
||||
// keeps drain progress monotone across ticks.
|
||||
// Skip rows whose current bytes already failed permanently
|
||||
// (`mark_clip_failed`); a changed hash makes them eligible.
|
||||
let failed = schema::clip_failures::table
|
||||
.filter(schema::clip_failures::library_id.eq(library_id))
|
||||
.filter(schema::clip_failures::rel_path.eq(rel_path))
|
||||
.filter(
|
||||
schema::clip_failures::content_hash
|
||||
.nullable()
|
||||
.eq(content_hash),
|
||||
);
|
||||
image_exif
|
||||
.filter(library_id.eq(library_id_val))
|
||||
.filter(clip_embedding.is_null())
|
||||
.filter(content_hash.is_not_null())
|
||||
.filter(diesel::dsl::not(diesel::dsl::exists(failed)))
|
||||
.select((rel_path, content_hash.assume_not_null()))
|
||||
.order(id.asc())
|
||||
.limit(limit)
|
||||
@@ -1585,6 +1607,41 @@ impl ExifDao for SqliteExifDao {
|
||||
})
|
||||
}
|
||||
|
||||
fn mark_clip_failed(
|
||||
&mut self,
|
||||
library_id_val: i32,
|
||||
rel_path_val: &str,
|
||||
content_hash_val: &str,
|
||||
error_val: &str,
|
||||
) -> Result<(), DbError> {
|
||||
trace_db_call("insert", "mark_clip_failed", |_span| {
|
||||
use schema::clip_failures::dsl::*;
|
||||
|
||||
let mut connection = self.connection.lock().expect("Unable to get ExifDao");
|
||||
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
diesel::insert_into(clip_failures)
|
||||
.values((
|
||||
library_id.eq(library_id_val),
|
||||
rel_path.eq(rel_path_val),
|
||||
content_hash.eq(content_hash_val),
|
||||
error.eq(error_val),
|
||||
failed_at.eq(now),
|
||||
))
|
||||
.on_conflict((library_id, rel_path))
|
||||
.do_update()
|
||||
.set((
|
||||
content_hash.eq(content_hash_val),
|
||||
error.eq(error_val),
|
||||
failed_at.eq(now),
|
||||
))
|
||||
.execute(connection.deref_mut())
|
||||
.map(|_| ())
|
||||
.map_err(|e| anyhow::anyhow!("Insert error: {}", e))
|
||||
})
|
||||
.map_err(|e| DbError::log(DbErrorKind::InsertError, e))
|
||||
}
|
||||
|
||||
fn list_clip_index(
|
||||
&mut self,
|
||||
library_ids_val: &[i32],
|
||||
@@ -2559,6 +2616,37 @@ mod exif_dao_tests {
|
||||
SqliteExifDao::from_connection(conn)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clip_failure_marker_skips_row_until_its_bytes_change() {
|
||||
let mut dao = setup_two_libraries();
|
||||
insert_row(&mut dao, 1, "bad.jpg", None);
|
||||
insert_row(&mut dao, 1, "good.jpg", None);
|
||||
insert_row(&mut dao, 2, "bad.jpg", None);
|
||||
for (lib, rel) in [(1, "bad.jpg"), (1, "good.jpg"), (2, "bad.jpg")] {
|
||||
dao.backfill_content_hash(lib, rel, "h1", 1).unwrap();
|
||||
}
|
||||
let rels = |dao: &mut SqliteExifDao, lib| -> Vec<String> {
|
||||
dao.list_clip_unencoded_candidates(lib, 10)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|(rel, _)| rel)
|
||||
.collect()
|
||||
};
|
||||
|
||||
dao.mark_clip_failed(1, "bad.jpg", "h1", "decode_failed")
|
||||
.unwrap();
|
||||
assert_eq!(rels(&mut dao, 1), vec!["good.jpg"]);
|
||||
// Same path in another library is a different file.
|
||||
assert_eq!(rels(&mut dao, 2), vec!["bad.jpg"]);
|
||||
// Re-marking upserts rather than erroring on the primary key.
|
||||
dao.mark_clip_failed(1, "bad.jpg", "h1", "decode_failed")
|
||||
.unwrap();
|
||||
|
||||
// The file was replaced: new bytes get another try.
|
||||
dao.backfill_content_hash(1, "bad.jpg", "h2", 1).unwrap();
|
||||
assert_eq!(rels(&mut dao, 1), vec!["bad.jpg", "good.jpg"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_all_with_date_taken_union_returns_all_libraries() {
|
||||
let mut dao = setup_two_libraries();
|
||||
|
||||
Reference in new issue
Block a user