fix(clip): stop re-sending undecodable files every pass
A 422 from Apollo's /encode_image (bytes it can't decode — corrupt files, non-images with an image extension) left clip_embedding NULL, so the drain re-sent the file and logged the same "permanent failure (will retry next pass)" on every scan. Permanent rejections now record a clip_failures marker scoped to the row's content_hash; list_clip_unencoded_candidates skips the row while the hash still matches, so a replaced file gets another try with no cleanup. Only 422 (new ClipError::Rejected) marks: other 4xx are deployment problems (auth, routing) and would otherwise blacklist the whole library. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a605ccd6f8
commit
e75e60786d
9 files changed
+179
-25
No files matched your search
+19
-9
@@ -27,8 +27,10 @@
|
||||
//! L2-normalized so the rerank reduces to a plain dot product.
|
||||
//!
|
||||
//! Error mapping (reflected in [`ClipError`]):
|
||||
//! - 422 `decode_failed` / `empty_text` → permanent: ImageApi marks the
|
||||
//! row failed or surfaces the empty-query error to the search caller.
|
||||
//! - 422 `decode_failed` / `empty_text` → rejected input: ImageApi marks
|
||||
//! the row failed or surfaces the empty-query error to the search caller.
|
||||
//! - Any other 4xx → permanent, but not the input's fault (auth, routing),
|
||||
//! so nothing is marked.
|
||||
//! - 503 `cuda_oom` / `engine_unavailable` → defer-and-retry: no marker.
|
||||
//! - Any other 5xx / network error → defer.
|
||||
|
||||
@@ -91,8 +93,12 @@ pub struct ClipHealth {
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum ClipError {
|
||||
/// Apollo refused for a reason that won't change on retry (decode
|
||||
/// failure on /encode_image, empty text on /encode_text).
|
||||
/// Apollo refused the input itself (422: decode failure on
|
||||
/// /encode_image, empty text on /encode_text). Retrying the same
|
||||
/// bytes will fail the same way.
|
||||
Rejected(anyhow::Error),
|
||||
/// Any other 4xx: won't fix itself on retry, but it's a deployment
|
||||
/// problem (auth, routing), not this input's.
|
||||
Permanent(anyhow::Error),
|
||||
/// Apollo couldn't process this turn but might next time (CUDA OOM,
|
||||
/// engine not loaded, network hiccup).
|
||||
@@ -105,6 +111,7 @@ pub enum ClipError {
|
||||
impl std::fmt::Display for ClipError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
ClipError::Rejected(e) => write!(f, "rejected: {e}"),
|
||||
ClipError::Permanent(e) => write!(f, "permanent: {e}"),
|
||||
ClipError::Transient(e) => write!(f, "transient: {e}"),
|
||||
ClipError::Disabled => write!(f, "clip client disabled"),
|
||||
@@ -278,7 +285,7 @@ fn classify_error_response(status: u16, body_text: &str) -> ClipError {
|
||||
.unwrap_or_default();
|
||||
|
||||
if status == 422 {
|
||||
return ClipError::Permanent(anyhow::anyhow!(
|
||||
return ClipError::Rejected(anyhow::anyhow!(
|
||||
"clip {} {}: {}",
|
||||
status,
|
||||
detail_code,
|
||||
@@ -323,6 +330,9 @@ fn classify_error_response(status: u16, body_text: &str) -> ClipError {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn is_rejected(e: &ClipError) -> bool {
|
||||
matches!(e, ClipError::Rejected(_))
|
||||
}
|
||||
fn is_permanent(e: &ClipError) -> bool {
|
||||
matches!(e, ClipError::Permanent(_))
|
||||
}
|
||||
@@ -331,16 +341,16 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_422_decode_failed_is_permanent() {
|
||||
assert!(is_permanent(&classify_error_response(
|
||||
fn classify_422_decode_failed_is_rejected() {
|
||||
assert!(is_rejected(&classify_error_response(
|
||||
422,
|
||||
r#"{"detail":"decode_failed: bad bytes"}"#
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_422_empty_text_is_permanent() {
|
||||
assert!(is_permanent(&classify_error_response(
|
||||
fn classify_422_empty_text_is_rejected() {
|
||||
assert!(is_rejected(&classify_error_response(
|
||||
422,
|
||||
r#"{"detail":"empty_text"}"#
|
||||
)));
|
||||
|
||||
Reference in new issue
Block a user