Retry insight persistence with exponential backoff to avoid SQLite lock contention

Add retry_with_backoff utility (100ms base, ±25% jitter, 3 attempts)
and replace all 11 direct DAO calls in insight_chat.rs and
insight_generator.rs. The Mutex guard drops between retries so the
maintenance connection can release its SQLite write lock.
This commit is contained in:
Cameron Cordes
2026-08-05 18:22:36 -04:00
parent 4029625dc0
commit 38aaddfd2a
4 changed files with 133 additions and 58 deletions
+55 -1
View File
@@ -1,4 +1,58 @@
use std::time::SystemTime;
use rand::Rng;
use std::time::{Duration, SystemTime};
/// Retry a fallible operation with exponential backoff + jitter.
///
/// Runs the closure immediately, then retries up to `max_retries` times on
/// failure. Each retry sleeps for `base_delay * 2^attempt` plus a uniform
/// jitter of ±25%. Non-final errors are logged at `debug` with the label;
/// the final error is logged at `error`.
///
/// # Examples
/// ```
/// use image_api::utils::retry_with_backoff;
///
/// let result = retry_with_backoff("my-op", 3, || {
/// // something that may fail transiently
/// Ok::<_, anyhow::Error>(42)
/// });
/// ```
pub fn retry_with_backoff<F, T, E>(label: &str, max_retries: u32, mut op: F) -> Result<T, E>
where
F: FnMut() -> Result<T, E>,
E: std::fmt::Debug,
{
let mut last_err = match op() {
Ok(v) => return Ok(v),
Err(e) => e,
};
for attempt in 1..=max_retries {
let base = Duration::from_millis(100).saturating_mul(1_u32.pow(attempt - 1));
let jitter_range = base.as_millis() as f64 * 0.25;
let jitter = rand::thread_rng().gen_range(-jitter_range..=jitter_range) as u128;
let delay = base.as_millis() as i128 + jitter as i128;
std::thread::sleep(Duration::from_millis(delay.max(0) as u64));
log::debug!(
"{}: attempt {}/{} failed ({:?}), retrying in {}ms",
label,
attempt,
max_retries,
last_err,
delay.max(0)
);
match op() {
Ok(v) => return Ok(v),
Err(e) => last_err = e,
}
}
log::error!(
"{}: all {} retries exhausted: {:?}",
label,
max_retries,
last_err
);
Err(last_err)
}
/// Normalize a file path to use forward slashes for cross-platform consistency
/// This ensures paths stored in the database always use `/` regardless of OS