Retry insight persistence with exponential backoff to avoid SQLite lock contention
Add retry_with_backoff utility (100ms base, ±25% jitter, 3 attempts) and replace all 11 direct DAO calls in insight_chat.rs and insight_generator.rs. The Mutex guard drops between retries so the maintenance connection can release its SQLite write lock.
This commit is contained in:
+55
-1
@@ -1,4 +1,58 @@
|
||||
use std::time::SystemTime;
|
||||
use rand::Rng;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
/// Retry a fallible operation with exponential backoff + jitter.
|
||||
///
|
||||
/// Runs the closure immediately, then retries up to `max_retries` times on
|
||||
/// failure. Each retry sleeps for `base_delay * 2^attempt` plus a uniform
|
||||
/// jitter of ±25%. Non-final errors are logged at `debug` with the label;
|
||||
/// the final error is logged at `error`.
|
||||
///
|
||||
/// # Examples
|
||||
/// ```
|
||||
/// use image_api::utils::retry_with_backoff;
|
||||
///
|
||||
/// let result = retry_with_backoff("my-op", 3, || {
|
||||
/// // something that may fail transiently
|
||||
/// Ok::<_, anyhow::Error>(42)
|
||||
/// });
|
||||
/// ```
|
||||
pub fn retry_with_backoff<F, T, E>(label: &str, max_retries: u32, mut op: F) -> Result<T, E>
|
||||
where
|
||||
F: FnMut() -> Result<T, E>,
|
||||
E: std::fmt::Debug,
|
||||
{
|
||||
let mut last_err = match op() {
|
||||
Ok(v) => return Ok(v),
|
||||
Err(e) => e,
|
||||
};
|
||||
for attempt in 1..=max_retries {
|
||||
let base = Duration::from_millis(100).saturating_mul(1_u32.pow(attempt - 1));
|
||||
let jitter_range = base.as_millis() as f64 * 0.25;
|
||||
let jitter = rand::thread_rng().gen_range(-jitter_range..=jitter_range) as u128;
|
||||
let delay = base.as_millis() as i128 + jitter as i128;
|
||||
std::thread::sleep(Duration::from_millis(delay.max(0) as u64));
|
||||
log::debug!(
|
||||
"{}: attempt {}/{} failed ({:?}), retrying in {}ms",
|
||||
label,
|
||||
attempt,
|
||||
max_retries,
|
||||
last_err,
|
||||
delay.max(0)
|
||||
);
|
||||
match op() {
|
||||
Ok(v) => return Ok(v),
|
||||
Err(e) => last_err = e,
|
||||
}
|
||||
}
|
||||
log::error!(
|
||||
"{}: all {} retries exhausted: {:?}",
|
||||
label,
|
||||
max_retries,
|
||||
last_err
|
||||
);
|
||||
Err(last_err)
|
||||
}
|
||||
|
||||
/// Normalize a file path to use forward slashes for cross-platform consistency
|
||||
/// This ensures paths stored in the database always use `/` regardless of OS
|
||||
|
||||
Reference in New Issue
Block a user