Apps/ImageApi
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Reels: bound disk/ledger growth (pre-gen prune + on-demand cache sweep)

Browse Source 
Nothing reaped reels before, so the on-disk cache and ledger grew
unbounded — each night's daily reel is a new ~4MB file + ledger row that's
stale within ~26h.

- Pre-gen self-prune: after recording a reel, prune_superseded keeps the
  newest PREGEN_KEEP_PER_SCOPE (2) rows per (span, library) and unlinks the
  superseded reels' mp4+sidecar. Caps the ledger/disk at ~spans×libraries×2.
- On-disk sweeper (spawn_reel_cache_sweeper): every 24h, removes reel mp4s
  with no ledger row and no live job older than REEL_CACHE_MAX_AGE_DAYS (7) —
  bounding the on-demand cache, which has no ledger row and otherwise grows
  forever — plus crashed-render cruft (.mp4.tmp/.concat.txt/orphan sidecars).
  Runs regardless of REEL_PREGEN_ENABLED; disable with REEL_CACHE_SWEEP_ENABLED=0.
- New DAO methods prune_superseded + all_cache_keys (with tests); env knobs
  documented in .env.example.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Cameron Cordes
2026-06-13 23:27:32 -04:00
parent 664b3694f8
commit 7e21213181
4 changed files with 296 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/reels/mod.rs
+169 -17
View File
@@ -46,6 +46,21 @@ const REEL_PRECOMPUTED_WEEK_MAX_AGE_HOURS: u64 = 192;
/// Maximum age for a precomputed month reel.
const REEL_PRECOMPUTED_MONTH_MAX_AGE_HOURS: u64 = 768;
/// How many precomputed reels to keep per (span, library). The newest is the
/// one served; one extra is a grace window so a regen mid-flight (or a client
/// that started a fetch just before the swap) isn't left without a file.
const PREGEN_KEEP_PER_SCOPE: usize = 2;
/// On-disk reel cache sweep: an unreferenced reel MP4 older than this is
/// removed. Catches the on-demand cache (which has no ledger row and otherwise
/// grows forever) and any pre-gen orphans. Tunable via `REEL_CACHE_MAX_AGE_DAYS`.
const REEL_CACHE_MAX_AGE_DAYS_DEFAULT: u64 = 7;
/// Interval between on-disk cache sweeps.
const REEL_CACHE_SWEEP_INTERVAL_SECS: u64 = 24 * 3600;
/// Transient render artifacts (`.mp4.tmp`, `.concat.txt`, orphaned sidecars)
/// older than this are leftovers from a crashed render and safe to remove.
const REEL_TMP_MAX_AGE_SECS: u64 = 3600;
/// Resolve a library request parameter to a stable key string.
/// Returns the library's id as a string when found, or `"all"` when
/// the param is absent or the lookup fails.
@@ -1142,28 +1157,165 @@ async fn pregen_one(
)
.await?;
// Record to ledger
let mut reel_dao = app_state.precomputed_reel_dao.lock().expect("lock");
reel_dao.record_reel(
&ctx,
&crate::database::models::InsertablePrecomputedReel {
span: span.to_string(),
library_key: library.to_string(),
cache_key: key.clone(),
output_path: mp4.to_string_lossy().to_string(),
title,
media_count,
render_version: RENDER_VERSION as i32,
tz_offset_minutes: tz,
voice: voice.clone(),
generated_at: now,
},
)?;
// Record to ledger, then retire superseded reels for this (span, library)
// — yesterday's daily, an older render-version, etc. — keeping a small
// grace window. Done under one lock so the prune sees the row we just wrote.
let superseded = {
let mut reel_dao = app_state.precomputed_reel_dao.lock().expect("lock");
reel_dao.record_reel(
&ctx,
&crate::database::models::InsertablePrecomputedReel {
span: span.to_string(),
library_key: library.to_string(),
cache_key: key.clone(),
output_path: mp4.to_string_lossy().to_string(),
title,
media_count,
render_version: RENDER_VERSION as i32,
tz_offset_minutes: tz,
voice: voice.clone(),
generated_at: now,
},
)?;
reel_dao
.prune_superseded(&ctx, span, library, PREGEN_KEEP_PER_SCOPE)
.unwrap_or_default()
};
for row in &superseded {
delete_reel_files(&row.output_path);
}
if !superseded.is_empty() {
log::info!(
"Pruned {} superseded precomputed reel(s) for span={}",
superseded.len(),
span
);
}
log::info!("Precomputed reel generated for span={}, key={}", span, key);
Ok(())
}
// --- On-disk cache sweep -----------------------------------------------------
/// Best-effort unlink of a reel's MP4 and its `.json` sidecar.
fn delete_reel_files(mp4_output_path: &str) {
let mp4 = Path::new(mp4_output_path);
let _ = std::fs::remove_file(mp4);
let _ = std::fs::remove_file(mp4.with_extension("json"));
}
/// Max age (seconds) before an unreferenced reel MP4 is swept.
fn reel_cache_max_age_secs() -> u64 {
std::env::var("REEL_CACHE_MAX_AGE_DAYS")
.ok()
.and_then(|v| v.trim().parse::<u64>().ok())
.filter(|d| *d > 0)
.unwrap_or(REEL_CACHE_MAX_AGE_DAYS_DEFAULT)
* 86_400
}
/// Spawn the periodic on-disk reel-cache sweeper. Runs independently of the
/// pre-gen scheduler because the on-demand cache grows whether or not pre-gen
/// is enabled. Disable with `REEL_CACHE_SWEEP_ENABLED=0`.
pub(crate) async fn spawn_reel_cache_sweeper(app_state: web::Data<AppState>) {
if std::env::var("REEL_CACHE_SWEEP_ENABLED").ok().as_deref() == Some("0") {
log::info!("Reel cache sweeper disabled (REEL_CACHE_SWEEP_ENABLED=0)");
return;
}
tokio::spawn(async move {
// Settle after startup, then sweep on a fixed cadence.
tokio::time::sleep(Duration::from_secs(300)).await;
loop {
let removed = sweep_reel_cache(&app_state);
if removed > 0 {
log::info!("Reel cache sweep removed {removed} stale file(s)");
}
tokio::time::sleep(Duration::from_secs(REEL_CACHE_SWEEP_INTERVAL_SECS)).await;
}
});
}
/// One sweep of `reels_path`. Removes: stale render artifacts (`.mp4.tmp`,
/// `.concat.txt`, orphaned sidecars) from crashed runs; and reel MP4s that no
/// ledger row references, that no live job points at, and that are older than
/// the cache max age (the on-demand cache, which has no ledger row). Returns the
/// number of files removed. Best-effort — any IO error on one entry is skipped.
fn sweep_reel_cache(app_state: &AppState) -> usize {
let dir = Path::new(&app_state.reels_path);
let read_dir = match std::fs::read_dir(dir) {
Ok(rd) => rd,
Err(_) => return 0, // dir not created yet → nothing to sweep
};
// Files a ledger row still points at (current pre-gen reels).
let protected: std::collections::HashSet<String> = {
let ctx = opentelemetry::Context::new();
let mut dao = app_state.precomputed_reel_dao.lock().expect("lock");
dao.all_cache_keys(&ctx)
.unwrap_or_default()
.into_iter()
.collect()
};
// Outputs of live in-memory jobs (a Done reel a client may still be fetching).
let active: std::collections::HashSet<String> = {
let jobs = REEL_JOBS.lock().unwrap();
jobs.values()
.filter_map(|j| j.output_path.as_ref())
.map(|p| p.to_string_lossy().to_string())
.collect()
};
let now = std::time::SystemTime::now();
let max_age = Duration::from_secs(reel_cache_max_age_secs());
let tmp_max_age = Duration::from_secs(REEL_TMP_MAX_AGE_SECS);
let mut removed = 0usize;
for entry in read_dir.flatten() {
let path = entry.path();
let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
continue;
};
let age = entry
.metadata()
.and_then(|m| m.modified())
.ok()
.and_then(|t| now.duration_since(t).ok())
.unwrap_or_default();
// Transient render artifacts from a crashed run.
if name.ends_with(".mp4.tmp") || name.ends_with(".concat.txt") {
if age > tmp_max_age && std::fs::remove_file(&path).is_ok() {
removed += 1;
}
continue;
}
// Reel MP4: keep if referenced (ledger or live job) or still recent.
if let Some(key) = name.strip_suffix(".mp4") {
let p = path.to_string_lossy().to_string();
if protected.contains(key) || active.contains(&p) || age < max_age {
continue;
}
if std::fs::remove_file(&path).is_ok() {
let _ = std::fs::remove_file(path.with_extension("json"));
removed += 1;
}
continue;
}
// Orphaned sidecar (its MP4 is gone).
if name.ends_with(".json")
&& !path.with_extension("mp4").exists()
&& age > tmp_max_age
&& std::fs::remove_file(&path).is_ok()
{
removed += 1;
}
}
removed
}
#[cfg(test)]
mod tests {
use super::*;