Add memory-reel backend: on-demand narrated photo slideshow
New POST /reels + GET /reels/{id} (+ /video) build an MP4 slideshow of a
memory span (day/week/month), narrated by the LLM in a cloned voice.
Pipeline (src/reels/): a selector resolves which photos + reel metadata,
the scripter writes one narration line per photo via a single LLM call
(reusing each photo's cached insight as context — no fresh vision calls,
so reel generation stays off the GPU's vision slot), each line is
synthesized to speech, and the renderer assembles stills + narration via
ffmpeg. Jobs run in the background (mirroring the TTS speech-job
registry) since a reel takes minutes; the finished MP4 is cached on disk
keyed by the selection so a repeat request is instant.
The segment model is media-typed (Photo today) so a video-clip segment
(phase 2) and a nightly pre-render (phase 3) slot in without reworking
the pipeline. Ken Burns motion is implemented but defaulted off pending a
visual check on the GPU box.
Supporting changes:
- memories: extract gather_memory_items() so the reel selector reuses the
exact window/exclusion/tz/sort logic behind /memories.
- ai::tts: add synthesize_serialized() so reel narration honors the same
single-GPU permit + write lease as user TTS requests.
- video::ffmpeg: make get_duration_seconds() pub for narration timing.
- AppState: reels_path (REELS_DIRECTORY, defaults beside preview clips).
Pure logic (cache key, script parsing, ffmpeg arg/filter construction,
even sampling, segment timing) is unit-tested (26 tests). The runtime
path (ffmpeg render, TTS, LLM) needs a real run on the GPU host to verify
end-to-end — not exercisable in CI.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
98274c3301
commit
e3f731b3b2
9 files changed
+1615
-30
No files matched your search
@@ -0,0 +1,252 @@
|
||||
//! Reel selectors: resolve "what goes in the reel" into an ordered media set
|
||||
//! plus the metadata the scripter needs. The renderer and scripter are
|
||||
//! selector-agnostic, so adding tag- or date-range-based reels later means
|
||||
//! adding a variant here, not touching the pipeline.
|
||||
//!
|
||||
//! Resolution is split in two so the handler can compute a cache key (and
|
||||
//! short-circuit on a cache hit) without the per-photo insight lookups:
|
||||
//! [`resolve`] is the cheap media-set pass; [`enrich`] adds cached insights and
|
||||
//! runs in the background job.
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use chrono::{DateTime, Datelike, FixedOffset};
|
||||
|
||||
use super::{PlannedSegment, ReelMeta, SegmentMedia};
|
||||
use crate::database::{ExifDao, InsightDao};
|
||||
use crate::file_types::is_image_file;
|
||||
use crate::memories::{self, MemoriesSpan};
|
||||
use crate::state::AppState;
|
||||
|
||||
/// Default and hard caps on how many photos a reel covers. The cap bounds the
|
||||
/// LLM/TTS/ffmpeg work per reel; when a span has more, [`sample_evenly`] keeps
|
||||
/// a representative spread across the years rather than just the oldest.
|
||||
pub const DEFAULT_MAX_SEGMENTS: usize = 24;
|
||||
pub const HARD_MAX_SEGMENTS: usize = 40;
|
||||
|
||||
/// What a reel is built from. v1 ships the memories (on this day/week/month)
|
||||
/// selector; tag and date-range variants slot in here later.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum ReelSelector {
|
||||
Memories {
|
||||
span: MemoriesSpan,
|
||||
tz_offset_minutes: i32,
|
||||
library: Option<String>,
|
||||
max_segments: usize,
|
||||
},
|
||||
}
|
||||
|
||||
impl ReelSelector {
|
||||
/// Stable string identity for the cache key. Captures everything that
|
||||
/// changes *which* media is selected (but not the non-deterministic
|
||||
/// narration, which can't be part of a pre-render key).
|
||||
pub fn descriptor(&self) -> String {
|
||||
match self {
|
||||
ReelSelector::Memories {
|
||||
span,
|
||||
tz_offset_minutes,
|
||||
library,
|
||||
max_segments,
|
||||
} => format!(
|
||||
"memories:span={:?}:tz={}:lib={}:max={}",
|
||||
span,
|
||||
tz_offset_minutes,
|
||||
library.as_deref().unwrap_or("all"),
|
||||
max_segments
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Pick at most `max` items spread evenly across the input, always keeping the
|
||||
/// first and last. Returns the input unchanged when it already fits.
|
||||
pub fn sample_evenly<T: Clone>(items: &[T], max: usize) -> Vec<T> {
|
||||
if max == 0 {
|
||||
return Vec::new();
|
||||
}
|
||||
if items.len() <= max {
|
||||
return items.to_vec();
|
||||
}
|
||||
if max == 1 {
|
||||
return vec![items[0].clone()];
|
||||
}
|
||||
let last = items.len() - 1;
|
||||
(0..max)
|
||||
.map(|i| {
|
||||
// Spread indices 0..=last across max picks, endpoints included.
|
||||
let idx = (i * last + (max - 1) / 2) / (max - 1);
|
||||
items[idx.min(last)].clone()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Cheap pass: resolve the selector into an ordered list of media (no insight
|
||||
/// lookups yet) plus reel metadata. `Err` only on an invalid library param.
|
||||
pub fn resolve(
|
||||
app_state: &AppState,
|
||||
exif_dao: &Mutex<Box<dyn ExifDao>>,
|
||||
span_context: &opentelemetry::Context,
|
||||
selector: &ReelSelector,
|
||||
) -> Result<(Vec<PlannedSegment>, ReelMeta), String> {
|
||||
match selector {
|
||||
ReelSelector::Memories {
|
||||
span,
|
||||
tz_offset_minutes,
|
||||
library,
|
||||
max_segments,
|
||||
} => {
|
||||
let client_tz = FixedOffset::east_opt(tz_offset_minutes * 60);
|
||||
let items = memories::gather_memory_items(
|
||||
app_state,
|
||||
exif_dao,
|
||||
span_context,
|
||||
*span,
|
||||
*tz_offset_minutes,
|
||||
client_tz,
|
||||
library.as_deref(),
|
||||
)?;
|
||||
|
||||
// Phase 1 is photos-only: drop videos (a clip segment type lands
|
||||
// in phase 2). Filter before sampling so the spread is over the
|
||||
// photos that will actually appear.
|
||||
let items: Vec<memories::MemoryItem> = items
|
||||
.into_iter()
|
||||
.filter(|it| is_image_file(Path::new(&it.path)))
|
||||
.collect();
|
||||
|
||||
let cap = (*max_segments).clamp(1, HARD_MAX_SEGMENTS);
|
||||
let items = sample_evenly(&items, cap);
|
||||
|
||||
let years = distinct_years(&items, client_tz);
|
||||
let meta = ReelMeta { span: *span, years };
|
||||
|
||||
let planned = items
|
||||
.into_iter()
|
||||
.map(|it| PlannedSegment {
|
||||
media: SegmentMedia::Photo {
|
||||
rel_path: it.path,
|
||||
library_id: it.library_id,
|
||||
},
|
||||
date: it.created,
|
||||
insight_title: None,
|
||||
insight_summary: None,
|
||||
})
|
||||
.collect();
|
||||
Ok((planned, meta))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Distinct calendar years represented by the selected media, in the client's
|
||||
/// timezone, ascending. Used to tell the scripter how far back the reel reaches.
|
||||
fn distinct_years(items: &[memories::MemoryItem], tz: Option<FixedOffset>) -> Vec<i32> {
|
||||
let mut years: Vec<i32> = items
|
||||
.iter()
|
||||
.filter_map(|it| it.created)
|
||||
.filter_map(|ts| DateTime::from_timestamp(ts, 0))
|
||||
.map(|dt| match tz {
|
||||
Some(off) => dt.with_timezone(&off).year(),
|
||||
None => dt.year(),
|
||||
})
|
||||
.collect();
|
||||
years.sort_unstable();
|
||||
years.dedup();
|
||||
years
|
||||
}
|
||||
|
||||
/// Background pass: fill each segment's cached insight (title + summary) where
|
||||
/// one exists. Best-effort — a missing or errored lookup leaves the fields
|
||||
/// `None` and the scripter narrates from the date alone.
|
||||
pub fn enrich(
|
||||
insight_dao: &Mutex<Box<dyn InsightDao>>,
|
||||
span_context: &opentelemetry::Context,
|
||||
planned: &mut [PlannedSegment],
|
||||
) {
|
||||
let Ok(mut dao) = insight_dao.lock() else {
|
||||
return;
|
||||
};
|
||||
for seg in planned.iter_mut() {
|
||||
let rel_path = match &seg.media {
|
||||
SegmentMedia::Photo { rel_path, .. } => rel_path,
|
||||
};
|
||||
if let Ok(Some(insight)) = dao.get_insight(span_context, rel_path) {
|
||||
seg.insight_title = Some(insight.title);
|
||||
seg.insight_summary = Some(insight.summary);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn sample_evenly_returns_all_when_under_cap() {
|
||||
let v = vec![1, 2, 3];
|
||||
assert_eq!(sample_evenly(&v, 5), vec![1, 2, 3]);
|
||||
assert_eq!(sample_evenly(&v, 3), vec![1, 2, 3]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sample_evenly_keeps_endpoints_and_spreads() {
|
||||
let v: Vec<i32> = (0..100).collect();
|
||||
let picked = sample_evenly(&v, 5);
|
||||
assert_eq!(picked.len(), 5);
|
||||
assert_eq!(picked[0], 0); // first kept
|
||||
assert_eq!(*picked.last().unwrap(), 99); // last kept
|
||||
// Strictly increasing, no dupes.
|
||||
assert!(picked.windows(2).all(|w| w[0] < w[1]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sample_evenly_handles_one_and_zero() {
|
||||
let v: Vec<i32> = (0..10).collect();
|
||||
assert_eq!(sample_evenly(&v, 1), vec![0]);
|
||||
assert!(sample_evenly(&v, 0).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn descriptor_is_stable_and_distinguishes_inputs() {
|
||||
let a = ReelSelector::Memories {
|
||||
span: MemoriesSpan::Day,
|
||||
tz_offset_minutes: -480,
|
||||
library: None,
|
||||
max_segments: 24,
|
||||
};
|
||||
let b = ReelSelector::Memories {
|
||||
span: MemoriesSpan::Week,
|
||||
tz_offset_minutes: -480,
|
||||
library: None,
|
||||
max_segments: 24,
|
||||
};
|
||||
assert_eq!(a.descriptor(), a.clone().descriptor());
|
||||
assert_ne!(a.descriptor(), b.descriptor());
|
||||
assert!(a.descriptor().contains("lib=all"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn distinct_years_dedupes_and_sorts() {
|
||||
let items = vec![
|
||||
memories::MemoryItem {
|
||||
path: "a".into(),
|
||||
created: Some(1_560_000_000), // 2019
|
||||
modified: None,
|
||||
library_id: 1,
|
||||
},
|
||||
memories::MemoryItem {
|
||||
path: "b".into(),
|
||||
created: Some(1_560_086_400), // 2019
|
||||
modified: None,
|
||||
library_id: 1,
|
||||
},
|
||||
memories::MemoryItem {
|
||||
path: "c".into(),
|
||||
created: Some(1_623_000_000), // 2021
|
||||
modified: None,
|
||||
library_id: 1,
|
||||
},
|
||||
];
|
||||
assert_eq!(distinct_years(&items, None), vec![2019, 2021]);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user