Apps/ImageApi
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Reels: mixed-media (video clip beats) + faster burst fade

Browse Source 
Videos in a span now appear as clip beats: the first few seconds of the
video (capped at CLIP_SECONDS=5, and to the source length) filled to the
portrait canvas like photos, with its live audio ducked under the
narration (amix at 0.35). If the narration outlasts the clip, the last
frame is held (tpad); clips with no audio track just play under narration.

Selection splits the beat budget between photo beats and clip beats —
clips get up to half (≥1 when present), photos the rest — then merges
both back into chronological order. SegmentMedia gains a Clip variant;
beats carry `media` (photos or one clip) and the cache key tags P/C so a
path used as a still vs a clip differ.

Also drops the burst fade from 0.15s to 0.08s so a quick burst reads
clearly differently from a held shot. Bumps RENDER_VERSION.

The clip filtergraph (fill + duck-mix + last-frame hold) is unit-tested
but, like the rest of the ffmpeg path, wants a real render check on the
GPU host.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Cameron Cordes
2026-06-13 00:02:51 -04:00
parent 299e32b014
commit 65793a2dda
4 changed files with 479 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/reels/mod.rs
+114 -47
View File
@@ -36,21 +36,40 @@ use crate::otel::extract_context_from_request;
use crate::state::AppState;
use selector::ReelSelector;
/// The media behind one shot. Photos-only for now; a `Clip` variant (a section
/// of a source video) is the phase-2 extension point.
/// The media behind one shot: a still photo, or a short section of a source
/// video (played with its live audio ducked under the narration). Both carry
/// just the library-relative path; the renderer applies fixed clip framing
/// (start/length) from constants.
#[derive(Debug, Clone)]
pub enum SegmentMedia {
Photo { rel_path: String, library_id: i32 },
Clip { rel_path: String, library_id: i32 },
}
/// A beat: one narration line over one or more photos. A single-photo beat is a
/// held shot; a multi-photo beat is a quick burst that flashes through several
/// moments of the same event while the line is read — so a week/month reel can
/// *show* everything it spans without a narration line (and the seconds that
/// come with it) per photo.
impl SegmentMedia {
fn rel_path(&self) -> &str {
match self {
SegmentMedia::Photo { rel_path, .. } | SegmentMedia::Clip { rel_path, .. } => rel_path,
}
}
fn library_id(&self) -> i32 {
match self {
SegmentMedia::Photo { library_id, .. } | SegmentMedia::Clip { library_id, .. } => {
*library_id
}
}
}
}
/// A beat: one narration line over its media. A photo beat holds one still (a
/// held shot) or several (a quick burst that flashes through moments of an
/// event while the line is read). A clip beat holds a single video clip. Either
/// way one narration line covers the whole beat, so a week/month reel can
/// *show* everything it spans without a narration line — and the seconds that
/// come with it — per item.
#[derive(Debug, Clone)]
pub struct PlannedBeat {
pub photos: Vec<SegmentMedia>,
pub media: Vec<SegmentMedia>,
pub date: Option<i64>,
pub insight_title: Option<String>,
pub insight_summary: Option<String>,
@@ -63,6 +82,11 @@ impl PlannedBeat {
let dt = DateTime::from_timestamp(ts, 0)?;
Some(dt.format("%B %-d, %Y").to_string())
}
/// True when this beat is a single video clip (vs one or more photos).
pub fn is_clip(&self) -> bool {
matches!(self.media.as_slice(), [SegmentMedia::Clip { .. }])
}
}
/// Reel-wide metadata the scripter uses for framing.
@@ -183,7 +207,7 @@ fn finish_job(
/// Render version: bump to invalidate every cached reel after a rendering /
/// scripting change that should produce a fresh result.
const RENDER_VERSION: u32 = 4;
const RENDER_VERSION: u32 = 5;
/// Narration expressiveness — Chatterbox's `exaggeration` knob. A slight bump
/// over the ~0.5 default warms up otherwise-flat narration without over-acting;
@@ -207,12 +231,13 @@ fn cache_key(selector: &ReelSelector, media: &[SegmentMedia], voice: Option<&str
voice.unwrap_or("default")
);
for m in media {
match m {
SegmentMedia::Photo {
rel_path,
library_id,
} => buf.push_str(&format!("{library_id}:{rel_path}|")),
}
// Tag photo vs clip so the same path used as a still and as a video
// clip produce different keys.
let tag = match m {
SegmentMedia::Photo { .. } => 'P',
SegmentMedia::Clip { .. } => 'C',
};
buf.push_str(&format!("{tag}{}:{}|", m.library_id(), m.rel_path()));
}
blake3::hash(buf.as_bytes()).to_hex().to_string()
}
@@ -309,9 +334,9 @@ pub async fn create_reel_handler(
}));
}
// Flatten every photo across beats (in order) into the cache key — the key
// tracks exactly which photos appear and in what sequence.
let media: Vec<SegmentMedia> = planned.iter().flat_map(|b| b.photos.clone()).collect();
// Flatten every media item across beats (in order) into the cache key — the
// key tracks exactly which photos/clips appear and in what sequence.
let media: Vec<SegmentMedia> = planned.iter().flat_map(|b| b.media.clone()).collect();
let voice = req.voice.clone().filter(|s| !s.is_empty());
let key = cache_key(&selector, &media, voice.as_deref());
@@ -462,7 +487,7 @@ async fn run_reel_job(
use anyhow::{Context, anyhow};
let started = Instant::now();
let total_photos: usize = planned.iter().map(|b| b.photos.len()).sum();
let total_photos: usize = planned.iter().map(|b| b.media.len()).sum();
log::info!(
"reel {job_id}: starting — span {:?}, {} beats, {} photos, voice={}",
meta.span,
@@ -510,15 +535,15 @@ async fn run_reel_job(
let beat_total = planned.len();
let mut beat_files: Vec<String> = Vec::new();
for (i, (beat, line)) in planned.iter().zip(script.lines.iter()).enumerate() {
// Resolve all of the beat's photos to absolute paths; drop any that
// don't resolve. An empty beat is skipped.
let image_paths: Vec<PathBuf> = beat
.photos
// Resolve the beat's media to absolute paths; drop any that don't
// resolve. An empty beat is skipped.
let paths: Vec<PathBuf> = beat
.media
.iter()
.filter_map(|m| resolve_image_path(app_state, m))
.filter_map(|m| resolve_media_path(app_state, m))
.collect();
if image_paths.is_empty() {
log::warn!("reel {job_id}: skipping beat {i}, no image paths resolved");
if paths.is_empty() {
log::warn!("reel {job_id}: skipping beat {i}, no media paths resolved");
continue;
}
@@ -551,17 +576,26 @@ async fn run_reel_job(
.unwrap_or(render::MIN_SEGMENT_SECONDS);
set_stage(job_id, "rendering");
log::info!(
"reel {job_id}: beat {}/{} — {} photo(s), narration {:.1}s",
i + 1,
beat_total,
image_paths.len(),
narration_secs
);
let beat_out = work.path().join(format!("beat_{i:03}.mp4"));
if let Err(e) =
render::render_beat(&image_paths, &audio_path, &beat_out, narration_secs, &opts).await
{
let render_result = if beat.is_clip() {
log::info!(
"reel {job_id}: beat {}/{} — video clip, narration {:.1}s",
i + 1,
beat_total,
narration_secs
);
render::render_clip_beat(&paths[0], &audio_path, &beat_out, narration_secs, &opts).await
} else {
log::info!(
"reel {job_id}: beat {}/{} — {} photo(s), narration {:.1}s",
i + 1,
beat_total,
paths.len(),
narration_secs
);
render::render_beat(&paths, &audio_path, &beat_out, narration_secs, &opts).await
};
if let Err(e) = render_result {
log::warn!("reel {job_id}: skipping beat {i}, render failed: {e}");
continue;
}
@@ -603,15 +637,12 @@ async fn run_reel_job(
Ok((script.title, final_path))
}
/// Resolve a photo segment's library-relative path to a validated absolute
/// path under its library root.
fn resolve_image_path(app_state: &AppState, media: &SegmentMedia) -> Option<PathBuf> {
let SegmentMedia::Photo {
rel_path,
library_id,
} = media;
let lib = app_state.library_by_id(*library_id)?;
crate::files::is_valid_full_path(&lib.root_path, rel_path, false)
/// Resolve a media item's library-relative path to a validated absolute path
/// under its library root (works for both photos and clips).
fn resolve_media_path(app_state: &AppState, media: &SegmentMedia) -> Option<PathBuf> {
let lib = app_state.library_by_id(media.library_id())?;
let rel = media.rel_path().to_string();
crate::files::is_valid_full_path(&lib.root_path, &rel, false)
}
#[cfg(test)]
@@ -625,6 +656,13 @@ mod tests {
}
}
fn clip(p: &str, lib: i32) -> SegmentMedia {
SegmentMedia::Clip {
rel_path: p.to_string(),
library_id: lib,
}
}
fn day_selector() -> ReelSelector {
ReelSelector::Memories {
span: MemoriesSpan::Day,
@@ -668,6 +706,35 @@ mod tests {
assert_ne!(base, cache_key(&week, &media, Some("grandma")));
}
#[test]
fn cache_key_distinguishes_photo_from_clip() {
// Same path/library used as a still vs a video clip must differ.
let as_photo = vec![photo("v.mp4", 1)];
let as_clip = vec![clip("v.mp4", 1)];
assert_ne!(
cache_key(&day_selector(), &as_photo, None),
cache_key(&day_selector(), &as_clip, None)
);
}
#[test]
fn is_clip_only_for_single_clip_beat() {
let clip_beat = PlannedBeat {
media: vec![clip("v.mp4", 1)],
date: None,
insight_title: None,
insight_summary: None,
};
let photo_beat = PlannedBeat {
media: vec![photo("a.jpg", 1), photo("b.jpg", 1)],
date: None,
insight_title: None,
insight_summary: None,
};
assert!(clip_beat.is_clip());
assert!(!photo_beat.is_clip());
}
#[test]
fn span_phrase_maps_each_span() {
let mk = |span| ReelMeta {
@@ -682,7 +749,7 @@ mod tests {
#[test]
fn date_label_formats_or_none() {
let beat = PlannedBeat {
photos: vec![photo("a.jpg", 1)],
media: vec![photo("a.jpg", 1)],
date: Some(1_560_384_000), // 2019-06-13 UTC
insight_title: None,
insight_summary: None,
@@ -690,7 +757,7 @@ mod tests {
assert!(beat.date_label().unwrap().contains("2019"));
let undated = PlannedBeat {
photos: vec![photo("a.jpg", 1)],
media: vec![photo("a.jpg", 1)],
date: None,
insight_title: None,
insight_summary: None,