hls: add hash-keyed path helpers + VideoToQueue type
Foundation for migrating HLS playlist output from basename-keyed
(`$VIDEO_PATH/{basename}.m3u8`) to content-hash-keyed
(`$VIDEO_PATH/{hash[..2]}/{hash}/playlist.m3u8`). The basename layout
collides whenever two source videos share a filename — common with
iPhone-style sequential naming (`IMG_NNNN.MOV`) across libraries — so
the loser's playlist gets overwritten and ffmpeg keeps re-queueing the
file every scan.
This commit adds the path layout and type plumbing without touching the
actor pipeline, watcher, or HTTP handlers yet:
- `src/video/hls_paths.rs`: `playlist_for_hash`, `sentinel_for_hash`,
`segment_template_for_hash` built on top of `content_hash::hls_dir`,
with constants for the filenames inside the hash dir. Unit tests
cover the sharded layout and the playlist/sentinel/segment paths
all landing in the same directory (so HLS relative refs resolve).
- `src/content_hash::hls_dir` un-deaded — was waiting for this branch.
- `VideoToQueue` struct in `actors.rs`: pairs a source path with its
content hash so callers that lack a hash (rows mid-backfill) skip
the video rather than fabricate one.
- `playlist_file_for` / `playlist_unsupported_sentinel` retained as
migration-only helpers — they're only needed by the one-shot startup
pass that retires pre-content-hash output.
Follow-ups (separate commits on this branch): wire `hls_paths` through
the queue handler + `PlaylistGenerator`, update the watcher's
`process_new_files` to build `VideoToQueue`, switch `/video/generate`
and `/video/stream` to resolve path→hash and return stable URLs, add
the legacy-layout migration, rewrite `cleanup_orphaned_playlists` for
the new dir shape, and surface progress via Prometheus + `/hls/stats`.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -52,12 +52,9 @@ pub fn thumbnail_path(thumbs_dir: &Path, hash: &str) -> PathBuf {
|
||||
|
||||
/// Hash-keyed HLS output directory: `<video_dir>/<hash[..2]>/<hash>/`.
|
||||
/// The playlist lives at `playlist.m3u8` inside this directory and its
|
||||
/// segments are co-located so HLS relative references Just Work.
|
||||
///
|
||||
/// Allow-dead until Branch B/C rewires the HLS pipeline to use it; the
|
||||
/// helper lives here today so Branch A's path layout decisions stay
|
||||
/// adjacent to thumbnail/legacy ones.
|
||||
#[allow(dead_code)]
|
||||
/// segments are co-located so HLS relative references Just Work. See
|
||||
/// [`crate::video::hls_paths`] for the filename constants and the
|
||||
/// per-file helpers built on this dir.
|
||||
pub fn hls_dir(video_dir: &Path, hash: &str) -> PathBuf {
|
||||
let shard = shard_prefix(hash);
|
||||
video_dir.join(shard).join(hash)
|
||||
|
||||
@@ -47,6 +47,19 @@ impl Handler<ProcessMessage> for StreamActor {
|
||||
}
|
||||
}
|
||||
|
||||
/// A video paired with its content hash, ready to be queued for HLS
|
||||
/// playlist generation. Hash is required because all output paths are
|
||||
/// keyed on it; callers that lack a hash (rows mid-backfill) must skip
|
||||
/// the video rather than fabricate one.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct VideoToQueue {
|
||||
pub video_path: PathBuf,
|
||||
pub content_hash: String,
|
||||
}
|
||||
|
||||
/// Legacy basename-keyed playlist path. Used only by the one-shot startup
|
||||
/// migration that retires pre-content-hash output; once cleared, all new
|
||||
/// playlist writes go through [`hls_paths::playlist_for_hash`].
|
||||
pub fn playlist_file_for(playlist_dir: &str, video_path: &Path) -> PathBuf {
|
||||
let filename = video_path
|
||||
.file_name()
|
||||
@@ -55,10 +68,8 @@ pub fn playlist_file_for(playlist_dir: &str, video_path: &Path) -> PathBuf {
|
||||
PathBuf::from(format!("{}/{}.m3u8", playlist_dir, filename))
|
||||
}
|
||||
|
||||
/// Sentinel path written next to a would-be playlist when ffmpeg cannot
|
||||
/// transcode the source (e.g. truncated mp4 with no moov atom). Its presence
|
||||
/// causes future scans to skip the file instead of re-running ffmpeg every
|
||||
/// pass. Delete the `.unsupported` file to force a retry.
|
||||
/// Legacy basename-keyed sentinel path. Same migration-only contract as
|
||||
/// [`playlist_file_for`].
|
||||
pub fn playlist_unsupported_sentinel(playlist_file: &Path) -> PathBuf {
|
||||
let mut s = playlist_file.as_os_str().to_owned();
|
||||
s.push(".unsupported");
|
||||
|
||||
84
src/video/hls_paths.rs
Normal file
84
src/video/hls_paths.rs
Normal file
@@ -0,0 +1,84 @@
|
||||
//! Path layout for hash-keyed HLS output.
|
||||
//!
|
||||
//! Source-of-truth is [`crate::content_hash::hls_dir`], which gives
|
||||
//! `<video_dir>/<hash[..2]>/<hash>/`. The playlist, the per-segment files,
|
||||
//! and the "ffmpeg refused" sentinel all live inside that directory so a
|
||||
//! `.m3u8` written with relative segment references resolves correctly
|
||||
//! at serve time without any URL rewriting.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::content_hash;
|
||||
|
||||
/// Standard filename for the HLS playlist inside a hash dir. Fixed so
|
||||
/// the URL contract is `playlist.m3u8` regardless of the source video's
|
||||
/// original basename.
|
||||
pub const PLAYLIST_FILENAME: &str = "playlist.m3u8";
|
||||
|
||||
/// Sentinel filename written when ffmpeg refused to transcode the
|
||||
/// source. Presence in the hash dir tells future scans to skip the file
|
||||
/// instead of re-running ffmpeg every tick. Delete to force a retry.
|
||||
pub const UNSUPPORTED_SENTINEL_FILENAME: &str = "playlist.unsupported";
|
||||
|
||||
/// Segment-name template passed to ffmpeg via `-hls_segment_filename`.
|
||||
/// Segments live inside the hash dir; the playlist's relative refs
|
||||
/// resolve to siblings automatically.
|
||||
pub const SEGMENT_TEMPLATE: &str = "segment_%03d.ts";
|
||||
|
||||
/// Path to the HLS playlist for a video identified by content hash.
|
||||
pub fn playlist_for_hash(video_dir: &Path, hash: &str) -> PathBuf {
|
||||
content_hash::hls_dir(video_dir, hash).join(PLAYLIST_FILENAME)
|
||||
}
|
||||
|
||||
/// Path to the unsupported-source sentinel for a hash.
|
||||
pub fn sentinel_for_hash(video_dir: &Path, hash: &str) -> PathBuf {
|
||||
content_hash::hls_dir(video_dir, hash).join(UNSUPPORTED_SENTINEL_FILENAME)
|
||||
}
|
||||
|
||||
/// Absolute path used as ffmpeg's `-hls_segment_filename` value.
|
||||
pub fn segment_template_for_hash(video_dir: &Path, hash: &str) -> PathBuf {
|
||||
content_hash::hls_dir(video_dir, hash).join(SEGMENT_TEMPLATE)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn playlist_path_lives_inside_sharded_hash_dir() {
|
||||
let video = Path::new("/var/video");
|
||||
let p = playlist_for_hash(video, "abcdef0123456789");
|
||||
assert_eq!(
|
||||
p,
|
||||
PathBuf::from("/var/video/ab/abcdef0123456789/playlist.m3u8")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sentinel_path_lives_alongside_playlist() {
|
||||
let video = Path::new("/var/video");
|
||||
let s = sentinel_for_hash(video, "abcdef0123456789");
|
||||
assert_eq!(
|
||||
s,
|
||||
PathBuf::from("/var/video/ab/abcdef0123456789/playlist.unsupported")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn segment_template_lives_alongside_playlist() {
|
||||
let video = Path::new("/var/video");
|
||||
let t = segment_template_for_hash(video, "abcdef0123456789");
|
||||
assert_eq!(
|
||||
t,
|
||||
PathBuf::from("/var/video/ab/abcdef0123456789/segment_%03d.ts")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn distinct_hashes_yield_distinct_dirs() {
|
||||
let video = Path::new("/var/video");
|
||||
let a = playlist_for_hash(video, "1111aaaa");
|
||||
let b = playlist_for_hash(video, "2222bbbb");
|
||||
assert_ne!(a.parent(), b.parent());
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@ use walkdir::WalkDir;
|
||||
|
||||
pub mod actors;
|
||||
pub mod ffmpeg;
|
||||
pub mod hls_paths;
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub async fn generate_video_gifs() {
|
||||
|
||||
Reference in New Issue
Block a user