From 8e6de37345cbd1d4d17848a8b8ced6af0842734a Mon Sep 17 00:00:00 2001 From: Cameron Cordes Date: Mon, 2 Nov 2026 15:25:19 -0500 Subject: [PATCH] HLS/previews: rerun on the CPU when the GPU can't be used right now A busy GPU (llama-swap holding the 3090) made ffmpeg's CUDA setup fail with CUDA_ERROR_OUT_OF_MEMORY, exit 187. The playlist generator took that for a bad file and wrote playlist.unsupported, so the video never played; preview clips were marked failed the same way. is_gpu_init_failure() recognises CUDA/NVENC setup errors in ffmpeg's stderr. When an NVENC run fails that way, the playlist and the preview clip rerun once on libx264. A file that also fails on the CPU still gets the sentinel. The HLS command and the timed run are pulled out of the handler (hls_command, run_ffmpeg) so both attempts share them. Fixes #215 Co-Authored-By: Claude Opus 5.5 --- src/video/actors.rs | 261 +++++++++++++++++++++++++++----------------- src/video/ffmpeg.rs | 105 ++++++++++++++---- 2 files changed, 245 insertions(+), 121 deletions(-) diff --git a/src/video/actors.rs b/src/video/actors.rs index a884e69..d3688c0 100644 --- a/src/video/actors.rs +++ b/src/video/actors.rs @@ -635,110 +635,37 @@ impl Handler for PlaylistGenerator { let nvenc = !use_copy && crate::video::ffmpeg::is_nvenc_available().await; - let mut cmd = tokio::process::Command::new("ffmpeg"); - cmd.arg("-y"); - if nvenc { - // Decode on NVDEC too. Without it a 4K source is decoded on - // the CPU and only the encode runs on the GPU: ~4.5x the CPU - // time. Frames come back to system memory, so the CPU - // filters (autorotate, scale) still apply, and ffmpeg falls - // back to software decode for codecs NVDEC lacks (ProRes). - cmd.arg("-hwaccel").arg("cuda"); - } - cmd.arg("-i").arg(&video_file); - - if use_copy { - cmd.arg("-c:v").arg("copy"); - cmd.arg("-c:a").arg("aac"); - } else { - if nvenc { - // NVENC: no CRF, use VBR + target CQ. p1 = fastest - // preset — prioritizes encoder throughput over bitrate - // efficiency. CQ 23 roughly matches libx264 crf 21 - // visually; NVENC has slightly lower compression - // efficiency per quality. - cmd.arg("-c:v").arg("h264_nvenc"); - cmd.arg("-preset").arg("p1"); - cmd.arg("-rc").arg("vbr"); - cmd.arg("-cq").arg("23"); - cmd.arg("-pix_fmt").arg("yuv420p"); - // Without this NVENC turns -force_key_frames into plain - // I-frames, which the HLS muxer can't split on: segments - // fall back to NVENC's 250-frame GOP (8.34 s at 29.97). - cmd.arg("-forced-idr").arg("1"); - } else { - cmd.arg("-c:v").arg("h264"); - cmd.arg("-crf").arg("21"); - cmd.arg("-preset").arg("veryfast"); - } - // Cap the short side so landscape and portrait both land at - // real 1080p. iw/ih are post-autorotation display dims. - cmd.arg("-vf").arg(format!( - "scale='if(gt(iw,ih),-2,min({m},iw))':'if(gt(iw,ih),min({m},ih),-2)',setsar=1:1", - m = HLS_MAX_SHORT_SIDE - )); - cmd.arg("-c:a").arg("aac"); - // Force an IDR frame every hls_time seconds so each HLS - // segment starts on a keyframe — accurate seeking without - // players having to decode from a prior segment. - cmd.arg("-force_key_frames").arg("expr:gte(t,n_forced*3)"); - // Keep source frame timestamps. Variable-frame-rate phone - // clips and screen recordings report r_frame_rate as their - // timebase (90000/1); the mpegts muxer's default CFR mode - // then duplicates frames to fill it — a 2.6 s clip came out - // as 233k frames / 221 MB. vfr leaves timing untouched. - cmd.arg("-fps_mode").arg("vfr"); - } - - // -f hls is required because the playlist is written to a .tmp - // path during encoding — ffmpeg normally infers the muxer from - // the output extension and doesn't recognize ".m3u8.tmp". - cmd.arg("-f").arg("hls"); - cmd.arg("-hls_time").arg("3"); - cmd.arg("-hls_list_size").arg("0"); - cmd.arg("-hls_playlist_type").arg("vod"); - // independent_segments advertises that each segment can be - // decoded without reference to any other — the matching guarantee - // for the forced keyframes above. - cmd.arg("-hls_flags").arg("independent_segments"); - cmd.arg("-hls_segment_filename").arg(&segment_pattern); - cmd.arg(&playlist_tmp); - cmd.stdout(Stdio::null()); - cmd.stderr(Stdio::piped()); - cmd.kill_on_drop(true); - - // Spawn + wait under a timeout so a hung ffmpeg (corrupt source, - // NFS stall, etc.) doesn't permanently hold a semaphore slot. - // Default is generous — a long 4K transcode on CPU can take hours. + // Default timeout is generous — a long 4K transcode on CPU can take hours. let timeout_secs = std::env::var("HLS_TIMEOUT_SECONDS") .ok() .and_then(|v| v.parse::().ok()) .unwrap_or(7200); - let ffmpeg_result = match cmd.spawn() { - Ok(child) => { - match tokio::time::timeout( - std::time::Duration::from_secs(timeout_secs), - child.wait_with_output(), - ) - .await - { - Ok(res) => res - .inspect_err(|e| { - error!("Failed to wait on ffmpeg child process: {}", e) - }) - .map_err(|e| std::io::Error::other(e.to_string())), - Err(_) => Err(std::io::Error::other(format!( - "ffmpeg exceeded {}s timeout", - timeout_secs - ))), - } - } - Err(e) => { - error!("Failed to spawn ffmpeg: {}", e); - Err(std::io::Error::other(e.to_string())) - } - }; + let mut ffmpeg_result = run_ffmpeg( + hls_command(&video_file, &playlist_tmp, &segment_pattern, use_copy, nvenc), + timeout_secs, + ) + .await; + + // A GPU that's busy right now (#215: llama-swap holding the 3090) + // says nothing about the file. Encode on the CPU instead of + // writing a sentinel that would skip this video for good. + if nvenc + && matches!(&ffmpeg_result, Ok(out) if !out.status.success() + && crate::video::ffmpeg::is_gpu_init_failure(&String::from_utf8_lossy(&out.stderr))) + { + warn!( + "GPU unavailable for {}, retrying the playlist on the CPU encoder", + video_file + ); + span.add_event("GPU unavailable, retrying on CPU", vec![]); + cleanup_partial_hls(&hash_dir).await; + ffmpeg_result = run_ffmpeg( + hls_command(&video_file, &playlist_tmp, &segment_pattern, use_copy, false), + timeout_secs, + ) + .await; + } drop(permit); @@ -789,6 +716,118 @@ impl Handler for PlaylistGenerator { } } +/// The ffmpeg command that writes a video's HLS playlist and segments. +fn hls_command( + video_file: &str, + playlist_tmp: &str, + segment_pattern: &str, + use_copy: bool, + nvenc: bool, +) -> tokio::process::Command { + let mut cmd = tokio::process::Command::new("ffmpeg"); + cmd.arg("-y"); + if nvenc { + // Decode on NVDEC too. Without it a 4K source is decoded on + // the CPU and only the encode runs on the GPU: ~4.5x the CPU + // time. Frames come back to system memory, so the CPU + // filters (autorotate, scale) still apply, and ffmpeg falls + // back to software decode for codecs NVDEC lacks (ProRes). + cmd.arg("-hwaccel").arg("cuda"); + } + cmd.arg("-i").arg(video_file); + + if use_copy { + cmd.arg("-c:v").arg("copy"); + cmd.arg("-c:a").arg("aac"); + } else { + if nvenc { + // NVENC: no CRF, use VBR + target CQ. p1 = fastest + // preset — prioritizes encoder throughput over bitrate + // efficiency. CQ 23 roughly matches libx264 crf 21 + // visually; NVENC has slightly lower compression + // efficiency per quality. + cmd.arg("-c:v").arg("h264_nvenc"); + cmd.arg("-preset").arg("p1"); + cmd.arg("-rc").arg("vbr"); + cmd.arg("-cq").arg("23"); + cmd.arg("-pix_fmt").arg("yuv420p"); + // Without this NVENC turns -force_key_frames into plain + // I-frames, which the HLS muxer can't split on: segments + // fall back to NVENC's 250-frame GOP (8.34 s at 29.97). + cmd.arg("-forced-idr").arg("1"); + } else { + cmd.arg("-c:v").arg("h264"); + cmd.arg("-crf").arg("21"); + cmd.arg("-preset").arg("veryfast"); + } + // Cap the short side so landscape and portrait both land at + // real 1080p. iw/ih are post-autorotation display dims. + cmd.arg("-vf").arg(format!( + "scale='if(gt(iw,ih),-2,min({m},iw))':'if(gt(iw,ih),min({m},ih),-2)',setsar=1:1", + m = HLS_MAX_SHORT_SIDE + )); + cmd.arg("-c:a").arg("aac"); + // Force an IDR frame every hls_time seconds so each HLS + // segment starts on a keyframe — accurate seeking without + // players having to decode from a prior segment. + cmd.arg("-force_key_frames").arg("expr:gte(t,n_forced*3)"); + // Keep source frame timestamps. Variable-frame-rate phone + // clips and screen recordings report r_frame_rate as their + // timebase (90000/1); the mpegts muxer's default CFR mode + // then duplicates frames to fill it — a 2.6 s clip came out + // as 233k frames / 221 MB. vfr leaves timing untouched. + cmd.arg("-fps_mode").arg("vfr"); + } + + // -f hls is required because the playlist is written to a .tmp + // path during encoding — ffmpeg normally infers the muxer from + // the output extension and doesn't recognize ".m3u8.tmp". + cmd.arg("-f").arg("hls"); + cmd.arg("-hls_time").arg("3"); + cmd.arg("-hls_list_size").arg("0"); + cmd.arg("-hls_playlist_type").arg("vod"); + // independent_segments advertises that each segment can be + // decoded without reference to any other — the matching guarantee + // for the forced keyframes above. + cmd.arg("-hls_flags").arg("independent_segments"); + cmd.arg("-hls_segment_filename").arg(segment_pattern); + cmd.arg(playlist_tmp); + cmd.stdout(Stdio::null()); + cmd.stderr(Stdio::piped()); + cmd.kill_on_drop(true); + cmd +} + +/// Run ffmpeg under a timeout, so a hung run (corrupt source, NFS stall) +/// doesn't hold a semaphore slot forever. +async fn run_ffmpeg( + mut cmd: tokio::process::Command, + timeout_secs: u64, +) -> Result { + match cmd.spawn() { + Ok(child) => { + match tokio::time::timeout( + std::time::Duration::from_secs(timeout_secs), + child.wait_with_output(), + ) + .await + { + Ok(res) => res + .inspect_err(|e| error!("Failed to wait on ffmpeg child process: {}", e)) + .map_err(|e| std::io::Error::other(e.to_string())), + Err(_) => Err(std::io::Error::other(format!( + "ffmpeg exceeded {}s timeout", + timeout_secs + ))), + } + } + Err(e) => { + error!("Failed to spawn ffmpeg: {}", e); + Err(std::io::Error::other(e.to_string())) + } + } +} + /// Delete the partial playlist (.tmp) and any segment files left behind by /// a failed ffmpeg run. Wipes every non-sentinel file in the hash dir; /// retains the sentinel if one has already been written by an earlier @@ -958,7 +997,29 @@ impl Handler for PreviewClipGenerator { #[cfg(test)] mod tests { - use super::{VideoStreamMeta, fits_hls_copy_limits, parse_ffprobe_rational}; + use super::{VideoStreamMeta, fits_hls_copy_limits, hls_command, parse_ffprobe_rational}; + + fn args(nvenc: bool) -> Vec { + hls_command("in.mp4", "out.m3u8.tmp", "seg_%03d.ts", false, nvenc) + .as_std() + .get_args() + .map(|a| a.to_string_lossy().into_owned()) + .collect() + } + + #[test] + fn cpu_fallback_command_never_touches_the_gpu() { + // #215: the retry after a GPU init failure must not ask for CUDA again. + let cpu = args(false); + assert!(!cpu.iter().any(|a| a == "-hwaccel" || a == "h264_nvenc")); + assert!(cpu.windows(2).any(|w| w[0] == "-c:v" && w[1] == "h264")); + let gpu = args(true); + assert!(gpu.windows(2).any(|w| w[0] == "-hwaccel" && w[1] == "cuda")); + assert!( + gpu.windows(2) + .any(|w| w[0] == "-c:v" && w[1] == "h264_nvenc") + ); + } const LIMIT: u64 = 20_000_000; diff --git a/src/video/ffmpeg.rs b/src/video/ffmpeg.rs index 5c88fcd..fe83319 100644 --- a/src/video/ffmpeg.rs +++ b/src/video/ffmpeg.rs @@ -36,6 +36,24 @@ pub async fn is_nvenc_available() -> bool { available } +/// Whether ffmpeg's stderr says the GPU couldn't be used *right now*, as +/// opposed to the input being bad (#215). `is_nvenc_available` only checks +/// that ffmpeg was built with NVENC; on a GPU shared with llama-swap a resident +/// model can leave no memory for a CUDA context, and ffmpeg then exits 187 +/// with `cuCtxCreate(...) failed -> CUDA_ERROR_OUT_OF_MEMORY`. Callers rerun +/// such a failure on the CPU encoder instead of recording the file as broken. +pub fn is_gpu_init_failure(stderr: &str) -> bool { + const MARKERS: &[&str] = &[ + "CUDA_ERROR_", + "cuCtxCreate", + "OpenEncodeSessionEx failed", + "No capable devices found", + "Cannot load libcuda", + "Device creation failed", + ]; + MARKERS.iter().any(|m| stderr.contains(m)) +} + pub struct Ffmpeg; pub enum GifType { @@ -271,25 +289,13 @@ fn parse_ffprobe_duration(stdout: &str) -> Option { None } -/// Generate a preview clip from a video file. -/// -/// Creates a ~10 second MP4 by extracting up to 10 equally-spaced 1-second segments -/// at 480p with H.264 video and AAC audio. For short videos (<10s), uses fewer segments. -/// For very short videos (<1s), transcodes the entire video. -/// -/// Returns (duration_seconds, file_size_bytes) on success. -pub async fn generate_preview_clip(input_file: &str, output_file: &str) -> Result<(f64, u64)> { - info!("Generating preview clip for: '{}'", input_file); - let start = Instant::now(); - - let duration = get_duration_seconds(input_file).await?; - let use_nvenc = is_nvenc_available().await; - - // Create parent directories for output - if let Some(parent) = std::path::Path::new(output_file).parent() { - std::fs::create_dir_all(parent)?; - } - +/// The ffmpeg command for a preview clip, and the clip duration it produces. +fn preview_clip_command( + input_file: &str, + output_file: &str, + duration: Option, + use_nvenc: bool, +) -> (Command, f64) { let mut cmd = Command::new("ffmpeg"); cmd.arg("-y"); @@ -350,7 +356,43 @@ pub async fn generate_preview_clip(input_file: &str, output_file: &str) -> Resul cmd.stdout(Stdio::null()); cmd.stderr(Stdio::piped()); - let output = cmd.output().await?; + (cmd, clip_duration) +} + +/// Generate a preview clip from a video file. +/// +/// Creates a ~10 second MP4 by extracting up to 10 equally-spaced 1-second segments +/// at 480p with H.264 video and AAC audio. For short videos (<10s), uses fewer segments. +/// For very short videos (<1s), transcodes the entire video. +/// +/// Returns (duration_seconds, file_size_bytes) on success. +pub async fn generate_preview_clip(input_file: &str, output_file: &str) -> Result<(f64, u64)> { + info!("Generating preview clip for: '{}'", input_file); + let start = Instant::now(); + + let duration = get_duration_seconds(input_file).await?; + let use_nvenc = is_nvenc_available().await; + + // Create parent directories for output + if let Some(parent) = std::path::Path::new(output_file).parent() { + std::fs::create_dir_all(parent)?; + } + + let (mut cmd, clip_duration) = + preview_clip_command(input_file, output_file, duration, use_nvenc); + let mut output = cmd.output().await?; + + if use_nvenc + && !output.status.success() + && is_gpu_init_failure(&String::from_utf8_lossy(&output.stderr)) + { + warn!( + "GPU unavailable for preview of '{}', retrying on the CPU encoder", + input_file + ); + let (mut cmd, _) = preview_clip_command(input_file, output_file, duration, false); + output = cmd.output().await?; + } if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); @@ -376,7 +418,28 @@ pub async fn generate_preview_clip(input_file: &str, output_file: &str) -> Resul #[cfg(test)] mod tests { - use super::parse_ffprobe_duration; + use super::{is_gpu_init_failure, parse_ffprobe_duration}; + + #[test] + fn gpu_busy_is_a_gpu_init_failure() { + // Verbatim from the e2e server with a llama-swap model resident (#215). + let oom = "[CUDA @ 0x5d1c2c0] cu->cuCtxCreate(&hwctx->cuda_ctx, desired_flags, hwctx->internal->cuda_device) failed -> CUDA_ERROR_OUT_OF_MEMORY: out of memory\nDevice creation failed: -542398533."; + assert!(is_gpu_init_failure(oom)); + assert!(is_gpu_init_failure( + "[h264_nvenc @ 0x1] OpenEncodeSessionEx failed: out of memory (10): (no details)" + )); + assert!(is_gpu_init_failure( + "[h264_nvenc @ 0x1] No capable devices found" + )); + } + + #[test] + fn bad_input_is_not_a_gpu_init_failure() { + assert!(!is_gpu_init_failure( + "[mov,mp4,m4a,3gp,3g2,mj2 @ 0x1] moov atom not found\nclip.mp4: Invalid data found when processing input" + )); + assert!(!is_gpu_init_failure("Conversion failed!")); + } #[test] fn empty_output_returns_none() {