Scan and generate Video HLS playlists on startup

Refactored and improved video path state. Bumped versions of some dependencies.

src/main.rs

@@ -212,9 +212,9 @@ async fn generate_video(
 ) -> impl Responder {
     let filename = PathBuf::from(&body.path);
 
-    if let Some(name) = filename.file_stem() {
+    if let Some(name) = filename.file_name() {
         let filename = name.to_str().expect("Filename should convert to string");
-        let playlist = format!("tmp/{}.m3u8", filename);
+        let playlist = format!("{}/{}.m3u8", app_state.video_path, filename);
         if let Some(path) = is_valid_full_path(&app_state.base_path, &body.path, false) {
             if let Ok(child) = create_playlist(path.to_str().unwrap(), &playlist).await {
                 app_state
@@ -243,7 +243,7 @@ async fn stream_video(
     debug!("Playlist: {}", playlist);
 
     // Extract video playlist dir to dotenv
-    if !playlist.starts_with("tmp")
+    if !playlist.starts_with(&app_state.video_path)
         && is_valid_full_path(&app_state.base_path, playlist, false).is_some()
     {
         HttpResponse::BadRequest().finish()
@@ -259,14 +259,19 @@ async fn get_video_part(
     request: HttpRequest,
     _: Claims,
     path: web::Path<ThumbnailRequest>,
+    app_state: Data<AppState>,
 ) -> impl Responder {
     let part = &path.path;
     debug!("Video part: {}", part);
 
-    if let Ok(file) = NamedFile::open(String::from("tmp/") + part) {
+    let mut file_part = PathBuf::new();
+    file_part.push(app_state.video_path.clone());
+    file_part.push(part);
+    // TODO: Do we need to guard against directory attacks here?
+    if let Ok(file) = NamedFile::open(&file_part) {
         file.into_response(&request)
     } else {
-        error!("Video part not found: tmp/{}", part);
+        error!("Video part not found: {:?}", file_part);
         HttpResponse::NotFound().finish()
     }
 }
@@ -450,6 +455,7 @@ fn is_image(entry: &DirEntry) -> bool {
         .path()
         .extension()
         .and_then(|ext| ext.to_str())
+        .map(|ext| ext.to_lowercase())
         .map(|ext| ext == "jpg" || ext == "jpeg" || ext == "png" || ext == "nef")
         .unwrap_or(false)
 }
@@ -459,6 +465,7 @@ fn is_video(entry: &DirEntry) -> bool {
         .path()
         .extension()
         .and_then(|ext| ext.to_str())
+        .map(|ext| ext.to_lowercase())
         .map(|ext| ext == "mp4" || ext == "mov")
         .unwrap_or(false)
 }
@@ -493,6 +500,13 @@ fn main() -> std::io::Result<()> {
             .register(Box::new(VIDEO_GAUGE.clone()))
             .unwrap();
 
+        let app_state = app_data.clone();
+        app_state
+            .playlist_manager
+            .do_send(ScanDirectoryMessage {
+                directory: app_state.base_path.clone(),
+            });
+
         HttpServer::new(move || {
             let user_dao = SqliteUserDao::new();
             let favorites_dao = SqliteFavoriteDao::new();

src/state.rs

@@ -1,11 +1,14 @@
+use crate::video::{PlaylistGenerator, VideoPlaylistManager};
 use crate::StreamActor;
 use actix::{Actor, Addr};
 use std::{env, sync::Arc};
 
 pub struct AppState {
     pub stream_manager: Arc<Addr<StreamActor>>,
+    pub playlist_manager: Arc<Addr<VideoPlaylistManager>>,
     pub base_path: String,
     pub thumbnail_path: String,
+    pub video_path: String,
 }
 
 impl AppState {
@@ -13,11 +16,18 @@ impl AppState {
         stream_manager: Arc<Addr<StreamActor>>,
         base_path: String,
         thumbnail_path: String,
+        video_path: String,
     ) -> Self {
+        let playlist_generator = PlaylistGenerator::new();
+        let video_playlist_manager =
+            VideoPlaylistManager::new(video_path.clone(), playlist_generator.start());
+
         Self {
             stream_manager,
+            playlist_manager: Arc::new(video_playlist_manager.start()),
             base_path,
             thumbnail_path,
+            video_path,
         }
     }
 }
@@ -28,6 +38,7 @@ impl Default for AppState {
             Arc::new(StreamActor {}.start()),
             env::var("BASE_PATH").expect("BASE_PATH was not set in the env"),
             env::var("THUMBNAILS").expect("THUMBNAILS was not set in the env"),
+            env::var("VIDEO_PATH").expect("VIDEO_PATH was not set in the env"),
         )
     }
 }

src/video.rs

@@ -1,10 +1,13 @@
-use std::io::Result;
-use std::path::Path;
-use std::process::{Child, Command, ExitStatus, Stdio};
-
+use crate::is_video;
 use actix::prelude::*;
-use log::{debug, trace};
-
+use futures::TryFutureExt;
+use log::{debug, info, trace, warn};
+use std::io::Result;
+use std::path::{Path, PathBuf};
+use std::process::{Child, Command, ExitStatus, Stdio};
+use std::sync::Arc;
+use tokio::sync::Semaphore;
+use walkdir::{DirEntry, WalkDir};
 // ffmpeg -i test.mp4 -c:v h264 -flags +cgop -g 30 -hls_time 3 out.m3u8
 // ffmpeg -i "filename.mp4" -preset veryfast -c:v libx264 -f hls -hls_list_size 100 -hls_time 2 -crf 24 -vf scale=1080:-2,setsar=1:1 attempt/vid_out.m3u8
 
@@ -93,3 +96,215 @@ pub fn generate_video_thumbnail(path: &Path, destination: &Path) {
         .output()
         .expect("Failure to create video frame");
 }
+
+pub struct VideoPlaylistManager {
+    playlist_dir: PathBuf,
+    playlist_generator: Addr<PlaylistGenerator>,
+}
+
+impl VideoPlaylistManager {
+    pub fn new<P: Into<PathBuf>>(
+        playlist_dir: P,
+        playlist_generator: Addr<PlaylistGenerator>,
+    ) -> Self {
+        Self {
+            playlist_dir: playlist_dir.into(),
+            playlist_generator,
+        }
+    }
+}
+
+impl Actor for VideoPlaylistManager {
+    type Context = Context<Self>;
+}
+
+impl Handler<ScanDirectoryMessage> for VideoPlaylistManager {
+    type Result = ResponseFuture<()>;
+
+    fn handle(&mut self, msg: ScanDirectoryMessage, _ctx: &mut Self::Context) -> Self::Result {
+        let start = std::time::Instant::now();
+        info!(
+            "Starting scan directory for video playlist generation: {}",
+            msg.directory
+        );
+
+        let video_files = WalkDir::new(&msg.directory)
+            .into_iter()
+            .filter_map(|e| e.ok())
+            .filter(|e| e.file_type().is_file())
+            .filter(|e| is_video(e))
+            .collect::<Vec<DirEntry>>();
+
+        let scan_dir_name = msg.directory.clone();
+        let playlist_output_dir = self.playlist_dir.clone();
+        let playlist_generator = self.playlist_generator.clone();
+
+        Box::pin(async move {
+            for e in video_files {
+                let path = e.path();
+                let path_as_str = path.to_str().unwrap();
+                debug!(
+                    "Sending generate playlist message for path: {}",
+                    path_as_str
+                );
+
+                match playlist_generator
+                    .send(GeneratePlaylistMessage {
+                        playlist_path: playlist_output_dir.to_str().unwrap().to_string(),
+                        video_path: PathBuf::from(path),
+                    })
+                    .await
+                    .expect("Failed to send generate playlist message")
+                {
+                    Ok(_) => {}
+                    Err(e) => {
+                        warn!("Failed to generate playlist for path '{:?}'. {:?}", path, e);
+                    }
+                }
+                // .expect("Failed to generate video playlist");
+            }
+
+            info!(
+                "Finished directory scan of '{}' in {:?}",
+                scan_dir_name,
+                start.elapsed()
+            );
+        })
+    }
+}
+
+#[derive(Message)]
+#[rtype(result = "()")]
+pub struct ScanDirectoryMessage {
+    pub(crate) directory: String,
+}
+
+#[derive(Message)]
+#[rtype(result = "Result<String>")]
+struct GeneratePlaylistMessage {
+    video_path: PathBuf,
+    playlist_path: String,
+}
+
+pub struct PlaylistGenerator {
+    semaphore: Arc<Semaphore>,
+}
+
+impl PlaylistGenerator {
+    pub(crate) fn new() -> Self {
+        PlaylistGenerator {
+            semaphore: Arc::new(Semaphore::new(2)),
+        }
+    }
+}
+
+impl Actor for PlaylistGenerator {
+    type Context = Context<Self>;
+}
+
+impl Handler<GeneratePlaylistMessage> for PlaylistGenerator {
+    type Result = ResponseFuture<Result<String>>;
+
+    fn handle(&mut self, msg: GeneratePlaylistMessage, _ctx: &mut Self::Context) -> Self::Result {
+        let video_file = msg.video_path.to_str().unwrap().to_owned();
+        let playlist_path = msg.playlist_path.as_str().to_owned();
+        let semaphore = self.semaphore.clone();
+
+        let playlist_file = format!(
+            "{}/{}.m3u8",
+            playlist_path,
+            msg.video_path.file_name().unwrap().to_str().unwrap()
+        );
+        Box::pin(async move {
+            let wait_start = std::time::Instant::now();
+            let permit = semaphore
+                .acquire_owned()
+                .await
+                .expect("Unable to acquire semaphore");
+
+            debug!(
+                "Waited for {:?} before starting ffmpeg",
+                wait_start.elapsed()
+            );
+
+            if Path::new(&playlist_file).exists() {
+                debug!("Playlist already exists: {}", playlist_file);
+                return Err(std::io::Error::from(std::io::ErrorKind::AlreadyExists));
+            }
+
+            tokio::spawn(async move {
+                let ffmpeg_result = tokio::process::Command::new("ffmpeg")
+                    .arg("-i")
+                    .arg(&video_file)
+                    .arg("-c:v")
+                    .arg("h264")
+                    .arg("-crf")
+                    .arg("21")
+                    .arg("-preset")
+                    .arg("veryfast")
+                    .arg("-hls_time")
+                    .arg("3")
+                    .arg("-hls_list_size")
+                    .arg("100")
+                    .arg("-vf")
+                    .arg("scale=1080:-2,setsar=1:1")
+                    .arg(playlist_file)
+                    .stdout(Stdio::null())
+                    .stderr(Stdio::piped())
+                    .status()
+                    .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))
+                    .await;
+
+                // Hang on to the permit until we're done decoding and then explicitly drop
+                drop(permit);
+
+                ffmpeg_result
+            });
+
+            Ok("meeee".to_string())
+            // .spawn()
+            // .expect("Failed to spawn ffmpeg process");
+
+            // .expect("Failed to spawn child process")
+            // .wait()
+            // .await
+            // .inspect_err(|e| error!("Failed to wait on child process: {}", e));
+
+            /*                .map(|exit_status| {
+                                debug!(
+                                    "Finished waiting for playlist generate process for file '{}' with code: {}",
+                                    video_file,
+                                    exit_status
+                                );
+
+                                exit_status.to_string()
+                            })
+            */
+
+            /*  if let Some(stderr) = ffmpeg.stderr {
+                tokio::spawn(async move {
+                    let mut reader = BufReader::new(stderr).lines();
+                    while let Ok(Some(line)) = reader.next_line().await {
+                        println!("ffmpeg line: {:?}", line);
+                    }
+                });
+            }*/
+
+            /*            ffmpeg.wait().await.map(|exit_status| {
+                           debug!(
+                               "Finished waiting for playlist generate process for file '{}' with code: {}",
+                               video_file, exit_status
+                           );
+
+                           exit_status.to_string()
+                       })
+            */
+            // ffmpeg
+            //     .wait_with_output()
+            //     .await
+            //     .expect("TODO: panic message")
+            //
+            // Ok(video_file)
+        })
+    }
+}