diff --git a/Cargo.lock b/Cargo.lock index a404abc..d4a65c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -908,6 +908,12 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.6" @@ -1218,6 +1224,26 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "fax" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05de7d48f37cd6730705cbca900770cab77a89f413d23e100ad7fad7795a0ab" +dependencies = [ + "fax_derive", +] + +[[package]] +name = "fax_derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0aca10fb742cb43f9e7bb8467c91aa9bcb8e3ffbc6a6f7389bb93ffc920577d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "fdeflate" version = "0.3.7" @@ -1501,6 +1527,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -1843,11 +1880,14 @@ checksum = "1c6a3ce16143778e24df6f95365f12ed105425b22abefd289dd88a64bab59605" dependencies = [ "bytemuck", "byteorder-lite", + "image-webp", "moxcms", "num-traits", "png", "ravif", "rayon", + "rgb", + "tiff", "zune-core", "zune-jpeg", ] @@ -1908,6 +1948,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + [[package]] name = "imgref" version = "1.11.0" @@ -3712,6 +3762,20 @@ dependencies = [ "syn", ] +[[package]] +name = "tiff" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9605de7fee8d9551863fd692cce7637f548dbd9db9180fcc07ccc6d26c336f" +dependencies = [ + "fax", + "flate2", + "half", + "quick-error", + "weezl", + "zune-jpeg", +] + [[package]] name = "time" version = "0.3.42" @@ -4269,6 +4333,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 2b966b7..847c9f6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,7 +29,7 @@ chrono = "0.4" clap = { version = "4.5", features = ["derive"] } dotenv = "0.15" bcrypt = "0.17.1" -image = { version = "0.25.5", default-features = false, features = ["jpeg", "png", "rayon"] } +image = { version = "0.25.5", default-features = false, features = ["jpeg", "png", "rayon", "webp", "tiff", "avif"] } infer = "0.16" walkdir = "2.4.0" rayon = "1.5" diff --git a/README.md b/README.md index b625b04..31978d9 100644 --- a/README.md +++ b/README.md @@ -14,14 +14,43 @@ Upon first run it will generate thumbnails for all images and videos at `BASE_PA - **RAG-based Context Retrieval** - Semantic search over daily conversation summaries - **Automatic Daily Summaries** - LLM-generated summaries of daily conversations with embeddings +## External Dependencies + +### ffmpeg (required) +`ffmpeg` must be on `PATH`. It is used for: +- **HLS video streaming** — transcoding/segmenting source videos into `.m3u8` + `.ts` playlists +- **Video thumbnails** — extracting a frame at the 3-second mark +- **Video preview clips** — short looping previews for the Video Wall +- **HEIC / HEIF thumbnails** — decoding Apple's HEIC format (your ffmpeg build must include + `libheif`; most modern builds do) + +Builds used in development: the `gyan.dev` full build on Windows, and distro `ffmpeg` +packages on Linux work fine. If HEIC thumbnails silently fail, check +`ffmpeg -formats | grep heif` to confirm HEIF support. + +### RAW photo thumbnails (no extra dependency) +RAW formats (ARW, NEF, CR2, CR3, DNG, RAF, ORF, RW2, PEF, SRW, TIFF) are thumbnailed +by reading the embedded JPEG preview from the TIFF IFD1 using `kamadak-exif`. No +external RAW decoder (libraw / dcraw) is required. Files without an embedded preview +fall back to ffmpeg (works for most NEF files), and anything that still can't be +decoded is marked with a `.unsupported` sentinel in the thumbnail directory +so we don't retry it every scan. Delete those sentinels to force retries after a +tooling upgrade. + ## Environment There are a handful of required environment variables to have the API run. They should be defined where the binary is located or above it in an `.env` file. -You must have `ffmpeg` installed for streaming video and generating video thumbnails. - `DATABASE_URL` is a path or url to a database (currently only SQLite is tested) - `BASE_PATH` is the root from which you want to serve images and videos -- `THUMBNAILS` is a path where generated thumbnails should be stored +- `THUMBNAILS` is a path where generated thumbnails should be stored. Thumbnails + mirror the source tree under `BASE_PATH` and keep the source's original + extension (e.g. `foo.arw` or `bar.mp4`), though the file contents are always + JPEG bytes — browsers content-sniff. Files that can't be thumbnailed by the + `image` crate, ffmpeg, or an embedded RAW preview get a zero-byte + `.unsupported` sentinel in this directory so subsequent scans + skip them. Delete the `*.unsupported` files to force retries (for example + after upgrading ffmpeg or adding libheif) - `VIDEO_PATH` is a path where HLS playlists and video parts should be stored - `GIFS_DIRECTORY` is a path where generated video GIF thumbnails should be stored - `BIND_URL` is the url and port to bind to (typically your own IP address) diff --git a/src/exif.rs b/src/exif.rs index c096f71..0cd29d9 100644 --- a/src/exif.rs +++ b/src/exif.rs @@ -1,5 +1,5 @@ use std::fs::File; -use std::io::BufReader; +use std::io::{BufReader, Read, Seek, SeekFrom}; use std::path::Path; use anyhow::{Result, anyhow}; @@ -25,6 +25,60 @@ pub struct ExifData { pub date_taken: Option, } +/// TIFF-based RAW formats where `JPEGInterchangeFormat` offsets are +/// absolute file offsets (the file itself is a TIFF container). +fn is_tiff_raw(path: &Path) -> bool { + matches!( + path.extension() + .and_then(|e| e.to_str()) + .map(|s| s.to_lowercase()) + .as_deref(), + Some( + "tiff" | "tif" | "nef" | "cr2" | "arw" | "dng" | "raf" | "orf" | "rw2" | "pef" | "srw" + ) + ) +} + +/// Returns the bytes of the embedded JPEG thumbnail in a TIFF-based RAW or +/// TIFF file. Used to thumbnail formats whose RAW pixel data can't be decoded +/// by our normal tools (e.g. Sony ARW). Returns `None` if no preview is +/// present, the file isn't a TIFF container, or the data doesn't look like +/// a valid JPEG. +pub fn extract_embedded_jpeg_preview(path: &Path) -> Option> { + if !is_tiff_raw(path) { + return None; + } + + let file = File::open(path).ok()?; + let mut bufreader = BufReader::new(file); + let exif = Reader::new().read_from_container(&mut bufreader).ok()?; + + let offset = exif + .get_field(Tag::JPEGInterchangeFormat, In::THUMBNAIL)? + .value + .get_uint(0)?; + let length = exif + .get_field(Tag::JPEGInterchangeFormatLength, In::THUMBNAIL)? + .value + .get_uint(0)?; + if length == 0 { + return None; + } + + let mut file = File::open(path).ok()?; + file.seek(SeekFrom::Start(offset as u64)).ok()?; + let mut buf = vec![0u8; length as usize]; + file.read_exact(&mut buf).ok()?; + + // JPEG SOI marker sanity check — MakerNote offsets sometimes point at + // TIFF-wrapped previews or other non-JPEG data. + if buf.len() < 2 || buf[0] != 0xFF || buf[1] != 0xD8 { + return None; + } + + Some(buf) +} + pub fn supports_exif(path: &Path) -> bool { if let Some(ext) = path.extension() { let ext_lower = ext.to_string_lossy().to_lowercase(); diff --git a/src/file_types.rs b/src/file_types.rs index c1249d0..f312916 100644 --- a/src/file_types.rs +++ b/src/file_types.rs @@ -3,9 +3,22 @@ use walkdir::DirEntry; /// Supported image file extensions pub const IMAGE_EXTENSIONS: &[&str] = &[ - "jpg", "jpeg", "png", "webp", "tiff", "tif", "heif", "heic", "avif", "nef", + "jpg", "jpeg", "png", "webp", "tiff", "tif", "heif", "heic", "avif", "nef", "arw", ]; +/// Extensions the `image` crate cannot decode — we fall back to ffmpeg to +/// extract an embedded preview or decode the frame. +pub const FFMPEG_THUMBNAIL_EXTENSIONS: &[&str] = &["heif", "heic", "nef", "arw"]; + +/// Returns true if thumbnail generation should go through ffmpeg instead of +/// the `image` crate (RAW formats, HEIF/HEIC). +pub fn needs_ffmpeg_thumbnail(path: &Path) -> bool { + match path.extension().and_then(|e| e.to_str()) { + Some(ext) => FFMPEG_THUMBNAIL_EXTENSIONS.contains(&ext.to_lowercase().as_str()), + None => false, + } +} + /// Supported video file extensions pub const VIDEO_EXTENSIONS: &[&str] = &["mp4", "mov", "avi", "mkv"]; diff --git a/src/main.rs b/src/main.rs index da0c4dd..db63ef0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -52,7 +52,8 @@ use crate::state::AppState; use crate::tags::*; use crate::video::actors::{ GeneratePreviewClipMessage, ProcessMessage, QueueVideosMessage, ScanDirectoryMessage, - VideoPlaylistManager, create_playlist, generate_video_thumbnail, + VideoPlaylistManager, create_playlist, generate_image_thumbnail_ffmpeg, + generate_video_thumbnail, }; use log::{debug, error, info, trace, warn}; use opentelemetry::trace::{Span, Status, TraceContextExt, Tracer}; @@ -1060,6 +1061,47 @@ async fn delete_favorite( } } +/// Sentinel path written next to a would-be thumbnail when a file cannot be +/// decoded by either the `image` crate or ffmpeg. Its presence causes future +/// scans to skip the file instead of re-logging the failure. +pub fn unsupported_thumbnail_sentinel(thumb_path: &Path) -> PathBuf { + let mut s = thumb_path.as_os_str().to_owned(); + s.push(".unsupported"); + PathBuf::from(s) +} + +fn generate_image_thumbnail(src: &Path, thumb_path: &Path) -> std::io::Result<()> { + // RAW formats (ARW/NEF/CR2/etc): try the file's embedded JPEG preview + // first. Avoids ffmpeg choking on proprietary RAW compression (Sony ARW + // in particular), and is faster than decoding RAW pixels anyway. + if let Some(preview) = exif::extract_embedded_jpeg_preview(src) { + let img = image::load_from_memory(&preview).map_err(|e| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("decode embedded preview {:?}: {}", src, e), + ) + })?; + let scaled = img.thumbnail(200, u32::MAX); + scaled + .save_with_format(thumb_path, image::ImageFormat::Jpeg) + .map_err(|e| std::io::Error::other(format!("save {:?}: {}", thumb_path, e)))?; + return Ok(()); + } + + if file_types::needs_ffmpeg_thumbnail(src) { + return generate_image_thumbnail_ffmpeg(src, thumb_path); + } + + let img = image::open(src).map_err(|e| { + std::io::Error::new(std::io::ErrorKind::InvalidData, format!("{:?}: {}", src, e)) + })?; + let scaled = img.thumbnail(200, u32::MAX); + scaled + .save(thumb_path) + .map_err(|e| std::io::Error::other(format!("save {:?}: {}", thumb_path, e)))?; + Ok(()) +} + fn create_thumbnails(libs: &[libraries::Library]) { let tracer = global_tracer(); let span = tracer.start("creating thumbnails"); @@ -1080,17 +1122,26 @@ fn create_thumbnails(libs: &[libraries::Library]) { .into_par_iter() .filter_map(|entry| entry.ok()) .filter(|entry| entry.file_type().is_file()) - .filter(|entry| { - if is_video(entry) { - let relative_path = &entry.path().strip_prefix(&images).unwrap(); - let thumb_path = Path::new(thumbnail_directory).join(relative_path); - std::fs::create_dir_all( - thumb_path - .parent() - .unwrap_or_else(|| panic!("Thumbnail {:?} has no parent?", thumb_path)), - ) - .expect("Error creating directory"); + .for_each(|entry| { + let src = entry.path(); + let Ok(relative_path) = src.strip_prefix(&images) else { + return; + }; + let thumb_path = Path::new(thumbnail_directory).join(relative_path); + if thumb_path.exists() || unsupported_thumbnail_sentinel(&thumb_path).exists() { + return; + } + + let Some(parent) = thumb_path.parent() else { + return; + }; + if let Err(e) = std::fs::create_dir_all(parent) { + error!("Failed to create thumbnail dir {:?}: {}", parent, e); + return; + } + + if is_video(&entry) { let mut video_span = tracer.start_with_context( "generate_video_thumbnail", &opentelemetry::Context::new() @@ -1103,37 +1154,24 @@ fn create_thumbnails(libs: &[libraries::Library]) { ]); debug!("Generating video thumbnail: {:?}", thumb_path); - generate_video_thumbnail(entry.path(), &thumb_path); + generate_video_thumbnail(src, &thumb_path); video_span.end(); - false - } else { - is_image(entry) + } else if is_image(&entry) { + match generate_image_thumbnail(src, &thumb_path) { + Ok(_) => info!("Saved thumbnail: {:?}", thumb_path), + Err(e) => { + let sentinel = unsupported_thumbnail_sentinel(&thumb_path); + error!( + "Unable to thumbnail {:?}: {}. Writing sentinel {:?}", + src, e, sentinel + ); + if let Err(se) = std::fs::write(&sentinel, b"") { + warn!("Failed to write sentinel {:?}: {}", sentinel, se); + } + } + } } - }) - .filter(|entry| { - let path = entry.path(); - let relative_path = &path.strip_prefix(&images).unwrap(); - let thumb_path = Path::new(thumbnail_directory).join(relative_path); - !thumb_path.exists() - }) - .map(|entry| (image::open(entry.path()), entry.path().to_path_buf())) - .filter(|(img, path)| { - if let Err(e) = img { - error!("Unable to open image: {:?}. {}", path, e); - } - img.is_ok() - }) - .map(|(img, path)| (img.unwrap(), path)) - .map(|(image, path)| (image.thumbnail(200, u32::MAX), path)) - .map(|(image, path)| { - let relative_path = &path.strip_prefix(&images).unwrap(); - let thumb_path = Path::new(thumbnail_directory).join(relative_path); - std::fs::create_dir_all(thumb_path.parent().unwrap()) - .expect("There was an issue creating directory"); - info!("Saving thumbnail: {:?}", thumb_path); - image.save(thumb_path).expect("Failure saving thumbnail"); - }) - .for_each(drop); + }); } debug!("Finished making thumbnails"); @@ -1744,7 +1782,8 @@ fn process_new_files( // not just photos with parseable EXIF. for (file_path, relative_path) in &files { let thumb_path = thumbnail_directory.join(relative_path); - let needs_thumbnail = !thumb_path.exists(); + let needs_thumbnail = + !thumb_path.exists() && !unsupported_thumbnail_sentinel(&thumb_path).exists(); let needs_row = !existing_exif_paths.contains_key(relative_path); if needs_thumbnail || needs_row { diff --git a/src/video/actors.rs b/src/video/actors.rs index 284c8e3..e85feb4 100644 --- a/src/video/actors.rs +++ b/src/video/actors.rs @@ -48,6 +48,14 @@ impl Handler for StreamActor { } } +pub fn playlist_file_for(playlist_dir: &str, video_path: &Path) -> PathBuf { + let filename = video_path + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("unknown"); + PathBuf::from(format!("{}/{}.m3u8", playlist_dir, filename)) +} + pub async fn create_playlist(video_path: &str, playlist_file: &str) -> Result { if Path::new(playlist_file).exists() { debug!("Playlist already exists: {}", playlist_file); @@ -103,6 +111,35 @@ pub fn generate_video_thumbnail(path: &Path, destination: &Path) { .expect("Failure to create video frame"); } +/// Use ffmpeg to extract a 200px-wide thumbnail from formats the `image` crate +/// can't decode (RAW: NEF/ARW, HEIC/HEIF). Writes JPEG bytes to `destination` +/// regardless of its extension. +pub fn generate_image_thumbnail_ffmpeg(path: &Path, destination: &Path) -> std::io::Result<()> { + let output = Command::new("ffmpeg") + .arg("-y") + .arg("-i") + .arg(path) + .arg("-vframes") + .arg("1") + .arg("-vf") + .arg("scale=200:-1") + .arg("-f") + .arg("image2") + .arg("-c:v") + .arg("mjpeg") + .arg(destination) + .output()?; + + if !output.status.success() { + return Err(std::io::Error::other(format!( + "ffmpeg failed ({}): {}", + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ))); + } + Ok(()) +} + /// Check if a video is already encoded with h264 codec /// Returns true if the video uses h264, false otherwise or if detection fails async fn is_h264_encoded(video_path: &str) -> bool { @@ -246,15 +283,18 @@ impl Handler for VideoPlaylistManager { msg.directory ); + let playlist_output_dir = self.playlist_dir.clone(); + let playlist_dir_str = playlist_output_dir.to_str().unwrap().to_string(); + let video_files = WalkDir::new(&msg.directory) .into_iter() .filter_map(|e| e.ok()) .filter(|e| e.file_type().is_file()) .filter(is_video) + .filter(|e| !playlist_file_for(&playlist_dir_str, e.path()).exists()) .collect::>(); let scan_dir_name = msg.directory.clone(); - let playlist_output_dir = self.playlist_dir.clone(); let playlist_generator = self.playlist_generator.clone(); Box::pin(async move { @@ -285,6 +325,9 @@ impl Handler for VideoPlaylistManager { path_as_str ); } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { + debug!("Playlist already exists for '{:?}', skipping", path); + } Err(e) => { warn!("Failed to generate playlist for path '{:?}'. {:?}", path, e); } @@ -318,14 +361,18 @@ impl Handler for VideoPlaylistManager { ); let playlist_output_dir = self.playlist_dir.clone(); + let playlist_dir_str = playlist_output_dir.to_str().unwrap().to_string(); let playlist_generator = self.playlist_generator.clone(); for video_path in msg.video_paths { + if playlist_file_for(&playlist_dir_str, &video_path).exists() { + continue; + } let path_str = video_path.to_string_lossy().to_string(); debug!("Queueing playlist generation for: {}", path_str); playlist_generator.do_send(GeneratePlaylistMessage { - playlist_path: playlist_output_dir.to_str().unwrap().to_string(), + playlist_path: playlist_dir_str.clone(), video_path, }); }