From ce5b337582c3157f7833f74875727c04837ecdcf Mon Sep 17 00:00:00 2001 From: Cameron Date: Fri, 17 Apr 2026 15:41:38 -0400 Subject: [PATCH] feat: make file watcher, thumbnails, and upload library-aware `watch_files` and `create_thumbnails` now iterate every configured library, tagging rows with the correct `library_id`. `process_new_files` takes a `&Library` so InsertImageExif no longer hardcodes the primary library. Upload accepts an optional `library` query param to pick a target library; omitted still defaults to primary for backwards compatibility. Hash-keyed thumbnail/HLS storage with dual-lookup fallback is deferred to Phase 5, where it's bundled with the content hash backfill that actually makes the hash-keyed paths meaningful. Until hashes are populated, the legacy mirrored layout is a no-op to change. Co-Authored-By: Claude Opus 4.7 --- src/files.rs | 5 +- src/lib.rs | 2 +- src/main.rs | 270 ++++++++++++++++++++++++++++++--------------------- 3 files changed, 164 insertions(+), 113 deletions(-) diff --git a/src/files.rs b/src/files.rs index 224c801..3c25597 100644 --- a/src/files.rs +++ b/src/files.rs @@ -1159,7 +1159,10 @@ impl Handler for StreamActor { let tracer = global_tracer(); let _ = tracer.start("RefreshThumbnailsMessage"); info!("Refreshing thumbnails after upload"); - create_thumbnails() + // The stub in lib.rs is a no-op; the real generation is driven by + // the file watcher tick in main.rs, which has access to the + // configured libraries. + create_thumbnails(&[]) } } diff --git a/src/lib.rs b/src/lib.rs index 9d785fe..12e0bc0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,7 +33,7 @@ pub use state::AppState; use std::path::Path; use walkdir::DirEntry; -pub fn create_thumbnails() { +pub fn create_thumbnails(_libs: &[libraries::Library]) { // Stub - implemented in main.rs } diff --git a/src/main.rs b/src/main.rs index 044d1a3..cec0474 100644 --- a/src/main.rs +++ b/src/main.rs @@ -290,10 +290,16 @@ async fn get_file_metadata( } } +#[derive(serde::Deserialize)] +struct UploadQuery { + library: Option, +} + #[post("/image")] async fn upload_image( _: Claims, request: HttpRequest, + query: web::Query, mut payload: mp::Multipart, app_state: Data, exif_dao: Data>>, @@ -304,6 +310,20 @@ async fn upload_image( let span_context = opentelemetry::Context::new().with_remote_span_context(span.span_context().clone()); + // Resolve the optional library selector. Absent → primary library + // (backwards-compatible with clients that don't yet send `library=`). + let target_library = match libraries::resolve_library_param( + &app_state, + query.library.as_deref(), + ) { + Ok(Some(lib)) => lib, + Ok(None) => app_state.primary_library(), + Err(msg) => { + span.set_status(Status::error(msg.clone())); + return HttpResponse::BadRequest().body(msg); + } + }; + let mut file_content: BytesMut = BytesMut::new(); let mut file_name: Option = None; let mut file_path: Option = None; @@ -333,7 +353,7 @@ async fn upload_image( } } - let path = file_path.unwrap_or_else(|| app_state.base_path.clone()); + let path = file_path.unwrap_or_else(|| target_library.root_path.clone()); if !file_content.is_empty() { if file_name.is_none() { span.set_status(Status::error("No filename provided")); @@ -341,7 +361,7 @@ async fn upload_image( } let full_path = PathBuf::from(&path).join(file_name.unwrap()); if let Some(full_path) = is_valid_full_path( - &app_state.base_path, + &target_library.root_path, &full_path.to_str().unwrap().to_string(), true, ) { @@ -382,8 +402,8 @@ async fn upload_image( // Extract and store EXIF data if file supports it if exif::supports_exif(&uploaded_path) { let relative_path = uploaded_path - .strip_prefix(&app_state.base_path) - .expect("Error stripping base path prefix") + .strip_prefix(&target_library.root_path) + .expect("Error stripping library root prefix") .to_str() .unwrap() .to_string(); @@ -392,7 +412,7 @@ async fn upload_image( Ok(exif_data) => { let timestamp = Utc::now().timestamp(); let insert_exif = InsertImageExif { - library_id: crate::libraries::PRIMARY_LIBRARY_ID, + library_id: target_library.id, file_path: relative_path.clone(), camera_make: exif_data.camera_make, camera_model: exif_data.camera_model, @@ -920,78 +940,87 @@ async fn delete_favorite( } } -fn create_thumbnails() { +fn create_thumbnails(libs: &[libraries::Library]) { let tracer = global_tracer(); let span = tracer.start("creating thumbnails"); let thumbs = &dotenv::var("THUMBNAILS").expect("THUMBNAILS not defined"); let thumbnail_directory: &Path = Path::new(thumbs); - let images = PathBuf::from(dotenv::var("BASE_PATH").unwrap()); + for lib in libs { + info!( + "Scanning thumbnails for library '{}' at {}", + lib.name, lib.root_path + ); + let images = PathBuf::from(&lib.root_path); - WalkDir::new(&images) - .into_iter() - .collect::>>() - .into_par_iter() - .filter_map(|entry| entry.ok()) - .filter(|entry| entry.file_type().is_file()) - .filter(|entry| { - if is_video(entry) { - let relative_path = &entry.path().strip_prefix(&images).unwrap(); + WalkDir::new(&images) + .into_iter() + .collect::>>() + .into_par_iter() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.file_type().is_file()) + .filter(|entry| { + if is_video(entry) { + let relative_path = &entry.path().strip_prefix(&images).unwrap(); + let thumb_path = Path::new(thumbnail_directory).join(relative_path); + std::fs::create_dir_all( + thumb_path + .parent() + .unwrap_or_else(|| panic!("Thumbnail {:?} has no parent?", thumb_path)), + ) + .expect("Error creating directory"); + + let mut video_span = tracer.start_with_context( + "generate_video_thumbnail", + &opentelemetry::Context::new() + .with_remote_span_context(span.span_context().clone()), + ); + video_span.set_attributes(vec![ + KeyValue::new("type", "video"), + KeyValue::new("file-name", thumb_path.display().to_string()), + KeyValue::new("library", lib.name.clone()), + ]); + + debug!("Generating video thumbnail: {:?}", thumb_path); + generate_video_thumbnail(entry.path(), &thumb_path); + video_span.end(); + false + } else { + is_image(entry) + } + }) + .filter(|entry| { + let path = entry.path(); + let relative_path = &path.strip_prefix(&images).unwrap(); let thumb_path = Path::new(thumbnail_directory).join(relative_path); - std::fs::create_dir_all( - thumb_path - .parent() - .unwrap_or_else(|| panic!("Thumbnail {:?} has no parent?", thumb_path)), - ) - .expect("Error creating directory"); - - let mut video_span = tracer.start_with_context( - "generate_video_thumbnail", - &opentelemetry::Context::new() - .with_remote_span_context(span.span_context().clone()), - ); - video_span.set_attributes(vec![ - KeyValue::new("type", "video"), - KeyValue::new("file-name", thumb_path.display().to_string()), - ]); - - debug!("Generating video thumbnail: {:?}", thumb_path); - generate_video_thumbnail(entry.path(), &thumb_path); - video_span.end(); - false - } else { - is_image(entry) - } - }) - .filter(|entry| { - let path = entry.path(); - let relative_path = &path.strip_prefix(&images).unwrap(); - let thumb_path = Path::new(thumbnail_directory).join(relative_path); - !thumb_path.exists() - }) - .map(|entry| (image::open(entry.path()), entry.path().to_path_buf())) - .filter(|(img, path)| { - if let Err(e) = img { - error!("Unable to open image: {:?}. {}", path, e); - } - img.is_ok() - }) - .map(|(img, path)| (img.unwrap(), path)) - .map(|(image, path)| (image.thumbnail(200, u32::MAX), path)) - .map(|(image, path)| { - let relative_path = &path.strip_prefix(&images).unwrap(); - let thumb_path = Path::new(thumbnail_directory).join(relative_path); - std::fs::create_dir_all(thumb_path.parent().unwrap()) - .expect("There was an issue creating directory"); - info!("Saving thumbnail: {:?}", thumb_path); - image.save(thumb_path).expect("Failure saving thumbnail"); - }) - .for_each(drop); + !thumb_path.exists() + }) + .map(|entry| (image::open(entry.path()), entry.path().to_path_buf())) + .filter(|(img, path)| { + if let Err(e) = img { + error!("Unable to open image: {:?}. {}", path, e); + } + img.is_ok() + }) + .map(|(img, path)| (img.unwrap(), path)) + .map(|(image, path)| (image.thumbnail(200, u32::MAX), path)) + .map(|(image, path)| { + let relative_path = &path.strip_prefix(&images).unwrap(); + let thumb_path = Path::new(thumbnail_directory).join(relative_path); + std::fs::create_dir_all(thumb_path.parent().unwrap()) + .expect("There was an issue creating directory"); + info!("Saving thumbnail: {:?}", thumb_path); + image.save(thumb_path).expect("Failure saving thumbnail"); + }) + .for_each(drop); + } debug!("Finished making thumbnails"); - update_media_counts(&images); + for lib in libs { + update_media_counts(Path::new(&lib.root_path)); + } } fn update_media_counts(media_dir: &Path) { @@ -1039,11 +1068,13 @@ fn main() -> std::io::Result<()> { otel::init_tracing(); } - create_thumbnails(); - // generate_video_gifs().await; - + // AppState construction loads (and seeds if needed) the libraries + // table; we use that list to drive the initial thumbnail sweep. let app_data = Data::new(AppState::default()); + create_thumbnails(&app_data.libraries); + // generate_video_gifs().await; + let labels = HashMap::new(); let prometheus = PrometheusMetricsBuilder::new("api") .const_labels(labels) @@ -1060,14 +1091,20 @@ fn main() -> std::io::Result<()> { .unwrap(); let app_state = app_data.clone(); - app_state.playlist_manager.do_send(ScanDirectoryMessage { - directory: app_state.base_path.clone(), - }); + for lib in &app_state.libraries { + app_state.playlist_manager.do_send(ScanDirectoryMessage { + directory: lib.root_path.clone(), + }); + } // Start file watcher with playlist manager and preview generator let playlist_mgr_for_watcher = app_state.playlist_manager.as_ref().clone(); let preview_gen_for_watcher = app_state.preview_clip_generator.as_ref().clone(); - watch_files(playlist_mgr_for_watcher, preview_gen_for_watcher); + watch_files( + app_state.libraries.clone(), + playlist_mgr_for_watcher, + preview_gen_for_watcher, + ); // Start orphaned playlist cleanup job cleanup_orphaned_playlists(); @@ -1376,13 +1413,11 @@ fn cleanup_orphaned_playlists() { } fn watch_files( + libs: Vec, playlist_manager: Addr, preview_generator: Addr, ) { std::thread::spawn(move || { - let base_str = dotenv::var("BASE_PATH").unwrap(); - let base_path = PathBuf::from(&base_str); - // Get polling intervals from environment variables // Quick scan: Check recently modified files (default: 60 seconds) let quick_interval_secs = dotenv::var("WATCH_QUICK_INTERVAL_SECONDS") @@ -1399,7 +1434,12 @@ fn watch_files( info!("Starting optimized file watcher"); info!(" Quick scan interval: {} seconds", quick_interval_secs); info!(" Full scan interval: {} seconds", full_interval_secs); - info!(" Watching directory: {}", base_str); + for lib in &libs { + info!( + " Watching library '{}' (id={}) at {}", + lib.name, lib.id, lib.root_path + ); + } // Create DAOs for tracking processed files let exif_dao = Arc::new(Mutex::new( @@ -1423,41 +1463,48 @@ fn watch_files( let is_full_scan = since_last_full.as_secs() >= full_interval_secs; - if is_full_scan { - info!("Running full scan (scan #{})", scan_count); - process_new_files( - &base_path, - Arc::clone(&exif_dao), - Arc::clone(&preview_dao), - None, - playlist_manager.clone(), - preview_generator.clone(), - ); - last_full_scan = now; - } else { - debug!( - "Running quick scan (checking files modified in last {} seconds)", - quick_interval_secs + 10 - ); - // Check files modified since last quick scan, plus 10 second buffer - let check_since = last_quick_scan - .checked_sub(Duration::from_secs(10)) - .unwrap_or(last_quick_scan); - process_new_files( - &base_path, - Arc::clone(&exif_dao), - Arc::clone(&preview_dao), - Some(check_since), - playlist_manager.clone(), - preview_generator.clone(), - ); + for lib in &libs { + if is_full_scan { + info!( + "Running full scan for library '{}' (scan #{})", + lib.name, scan_count + ); + process_new_files( + lib, + Arc::clone(&exif_dao), + Arc::clone(&preview_dao), + None, + playlist_manager.clone(), + preview_generator.clone(), + ); + } else { + debug!( + "Running quick scan for library '{}' (checking files modified in last {} seconds)", + lib.name, + quick_interval_secs + 10 + ); + let check_since = last_quick_scan + .checked_sub(Duration::from_secs(10)) + .unwrap_or(last_quick_scan); + process_new_files( + lib, + Arc::clone(&exif_dao), + Arc::clone(&preview_dao), + Some(check_since), + playlist_manager.clone(), + preview_generator.clone(), + ); + } + + // Update media counts per library (metric aggregates across all) + update_media_counts(Path::new(&lib.root_path)); } + if is_full_scan { + last_full_scan = now; + } last_quick_scan = now; scan_count += 1; - - // Update media counts - update_media_counts(&base_path); } }); } @@ -1486,7 +1533,7 @@ fn playlist_needs_generation(video_path: &Path, playlist_path: &Path) -> bool { } fn process_new_files( - base_path: &Path, + library: &libraries::Library, exif_dao: Arc>>, preview_dao: Arc>>, modified_since: Option, @@ -1496,6 +1543,7 @@ fn process_new_files( let context = opentelemetry::Context::new(); let thumbs = dotenv::var("THUMBNAILS").expect("THUMBNAILS not defined"); let thumbnail_directory = Path::new(&thumbs); + let base_path = Path::new(&library.root_path); // Collect all image and video files, optionally filtered by modification time let files: Vec<(PathBuf, String)> = WalkDir::new(base_path) @@ -1592,7 +1640,7 @@ fn process_new_files( Ok(exif_data) => { let timestamp = Utc::now().timestamp(); let insert_exif = InsertImageExif { - library_id: crate::libraries::PRIMARY_LIBRARY_ID, + library_id: library.id, file_path: relative_path.clone(), camera_make: exif_data.camera_make, camera_model: exif_data.camera_model, @@ -1710,7 +1758,7 @@ fn process_new_files( // Generate thumbnails for all files that need them if new_files_found { info!("Processing thumbnails for new files..."); - create_thumbnails(); + create_thumbnails(std::slice::from_ref(library)); } }