004 Multi-library Support #54

Merged
cameron merged 19 commits from 004-multi-library into master 2026-04-21 01:55:23 +00:00
3 changed files with 164 additions and 113 deletions
Showing only changes of commit ce5b337582 - Show all commits

View File

@@ -1159,7 +1159,10 @@ impl Handler<RefreshThumbnailsMessage> for StreamActor {
let tracer = global_tracer(); let tracer = global_tracer();
let _ = tracer.start("RefreshThumbnailsMessage"); let _ = tracer.start("RefreshThumbnailsMessage");
info!("Refreshing thumbnails after upload"); info!("Refreshing thumbnails after upload");
create_thumbnails() // The stub in lib.rs is a no-op; the real generation is driven by
// the file watcher tick in main.rs, which has access to the
// configured libraries.
create_thumbnails(&[])
} }
} }

View File

@@ -33,7 +33,7 @@ pub use state::AppState;
use std::path::Path; use std::path::Path;
use walkdir::DirEntry; use walkdir::DirEntry;
pub fn create_thumbnails() { pub fn create_thumbnails(_libs: &[libraries::Library]) {
// Stub - implemented in main.rs // Stub - implemented in main.rs
} }

View File

@@ -290,10 +290,16 @@ async fn get_file_metadata(
} }
} }
#[derive(serde::Deserialize)]
struct UploadQuery {
library: Option<String>,
}
#[post("/image")] #[post("/image")]
async fn upload_image( async fn upload_image(
_: Claims, _: Claims,
request: HttpRequest, request: HttpRequest,
query: web::Query<UploadQuery>,
mut payload: mp::Multipart, mut payload: mp::Multipart,
app_state: Data<AppState>, app_state: Data<AppState>,
exif_dao: Data<Mutex<Box<dyn ExifDao>>>, exif_dao: Data<Mutex<Box<dyn ExifDao>>>,
@@ -304,6 +310,20 @@ async fn upload_image(
let span_context = let span_context =
opentelemetry::Context::new().with_remote_span_context(span.span_context().clone()); opentelemetry::Context::new().with_remote_span_context(span.span_context().clone());
// Resolve the optional library selector. Absent → primary library
// (backwards-compatible with clients that don't yet send `library=`).
let target_library = match libraries::resolve_library_param(
&app_state,
query.library.as_deref(),
) {
Ok(Some(lib)) => lib,
Ok(None) => app_state.primary_library(),
Err(msg) => {
span.set_status(Status::error(msg.clone()));
return HttpResponse::BadRequest().body(msg);
}
};
let mut file_content: BytesMut = BytesMut::new(); let mut file_content: BytesMut = BytesMut::new();
let mut file_name: Option<String> = None; let mut file_name: Option<String> = None;
let mut file_path: Option<String> = None; let mut file_path: Option<String> = None;
@@ -333,7 +353,7 @@ async fn upload_image(
} }
} }
let path = file_path.unwrap_or_else(|| app_state.base_path.clone()); let path = file_path.unwrap_or_else(|| target_library.root_path.clone());
if !file_content.is_empty() { if !file_content.is_empty() {
if file_name.is_none() { if file_name.is_none() {
span.set_status(Status::error("No filename provided")); span.set_status(Status::error("No filename provided"));
@@ -341,7 +361,7 @@ async fn upload_image(
} }
let full_path = PathBuf::from(&path).join(file_name.unwrap()); let full_path = PathBuf::from(&path).join(file_name.unwrap());
if let Some(full_path) = is_valid_full_path( if let Some(full_path) = is_valid_full_path(
&app_state.base_path, &target_library.root_path,
&full_path.to_str().unwrap().to_string(), &full_path.to_str().unwrap().to_string(),
true, true,
) { ) {
@@ -382,8 +402,8 @@ async fn upload_image(
// Extract and store EXIF data if file supports it // Extract and store EXIF data if file supports it
if exif::supports_exif(&uploaded_path) { if exif::supports_exif(&uploaded_path) {
let relative_path = uploaded_path let relative_path = uploaded_path
.strip_prefix(&app_state.base_path) .strip_prefix(&target_library.root_path)
.expect("Error stripping base path prefix") .expect("Error stripping library root prefix")
.to_str() .to_str()
.unwrap() .unwrap()
.to_string(); .to_string();
@@ -392,7 +412,7 @@ async fn upload_image(
Ok(exif_data) => { Ok(exif_data) => {
let timestamp = Utc::now().timestamp(); let timestamp = Utc::now().timestamp();
let insert_exif = InsertImageExif { let insert_exif = InsertImageExif {
library_id: crate::libraries::PRIMARY_LIBRARY_ID, library_id: target_library.id,
file_path: relative_path.clone(), file_path: relative_path.clone(),
camera_make: exif_data.camera_make, camera_make: exif_data.camera_make,
camera_model: exif_data.camera_model, camera_model: exif_data.camera_model,
@@ -920,14 +940,19 @@ async fn delete_favorite(
} }
} }
fn create_thumbnails() { fn create_thumbnails(libs: &[libraries::Library]) {
let tracer = global_tracer(); let tracer = global_tracer();
let span = tracer.start("creating thumbnails"); let span = tracer.start("creating thumbnails");
let thumbs = &dotenv::var("THUMBNAILS").expect("THUMBNAILS not defined"); let thumbs = &dotenv::var("THUMBNAILS").expect("THUMBNAILS not defined");
let thumbnail_directory: &Path = Path::new(thumbs); let thumbnail_directory: &Path = Path::new(thumbs);
let images = PathBuf::from(dotenv::var("BASE_PATH").unwrap()); for lib in libs {
info!(
"Scanning thumbnails for library '{}' at {}",
lib.name, lib.root_path
);
let images = PathBuf::from(&lib.root_path);
WalkDir::new(&images) WalkDir::new(&images)
.into_iter() .into_iter()
@@ -954,6 +979,7 @@ fn create_thumbnails() {
video_span.set_attributes(vec![ video_span.set_attributes(vec![
KeyValue::new("type", "video"), KeyValue::new("type", "video"),
KeyValue::new("file-name", thumb_path.display().to_string()), KeyValue::new("file-name", thumb_path.display().to_string()),
KeyValue::new("library", lib.name.clone()),
]); ]);
debug!("Generating video thumbnail: {:?}", thumb_path); debug!("Generating video thumbnail: {:?}", thumb_path);
@@ -988,10 +1014,13 @@ fn create_thumbnails() {
image.save(thumb_path).expect("Failure saving thumbnail"); image.save(thumb_path).expect("Failure saving thumbnail");
}) })
.for_each(drop); .for_each(drop);
}
debug!("Finished making thumbnails"); debug!("Finished making thumbnails");
update_media_counts(&images); for lib in libs {
update_media_counts(Path::new(&lib.root_path));
}
} }
fn update_media_counts(media_dir: &Path) { fn update_media_counts(media_dir: &Path) {
@@ -1039,11 +1068,13 @@ fn main() -> std::io::Result<()> {
otel::init_tracing(); otel::init_tracing();
} }
create_thumbnails(); // AppState construction loads (and seeds if needed) the libraries
// generate_video_gifs().await; // table; we use that list to drive the initial thumbnail sweep.
let app_data = Data::new(AppState::default()); let app_data = Data::new(AppState::default());
create_thumbnails(&app_data.libraries);
// generate_video_gifs().await;
let labels = HashMap::new(); let labels = HashMap::new();
let prometheus = PrometheusMetricsBuilder::new("api") let prometheus = PrometheusMetricsBuilder::new("api")
.const_labels(labels) .const_labels(labels)
@@ -1060,14 +1091,20 @@ fn main() -> std::io::Result<()> {
.unwrap(); .unwrap();
let app_state = app_data.clone(); let app_state = app_data.clone();
for lib in &app_state.libraries {
app_state.playlist_manager.do_send(ScanDirectoryMessage { app_state.playlist_manager.do_send(ScanDirectoryMessage {
directory: app_state.base_path.clone(), directory: lib.root_path.clone(),
}); });
}
// Start file watcher with playlist manager and preview generator // Start file watcher with playlist manager and preview generator
let playlist_mgr_for_watcher = app_state.playlist_manager.as_ref().clone(); let playlist_mgr_for_watcher = app_state.playlist_manager.as_ref().clone();
let preview_gen_for_watcher = app_state.preview_clip_generator.as_ref().clone(); let preview_gen_for_watcher = app_state.preview_clip_generator.as_ref().clone();
watch_files(playlist_mgr_for_watcher, preview_gen_for_watcher); watch_files(
app_state.libraries.clone(),
playlist_mgr_for_watcher,
preview_gen_for_watcher,
);
// Start orphaned playlist cleanup job // Start orphaned playlist cleanup job
cleanup_orphaned_playlists(); cleanup_orphaned_playlists();
@@ -1376,13 +1413,11 @@ fn cleanup_orphaned_playlists() {
} }
fn watch_files( fn watch_files(
libs: Vec<libraries::Library>,
playlist_manager: Addr<VideoPlaylistManager>, playlist_manager: Addr<VideoPlaylistManager>,
preview_generator: Addr<video::actors::PreviewClipGenerator>, preview_generator: Addr<video::actors::PreviewClipGenerator>,
) { ) {
std::thread::spawn(move || { std::thread::spawn(move || {
let base_str = dotenv::var("BASE_PATH").unwrap();
let base_path = PathBuf::from(&base_str);
// Get polling intervals from environment variables // Get polling intervals from environment variables
// Quick scan: Check recently modified files (default: 60 seconds) // Quick scan: Check recently modified files (default: 60 seconds)
let quick_interval_secs = dotenv::var("WATCH_QUICK_INTERVAL_SECONDS") let quick_interval_secs = dotenv::var("WATCH_QUICK_INTERVAL_SECONDS")
@@ -1399,7 +1434,12 @@ fn watch_files(
info!("Starting optimized file watcher"); info!("Starting optimized file watcher");
info!(" Quick scan interval: {} seconds", quick_interval_secs); info!(" Quick scan interval: {} seconds", quick_interval_secs);
info!(" Full scan interval: {} seconds", full_interval_secs); info!(" Full scan interval: {} seconds", full_interval_secs);
info!(" Watching directory: {}", base_str); for lib in &libs {
info!(
" Watching library '{}' (id={}) at {}",
lib.name, lib.id, lib.root_path
);
}
// Create DAOs for tracking processed files // Create DAOs for tracking processed files
let exif_dao = Arc::new(Mutex::new( let exif_dao = Arc::new(Mutex::new(
@@ -1423,28 +1463,31 @@ fn watch_files(
let is_full_scan = since_last_full.as_secs() >= full_interval_secs; let is_full_scan = since_last_full.as_secs() >= full_interval_secs;
for lib in &libs {
if is_full_scan { if is_full_scan {
info!("Running full scan (scan #{})", scan_count); info!(
"Running full scan for library '{}' (scan #{})",
lib.name, scan_count
);
process_new_files( process_new_files(
&base_path, lib,
Arc::clone(&exif_dao), Arc::clone(&exif_dao),
Arc::clone(&preview_dao), Arc::clone(&preview_dao),
None, None,
playlist_manager.clone(), playlist_manager.clone(),
preview_generator.clone(), preview_generator.clone(),
); );
last_full_scan = now;
} else { } else {
debug!( debug!(
"Running quick scan (checking files modified in last {} seconds)", "Running quick scan for library '{}' (checking files modified in last {} seconds)",
lib.name,
quick_interval_secs + 10 quick_interval_secs + 10
); );
// Check files modified since last quick scan, plus 10 second buffer
let check_since = last_quick_scan let check_since = last_quick_scan
.checked_sub(Duration::from_secs(10)) .checked_sub(Duration::from_secs(10))
.unwrap_or(last_quick_scan); .unwrap_or(last_quick_scan);
process_new_files( process_new_files(
&base_path, lib,
Arc::clone(&exif_dao), Arc::clone(&exif_dao),
Arc::clone(&preview_dao), Arc::clone(&preview_dao),
Some(check_since), Some(check_since),
@@ -1453,11 +1496,15 @@ fn watch_files(
); );
} }
// Update media counts per library (metric aggregates across all)
update_media_counts(Path::new(&lib.root_path));
}
if is_full_scan {
last_full_scan = now;
}
last_quick_scan = now; last_quick_scan = now;
scan_count += 1; scan_count += 1;
// Update media counts
update_media_counts(&base_path);
} }
}); });
} }
@@ -1486,7 +1533,7 @@ fn playlist_needs_generation(video_path: &Path, playlist_path: &Path) -> bool {
} }
fn process_new_files( fn process_new_files(
base_path: &Path, library: &libraries::Library,
exif_dao: Arc<Mutex<Box<dyn ExifDao>>>, exif_dao: Arc<Mutex<Box<dyn ExifDao>>>,
preview_dao: Arc<Mutex<Box<dyn PreviewDao>>>, preview_dao: Arc<Mutex<Box<dyn PreviewDao>>>,
modified_since: Option<SystemTime>, modified_since: Option<SystemTime>,
@@ -1496,6 +1543,7 @@ fn process_new_files(
let context = opentelemetry::Context::new(); let context = opentelemetry::Context::new();
let thumbs = dotenv::var("THUMBNAILS").expect("THUMBNAILS not defined"); let thumbs = dotenv::var("THUMBNAILS").expect("THUMBNAILS not defined");
let thumbnail_directory = Path::new(&thumbs); let thumbnail_directory = Path::new(&thumbs);
let base_path = Path::new(&library.root_path);
// Collect all image and video files, optionally filtered by modification time // Collect all image and video files, optionally filtered by modification time
let files: Vec<(PathBuf, String)> = WalkDir::new(base_path) let files: Vec<(PathBuf, String)> = WalkDir::new(base_path)
@@ -1592,7 +1640,7 @@ fn process_new_files(
Ok(exif_data) => { Ok(exif_data) => {
let timestamp = Utc::now().timestamp(); let timestamp = Utc::now().timestamp();
let insert_exif = InsertImageExif { let insert_exif = InsertImageExif {
library_id: crate::libraries::PRIMARY_LIBRARY_ID, library_id: library.id,
file_path: relative_path.clone(), file_path: relative_path.clone(),
camera_make: exif_data.camera_make, camera_make: exif_data.camera_make,
camera_model: exif_data.camera_model, camera_model: exif_data.camera_model,
@@ -1710,7 +1758,7 @@ fn process_new_files(
// Generate thumbnails for all files that need them // Generate thumbnails for all files that need them
if new_files_found { if new_files_found {
info!("Processing thumbnails for new files..."); info!("Processing thumbnails for new files...");
create_thumbnails(); create_thumbnails(std::slice::from_ref(library));
} }
} }