Apps/ImageApi
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

feat: make file watcher, thumbnails, and upload library-aware

Browse Source 
`watch_files` and `create_thumbnails` now iterate every configured
library, tagging rows with the correct `library_id`. `process_new_files`
takes a `&Library` so InsertImageExif no longer hardcodes the primary
library. Upload accepts an optional `library` query param to pick a
target library; omitted still defaults to primary for backwards
compatibility.

Hash-keyed thumbnail/HLS storage with dual-lookup fallback is deferred
to Phase 5, where it's bundled with the content hash backfill that
actually makes the hash-keyed paths meaningful. Until hashes are
populated, the legacy mirrored layout is a no-op to change.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Cameron
2026-04-17 15:41:38 -04:00
committed by cameron
parent 48e5de6eab
commit ce5b337582
3 changed files with 164 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

270
src/main.rs
View File

@@ -290,10 +290,16 @@ async fn get_file_metadata(
}
}
#[derive(serde::Deserialize)]
struct UploadQuery {
library: Option<String>,
}
#[post("/image")]
async fn upload_image(
_: Claims,
request: HttpRequest,
query: web::Query<UploadQuery>,
mut payload: mp::Multipart,
app_state: Data<AppState>,
exif_dao: Data<Mutex<Box<dyn ExifDao>>>,
@@ -304,6 +310,20 @@ async fn upload_image(
let span_context =
opentelemetry::Context::new().with_remote_span_context(span.span_context().clone());
// Resolve the optional library selector. Absent → primary library
// (backwards-compatible with clients that don't yet send `library=`).
let target_library = match libraries::resolve_library_param(
&app_state,
query.library.as_deref(),
) {
Ok(Some(lib)) => lib,
Ok(None) => app_state.primary_library(),
Err(msg) => {
span.set_status(Status::error(msg.clone()));
return HttpResponse::BadRequest().body(msg);
}
};
let mut file_content: BytesMut = BytesMut::new();
let mut file_name: Option<String> = None;
let mut file_path: Option<String> = None;
@@ -333,7 +353,7 @@ async fn upload_image(
}
}
let path = file_path.unwrap_or_else(|| app_state.base_path.clone());
let path = file_path.unwrap_or_else(|| target_library.root_path.clone());
if !file_content.is_empty() {
if file_name.is_none() {
span.set_status(Status::error("No filename provided"));
@@ -341,7 +361,7 @@ async fn upload_image(
}
let full_path = PathBuf::from(&path).join(file_name.unwrap());
if let Some(full_path) = is_valid_full_path(
&app_state.base_path,
&target_library.root_path,
&full_path.to_str().unwrap().to_string(),
true,
) {
@@ -382,8 +402,8 @@ async fn upload_image(
// Extract and store EXIF data if file supports it
if exif::supports_exif(&uploaded_path) {
let relative_path = uploaded_path
.strip_prefix(&app_state.base_path)
.expect("Error stripping base path prefix")
.strip_prefix(&target_library.root_path)
.expect("Error stripping library root prefix")
.to_str()
.unwrap()
.to_string();
@@ -392,7 +412,7 @@ async fn upload_image(
Ok(exif_data) => {
let timestamp = Utc::now().timestamp();
let insert_exif = InsertImageExif {
library_id: crate::libraries::PRIMARY_LIBRARY_ID,
library_id: target_library.id,
file_path: relative_path.clone(),
camera_make: exif_data.camera_make,
camera_model: exif_data.camera_model,
@@ -920,78 +940,87 @@ async fn delete_favorite(
}
}
fn create_thumbnails() {
fn create_thumbnails(libs: &[libraries::Library]) {
let tracer = global_tracer();
let span = tracer.start("creating thumbnails");
let thumbs = &dotenv::var("THUMBNAILS").expect("THUMBNAILS not defined");
let thumbnail_directory: &Path = Path::new(thumbs);
let images = PathBuf::from(dotenv::var("BASE_PATH").unwrap());
for lib in libs {
info!(
"Scanning thumbnails for library '{}' at {}",
lib.name, lib.root_path
);
let images = PathBuf::from(&lib.root_path);
WalkDir::new(&images)
.into_iter()
.collect::<Vec<Result<_, _>>>()
.into_par_iter()
.filter_map(|entry| entry.ok())
.filter(|entry| entry.file_type().is_file())
.filter(|entry| {
if is_video(entry) {
let relative_path = &entry.path().strip_prefix(&images).unwrap();
WalkDir::new(&images)
.into_iter()
.collect::<Vec<Result<_, _>>>()
.into_par_iter()
.filter_map(|entry| entry.ok())
.filter(|entry| entry.file_type().is_file())
.filter(|entry| {
if is_video(entry) {
let relative_path = &entry.path().strip_prefix(&images).unwrap();
let thumb_path = Path::new(thumbnail_directory).join(relative_path);
std::fs::create_dir_all(
thumb_path
.parent()
.unwrap_or_else(|| panic!("Thumbnail {:?} has no parent?", thumb_path)),
)
.expect("Error creating directory");
let mut video_span = tracer.start_with_context(
"generate_video_thumbnail",
&opentelemetry::Context::new()
.with_remote_span_context(span.span_context().clone()),
);
video_span.set_attributes(vec![
KeyValue::new("type", "video"),
KeyValue::new("file-name", thumb_path.display().to_string()),
KeyValue::new("library", lib.name.clone()),
]);
debug!("Generating video thumbnail: {:?}", thumb_path);
generate_video_thumbnail(entry.path(), &thumb_path);
video_span.end();
false
} else {
is_image(entry)
}
})
.filter(|entry| {
let path = entry.path();
let relative_path = &path.strip_prefix(&images).unwrap();
let thumb_path = Path::new(thumbnail_directory).join(relative_path);
std::fs::create_dir_all(
thumb_path
.parent()
.unwrap_or_else(|| panic!("Thumbnail {:?} has no parent?", thumb_path)),
)
.expect("Error creating directory");
let mut video_span = tracer.start_with_context(
"generate_video_thumbnail",
&opentelemetry::Context::new()
.with_remote_span_context(span.span_context().clone()),
);
video_span.set_attributes(vec![
KeyValue::new("type", "video"),
KeyValue::new("file-name", thumb_path.display().to_string()),
]);
debug!("Generating video thumbnail: {:?}", thumb_path);
generate_video_thumbnail(entry.path(), &thumb_path);
video_span.end();
false
} else {
is_image(entry)
}
})
.filter(|entry| {
let path = entry.path();
let relative_path = &path.strip_prefix(&images).unwrap();
let thumb_path = Path::new(thumbnail_directory).join(relative_path);
!thumb_path.exists()
})
.map(|entry| (image::open(entry.path()), entry.path().to_path_buf()))
.filter(|(img, path)| {
if let Err(e) = img {
error!("Unable to open image: {:?}. {}", path, e);
}
img.is_ok()
})
.map(|(img, path)| (img.unwrap(), path))
.map(|(image, path)| (image.thumbnail(200, u32::MAX), path))
.map(|(image, path)| {
let relative_path = &path.strip_prefix(&images).unwrap();
let thumb_path = Path::new(thumbnail_directory).join(relative_path);
std::fs::create_dir_all(thumb_path.parent().unwrap())
.expect("There was an issue creating directory");
info!("Saving thumbnail: {:?}", thumb_path);
image.save(thumb_path).expect("Failure saving thumbnail");
})
.for_each(drop);
!thumb_path.exists()
})
.map(|entry| (image::open(entry.path()), entry.path().to_path_buf()))
.filter(|(img, path)| {
if let Err(e) = img {
error!("Unable to open image: {:?}. {}", path, e);
}
img.is_ok()
})
.map(|(img, path)| (img.unwrap(), path))
.map(|(image, path)| (image.thumbnail(200, u32::MAX), path))
.map(|(image, path)| {
let relative_path = &path.strip_prefix(&images).unwrap();
let thumb_path = Path::new(thumbnail_directory).join(relative_path);
std::fs::create_dir_all(thumb_path.parent().unwrap())
.expect("There was an issue creating directory");
info!("Saving thumbnail: {:?}", thumb_path);
image.save(thumb_path).expect("Failure saving thumbnail");
})
.for_each(drop);
}
debug!("Finished making thumbnails");
update_media_counts(&images);
for lib in libs {
update_media_counts(Path::new(&lib.root_path));
}
}
fn update_media_counts(media_dir: &Path) {
@@ -1039,11 +1068,13 @@ fn main() -> std::io::Result<()> {
otel::init_tracing();
}
create_thumbnails();
// generate_video_gifs().await;
// AppState construction loads (and seeds if needed) the libraries
// table; we use that list to drive the initial thumbnail sweep.
let app_data = Data::new(AppState::default());
create_thumbnails(&app_data.libraries);
// generate_video_gifs().await;
let labels = HashMap::new();
let prometheus = PrometheusMetricsBuilder::new("api")
.const_labels(labels)
@@ -1060,14 +1091,20 @@ fn main() -> std::io::Result<()> {
.unwrap();
let app_state = app_data.clone();
app_state.playlist_manager.do_send(ScanDirectoryMessage {
directory: app_state.base_path.clone(),
});
for lib in &app_state.libraries {
app_state.playlist_manager.do_send(ScanDirectoryMessage {
directory: lib.root_path.clone(),
});
}
// Start file watcher with playlist manager and preview generator
let playlist_mgr_for_watcher = app_state.playlist_manager.as_ref().clone();
let preview_gen_for_watcher = app_state.preview_clip_generator.as_ref().clone();
watch_files(playlist_mgr_for_watcher, preview_gen_for_watcher);
watch_files(
app_state.libraries.clone(),
playlist_mgr_for_watcher,
preview_gen_for_watcher,
);
// Start orphaned playlist cleanup job
cleanup_orphaned_playlists();
@@ -1376,13 +1413,11 @@ fn cleanup_orphaned_playlists() {
}
fn watch_files(
libs: Vec<libraries::Library>,
playlist_manager: Addr<VideoPlaylistManager>,
preview_generator: Addr<video::actors::PreviewClipGenerator>,
) {
std::thread::spawn(move || {
let base_str = dotenv::var("BASE_PATH").unwrap();
let base_path = PathBuf::from(&base_str);
// Get polling intervals from environment variables
// Quick scan: Check recently modified files (default: 60 seconds)
let quick_interval_secs = dotenv::var("WATCH_QUICK_INTERVAL_SECONDS")
@@ -1399,7 +1434,12 @@ fn watch_files(
info!("Starting optimized file watcher");
info!(" Quick scan interval: {} seconds", quick_interval_secs);
info!(" Full scan interval: {} seconds", full_interval_secs);
info!(" Watching directory: {}", base_str);
for lib in &libs {
info!(
" Watching library '{}' (id={}) at {}",
lib.name, lib.id, lib.root_path
);
}
// Create DAOs for tracking processed files
let exif_dao = Arc::new(Mutex::new(
@@ -1423,41 +1463,48 @@ fn watch_files(
let is_full_scan = since_last_full.as_secs() >= full_interval_secs;
if is_full_scan {
info!("Running full scan (scan #{})", scan_count);
process_new_files(
&base_path,
Arc::clone(&exif_dao),
Arc::clone(&preview_dao),
None,
playlist_manager.clone(),
preview_generator.clone(),
);
last_full_scan = now;
} else {
debug!(
"Running quick scan (checking files modified in last {} seconds)",
quick_interval_secs + 10
);
// Check files modified since last quick scan, plus 10 second buffer
let check_since = last_quick_scan
.checked_sub(Duration::from_secs(10))
.unwrap_or(last_quick_scan);
process_new_files(
&base_path,
Arc::clone(&exif_dao),
Arc::clone(&preview_dao),
Some(check_since),
playlist_manager.clone(),
preview_generator.clone(),
);
for lib in &libs {
if is_full_scan {
info!(
"Running full scan for library '{}' (scan #{})",
lib.name, scan_count
);
process_new_files(
lib,
Arc::clone(&exif_dao),
Arc::clone(&preview_dao),
None,
playlist_manager.clone(),
preview_generator.clone(),
);
} else {
debug!(
"Running quick scan for library '{}' (checking files modified in last {} seconds)",
lib.name,
quick_interval_secs + 10
);
let check_since = last_quick_scan
.checked_sub(Duration::from_secs(10))
.unwrap_or(last_quick_scan);
process_new_files(
lib,
Arc::clone(&exif_dao),
Arc::clone(&preview_dao),
Some(check_since),
playlist_manager.clone(),
preview_generator.clone(),
);
}
// Update media counts per library (metric aggregates across all)
update_media_counts(Path::new(&lib.root_path));
}
if is_full_scan {
last_full_scan = now;
}
last_quick_scan = now;
scan_count += 1;
// Update media counts
update_media_counts(&base_path);
}
});
}
@@ -1486,7 +1533,7 @@ fn playlist_needs_generation(video_path: &Path, playlist_path: &Path) -> bool {
}
fn process_new_files(
base_path: &Path,
library: &libraries::Library,
exif_dao: Arc<Mutex<Box<dyn ExifDao>>>,
preview_dao: Arc<Mutex<Box<dyn PreviewDao>>>,
modified_since: Option<SystemTime>,
@@ -1496,6 +1543,7 @@ fn process_new_files(
let context = opentelemetry::Context::new();
let thumbs = dotenv::var("THUMBNAILS").expect("THUMBNAILS not defined");
let thumbnail_directory = Path::new(&thumbs);
let base_path = Path::new(&library.root_path);
// Collect all image and video files, optionally filtered by modification time
let files: Vec<(PathBuf, String)> = WalkDir::new(base_path)
@@ -1592,7 +1640,7 @@ fn process_new_files(
Ok(exif_data) => {
let timestamp = Utc::now().timestamp();
let insert_exif = InsertImageExif {
library_id: crate::libraries::PRIMARY_LIBRARY_ID,
library_id: library.id,
file_path: relative_path.clone(),
camera_make: exif_data.camera_make,
camera_model: exif_data.camera_model,
@@ -1710,7 +1758,7 @@ fn process_new_files(
// Generate thumbnails for all files that need them
if new_files_found {
info!("Processing thumbnails for new files...");
create_thumbnails();
create_thumbnails(std::slice::from_ref(library));
}
}