refactor(watcher): extract register_new_files for targeted ingest
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Cameron CordesandClaude Opus 5.5 committed
1 parent
4105430681
commit
ebed15bd59
1 file changed
+108
-91
+108
-91
@@ -740,100 +740,19 @@ pub fn playlist_needs_generation(video_path: &Path, playlist_path: &Path) -> boo
|
||||
true
|
||||
}
|
||||
|
||||
pub fn process_new_files(
|
||||
/// Register files in `image_exif` (hash, perceptual hashes, EXIF, resolved
|
||||
/// date). Returns the `(absolute, rel_path)` pairs whose row was written.
|
||||
/// Shared by the watcher tick and the targeted ingest after an Enhance
|
||||
/// approval (#192).
|
||||
pub fn register_new_files(
|
||||
library: &libraries::Library,
|
||||
exif_dao: Arc<Mutex<Box<dyn ExifDao>>>,
|
||||
preview_dao: Arc<Mutex<Box<dyn PreviewDao>>>,
|
||||
face_dao: Arc<Mutex<Box<dyn faces::FaceDao>>>,
|
||||
tag_dao: Arc<Mutex<Box<dyn tags::TagDao>>>,
|
||||
version_dao: Arc<Mutex<Box<dyn VersionDao>>>,
|
||||
face_client: crate::ai::face_client::FaceClient,
|
||||
excluded_dirs: &[String],
|
||||
modified_since: Option<SystemTime>,
|
||||
playlist_manager: Addr<VideoPlaylistManager>,
|
||||
preview_generator: Addr<video::actors::PreviewClipGenerator>,
|
||||
) {
|
||||
let thumbs = dotenv::var("THUMBNAILS").expect("THUMBNAILS not defined");
|
||||
let thumbnail_directory = Path::new(&thumbs);
|
||||
let base_path = Path::new(&library.root_path);
|
||||
|
||||
// Walk, prune EXCLUDED_DIRS subtrees, and apply image/video + modified_since
|
||||
// filters. See `file_scan` for why exclusion has to happen at WalkDir
|
||||
// time (filter_entry) rather than at face-detect time.
|
||||
let files: Vec<(PathBuf, String)> =
|
||||
crate::file_scan::enumerate_indexable_files(base_path, excluded_dirs, modified_since);
|
||||
|
||||
if files.is_empty() {
|
||||
debug!("No files to process");
|
||||
return;
|
||||
}
|
||||
|
||||
debug!("Found {} files to check", files.len());
|
||||
|
||||
// Batch query: Get all EXIF data for these files in one query
|
||||
let file_paths: Vec<String> = files.iter().map(|(_, rel_path)| rel_path.clone()).collect();
|
||||
|
||||
// Map of rel_path -> Option<content_hash>. The presence of the key
|
||||
// tells us "row exists"; the Option value carries the hash for the
|
||||
// HLS pipeline so video files without a hash (mid-backfill) skip
|
||||
// this tick rather than fall back to a basename-colliding playlist.
|
||||
let existing_exif: HashMap<String, Option<String>> = {
|
||||
let mut dao = exif_dao.lock().expect("Unable to lock ExifDao");
|
||||
// Walk is per-library, so scope the lookup so a same-named file
|
||||
// in another library doesn't make this one look already-indexed.
|
||||
match dao.get_exif_batch(Some(library.id), &file_paths) {
|
||||
Ok(exif_records) => exif_records
|
||||
.into_iter()
|
||||
.map(|record| (record.file_path, record.content_hash))
|
||||
.collect(),
|
||||
Err(e) => {
|
||||
error!("Error batch querying EXIF data: {:?}", e);
|
||||
HashMap::new()
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let mut new_files_found = false;
|
||||
let mut files_needing_row = Vec::new();
|
||||
// Files whose image_exif row was written this tick (version linking
|
||||
// looks only at these; re-reading every PNG's XMP on each full scan
|
||||
// would be wasted I/O on a network share).
|
||||
files_needing_row: Vec<(PathBuf, String)>,
|
||||
exif_dao: &Arc<Mutex<Box<dyn ExifDao>>>,
|
||||
) -> Vec<(PathBuf, String)> {
|
||||
let mut newly_registered: Vec<(PathBuf, String)> = Vec::new();
|
||||
|
||||
// Register every image/video file in image_exif. Rows without EXIF
|
||||
// still carry library_id, rel_path, content_hash, and size_bytes so
|
||||
// derivative dedup and DB-indexed sort/filter work for every file,
|
||||
// not just photos with parseable EXIF.
|
||||
for (file_path, relative_path) in &files {
|
||||
// Check both the library-scoped legacy path (current shape) and
|
||||
// the bare-legacy path (pre-multi-library shape). Either one
|
||||
// existing means a thumbnail is already on disk for this file.
|
||||
let scoped_thumb_path = content_hash::library_scoped_legacy_path(
|
||||
thumbnail_directory,
|
||||
library.id,
|
||||
relative_path,
|
||||
);
|
||||
let bare_legacy_thumb_path = thumbnail_directory.join(relative_path);
|
||||
let needs_thumbnail = !scoped_thumb_path.exists()
|
||||
&& !bare_legacy_thumb_path.exists()
|
||||
&& !thumbnails::unsupported_thumbnail_sentinel(&scoped_thumb_path).exists()
|
||||
&& !thumbnails::unsupported_thumbnail_sentinel(&bare_legacy_thumb_path).exists();
|
||||
let needs_row = !existing_exif.contains_key(relative_path);
|
||||
|
||||
if needs_thumbnail || needs_row {
|
||||
new_files_found = true;
|
||||
|
||||
if needs_thumbnail {
|
||||
info!("New file detected (missing thumbnail): {}", relative_path);
|
||||
if files_needing_row.is_empty() {
|
||||
return newly_registered;
|
||||
}
|
||||
|
||||
if needs_row {
|
||||
files_needing_row.push((file_path.clone(), relative_path.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !files_needing_row.is_empty() {
|
||||
info!(
|
||||
"Registering {} new files in image_exif",
|
||||
files_needing_row.len()
|
||||
@@ -947,6 +866,104 @@ pub fn process_new_files(
|
||||
newly_registered.push(registered_file);
|
||||
}
|
||||
}
|
||||
newly_registered
|
||||
}
|
||||
|
||||
pub fn process_new_files(
|
||||
library: &libraries::Library,
|
||||
exif_dao: Arc<Mutex<Box<dyn ExifDao>>>,
|
||||
preview_dao: Arc<Mutex<Box<dyn PreviewDao>>>,
|
||||
face_dao: Arc<Mutex<Box<dyn faces::FaceDao>>>,
|
||||
tag_dao: Arc<Mutex<Box<dyn tags::TagDao>>>,
|
||||
version_dao: Arc<Mutex<Box<dyn VersionDao>>>,
|
||||
face_client: crate::ai::face_client::FaceClient,
|
||||
excluded_dirs: &[String],
|
||||
modified_since: Option<SystemTime>,
|
||||
playlist_manager: Addr<VideoPlaylistManager>,
|
||||
preview_generator: Addr<video::actors::PreviewClipGenerator>,
|
||||
) {
|
||||
let thumbs = dotenv::var("THUMBNAILS").expect("THUMBNAILS not defined");
|
||||
let thumbnail_directory = Path::new(&thumbs);
|
||||
let base_path = Path::new(&library.root_path);
|
||||
|
||||
// Walk, prune EXCLUDED_DIRS subtrees, and apply image/video + modified_since
|
||||
// filters. See `file_scan` for why exclusion has to happen at WalkDir
|
||||
// time (filter_entry) rather than at face-detect time.
|
||||
let files: Vec<(PathBuf, String)> =
|
||||
crate::file_scan::enumerate_indexable_files(base_path, excluded_dirs, modified_since);
|
||||
|
||||
if files.is_empty() {
|
||||
debug!("No files to process");
|
||||
return;
|
||||
}
|
||||
|
||||
debug!("Found {} files to check", files.len());
|
||||
|
||||
// Batch query: Get all EXIF data for these files in one query
|
||||
let file_paths: Vec<String> = files.iter().map(|(_, rel_path)| rel_path.clone()).collect();
|
||||
|
||||
// Map of rel_path -> Option<content_hash>. The presence of the key
|
||||
// tells us "row exists"; the Option value carries the hash for the
|
||||
// HLS pipeline so video files without a hash (mid-backfill) skip
|
||||
// this tick rather than fall back to a basename-colliding playlist.
|
||||
let existing_exif: HashMap<String, Option<String>> = {
|
||||
let mut dao = exif_dao.lock().expect("Unable to lock ExifDao");
|
||||
// Walk is per-library, so scope the lookup so a same-named file
|
||||
// in another library doesn't make this one look already-indexed.
|
||||
match dao.get_exif_batch(Some(library.id), &file_paths) {
|
||||
Ok(exif_records) => exif_records
|
||||
.into_iter()
|
||||
.map(|record| (record.file_path, record.content_hash))
|
||||
.collect(),
|
||||
Err(e) => {
|
||||
error!("Error batch querying EXIF data: {:?}", e);
|
||||
HashMap::new()
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let mut new_files_found = false;
|
||||
let mut files_needing_row = Vec::new();
|
||||
// Files whose image_exif row was written this tick (version linking
|
||||
// looks only at these; re-reading every PNG's XMP on each full scan
|
||||
// would be wasted I/O on a network share).
|
||||
let mut newly_registered: Vec<(PathBuf, String)> = Vec::new();
|
||||
|
||||
// Register every image/video file in image_exif. Rows without EXIF
|
||||
// still carry library_id, rel_path, content_hash, and size_bytes so
|
||||
// derivative dedup and DB-indexed sort/filter work for every file,
|
||||
// not just photos with parseable EXIF.
|
||||
for (file_path, relative_path) in &files {
|
||||
// Check both the library-scoped legacy path (current shape) and
|
||||
// the bare-legacy path (pre-multi-library shape). Either one
|
||||
// existing means a thumbnail is already on disk for this file.
|
||||
let scoped_thumb_path = content_hash::library_scoped_legacy_path(
|
||||
thumbnail_directory,
|
||||
library.id,
|
||||
relative_path,
|
||||
);
|
||||
let bare_legacy_thumb_path = thumbnail_directory.join(relative_path);
|
||||
let needs_thumbnail = !scoped_thumb_path.exists()
|
||||
&& !bare_legacy_thumb_path.exists()
|
||||
&& !thumbnails::unsupported_thumbnail_sentinel(&scoped_thumb_path).exists()
|
||||
&& !thumbnails::unsupported_thumbnail_sentinel(&bare_legacy_thumb_path).exists();
|
||||
let needs_row = !existing_exif.contains_key(relative_path);
|
||||
|
||||
if needs_thumbnail || needs_row {
|
||||
new_files_found = true;
|
||||
|
||||
if needs_thumbnail {
|
||||
info!("New file detected (missing thumbnail): {}", relative_path);
|
||||
}
|
||||
|
||||
if needs_row {
|
||||
files_needing_row.push((file_path.clone(), relative_path.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !files_needing_row.is_empty() {
|
||||
newly_registered = register_new_files(library, files_needing_row, &exif_dao);
|
||||
}
|
||||
|
||||
// ── Photo versions (#191) ──────────────────────────────────────────
|
||||
|
||||
Reference in new issue
Block a user