refactor(watcher): extract register_new_files for targeted ingest
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4105430681
commit
ebed15bd59
1 file changed
+108
-91
+108
-91
@@ -740,100 +740,19 @@ pub fn playlist_needs_generation(video_path: &Path, playlist_path: &Path) -> boo
|
|||||||
true
|
true
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn process_new_files(
|
/// Register files in `image_exif` (hash, perceptual hashes, EXIF, resolved
|
||||||
|
/// date). Returns the `(absolute, rel_path)` pairs whose row was written.
|
||||||
|
/// Shared by the watcher tick and the targeted ingest after an Enhance
|
||||||
|
/// approval (#192).
|
||||||
|
pub fn register_new_files(
|
||||||
library: &libraries::Library,
|
library: &libraries::Library,
|
||||||
exif_dao: Arc<Mutex<Box<dyn ExifDao>>>,
|
files_needing_row: Vec<(PathBuf, String)>,
|
||||||
preview_dao: Arc<Mutex<Box<dyn PreviewDao>>>,
|
exif_dao: &Arc<Mutex<Box<dyn ExifDao>>>,
|
||||||
face_dao: Arc<Mutex<Box<dyn faces::FaceDao>>>,
|
) -> Vec<(PathBuf, String)> {
|
||||||
tag_dao: Arc<Mutex<Box<dyn tags::TagDao>>>,
|
|
||||||
version_dao: Arc<Mutex<Box<dyn VersionDao>>>,
|
|
||||||
face_client: crate::ai::face_client::FaceClient,
|
|
||||||
excluded_dirs: &[String],
|
|
||||||
modified_since: Option<SystemTime>,
|
|
||||||
playlist_manager: Addr<VideoPlaylistManager>,
|
|
||||||
preview_generator: Addr<video::actors::PreviewClipGenerator>,
|
|
||||||
) {
|
|
||||||
let thumbs = dotenv::var("THUMBNAILS").expect("THUMBNAILS not defined");
|
|
||||||
let thumbnail_directory = Path::new(&thumbs);
|
|
||||||
let base_path = Path::new(&library.root_path);
|
|
||||||
|
|
||||||
// Walk, prune EXCLUDED_DIRS subtrees, and apply image/video + modified_since
|
|
||||||
// filters. See `file_scan` for why exclusion has to happen at WalkDir
|
|
||||||
// time (filter_entry) rather than at face-detect time.
|
|
||||||
let files: Vec<(PathBuf, String)> =
|
|
||||||
crate::file_scan::enumerate_indexable_files(base_path, excluded_dirs, modified_since);
|
|
||||||
|
|
||||||
if files.is_empty() {
|
|
||||||
debug!("No files to process");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
debug!("Found {} files to check", files.len());
|
|
||||||
|
|
||||||
// Batch query: Get all EXIF data for these files in one query
|
|
||||||
let file_paths: Vec<String> = files.iter().map(|(_, rel_path)| rel_path.clone()).collect();
|
|
||||||
|
|
||||||
// Map of rel_path -> Option<content_hash>. The presence of the key
|
|
||||||
// tells us "row exists"; the Option value carries the hash for the
|
|
||||||
// HLS pipeline so video files without a hash (mid-backfill) skip
|
|
||||||
// this tick rather than fall back to a basename-colliding playlist.
|
|
||||||
let existing_exif: HashMap<String, Option<String>> = {
|
|
||||||
let mut dao = exif_dao.lock().expect("Unable to lock ExifDao");
|
|
||||||
// Walk is per-library, so scope the lookup so a same-named file
|
|
||||||
// in another library doesn't make this one look already-indexed.
|
|
||||||
match dao.get_exif_batch(Some(library.id), &file_paths) {
|
|
||||||
Ok(exif_records) => exif_records
|
|
||||||
.into_iter()
|
|
||||||
.map(|record| (record.file_path, record.content_hash))
|
|
||||||
.collect(),
|
|
||||||
Err(e) => {
|
|
||||||
error!("Error batch querying EXIF data: {:?}", e);
|
|
||||||
HashMap::new()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut new_files_found = false;
|
|
||||||
let mut files_needing_row = Vec::new();
|
|
||||||
// Files whose image_exif row was written this tick (version linking
|
|
||||||
// looks only at these; re-reading every PNG's XMP on each full scan
|
|
||||||
// would be wasted I/O on a network share).
|
|
||||||
let mut newly_registered: Vec<(PathBuf, String)> = Vec::new();
|
let mut newly_registered: Vec<(PathBuf, String)> = Vec::new();
|
||||||
|
if files_needing_row.is_empty() {
|
||||||
// Register every image/video file in image_exif. Rows without EXIF
|
return newly_registered;
|
||||||
// still carry library_id, rel_path, content_hash, and size_bytes so
|
|
||||||
// derivative dedup and DB-indexed sort/filter work for every file,
|
|
||||||
// not just photos with parseable EXIF.
|
|
||||||
for (file_path, relative_path) in &files {
|
|
||||||
// Check both the library-scoped legacy path (current shape) and
|
|
||||||
// the bare-legacy path (pre-multi-library shape). Either one
|
|
||||||
// existing means a thumbnail is already on disk for this file.
|
|
||||||
let scoped_thumb_path = content_hash::library_scoped_legacy_path(
|
|
||||||
thumbnail_directory,
|
|
||||||
library.id,
|
|
||||||
relative_path,
|
|
||||||
);
|
|
||||||
let bare_legacy_thumb_path = thumbnail_directory.join(relative_path);
|
|
||||||
let needs_thumbnail = !scoped_thumb_path.exists()
|
|
||||||
&& !bare_legacy_thumb_path.exists()
|
|
||||||
&& !thumbnails::unsupported_thumbnail_sentinel(&scoped_thumb_path).exists()
|
|
||||||
&& !thumbnails::unsupported_thumbnail_sentinel(&bare_legacy_thumb_path).exists();
|
|
||||||
let needs_row = !existing_exif.contains_key(relative_path);
|
|
||||||
|
|
||||||
if needs_thumbnail || needs_row {
|
|
||||||
new_files_found = true;
|
|
||||||
|
|
||||||
if needs_thumbnail {
|
|
||||||
info!("New file detected (missing thumbnail): {}", relative_path);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if needs_row {
|
|
||||||
files_needing_row.push((file_path.clone(), relative_path.clone()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if !files_needing_row.is_empty() {
|
|
||||||
info!(
|
info!(
|
||||||
"Registering {} new files in image_exif",
|
"Registering {} new files in image_exif",
|
||||||
files_needing_row.len()
|
files_needing_row.len()
|
||||||
@@ -947,6 +866,104 @@ pub fn process_new_files(
|
|||||||
newly_registered.push(registered_file);
|
newly_registered.push(registered_file);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
newly_registered
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn process_new_files(
|
||||||
|
library: &libraries::Library,
|
||||||
|
exif_dao: Arc<Mutex<Box<dyn ExifDao>>>,
|
||||||
|
preview_dao: Arc<Mutex<Box<dyn PreviewDao>>>,
|
||||||
|
face_dao: Arc<Mutex<Box<dyn faces::FaceDao>>>,
|
||||||
|
tag_dao: Arc<Mutex<Box<dyn tags::TagDao>>>,
|
||||||
|
version_dao: Arc<Mutex<Box<dyn VersionDao>>>,
|
||||||
|
face_client: crate::ai::face_client::FaceClient,
|
||||||
|
excluded_dirs: &[String],
|
||||||
|
modified_since: Option<SystemTime>,
|
||||||
|
playlist_manager: Addr<VideoPlaylistManager>,
|
||||||
|
preview_generator: Addr<video::actors::PreviewClipGenerator>,
|
||||||
|
) {
|
||||||
|
let thumbs = dotenv::var("THUMBNAILS").expect("THUMBNAILS not defined");
|
||||||
|
let thumbnail_directory = Path::new(&thumbs);
|
||||||
|
let base_path = Path::new(&library.root_path);
|
||||||
|
|
||||||
|
// Walk, prune EXCLUDED_DIRS subtrees, and apply image/video + modified_since
|
||||||
|
// filters. See `file_scan` for why exclusion has to happen at WalkDir
|
||||||
|
// time (filter_entry) rather than at face-detect time.
|
||||||
|
let files: Vec<(PathBuf, String)> =
|
||||||
|
crate::file_scan::enumerate_indexable_files(base_path, excluded_dirs, modified_since);
|
||||||
|
|
||||||
|
if files.is_empty() {
|
||||||
|
debug!("No files to process");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
debug!("Found {} files to check", files.len());
|
||||||
|
|
||||||
|
// Batch query: Get all EXIF data for these files in one query
|
||||||
|
let file_paths: Vec<String> = files.iter().map(|(_, rel_path)| rel_path.clone()).collect();
|
||||||
|
|
||||||
|
// Map of rel_path -> Option<content_hash>. The presence of the key
|
||||||
|
// tells us "row exists"; the Option value carries the hash for the
|
||||||
|
// HLS pipeline so video files without a hash (mid-backfill) skip
|
||||||
|
// this tick rather than fall back to a basename-colliding playlist.
|
||||||
|
let existing_exif: HashMap<String, Option<String>> = {
|
||||||
|
let mut dao = exif_dao.lock().expect("Unable to lock ExifDao");
|
||||||
|
// Walk is per-library, so scope the lookup so a same-named file
|
||||||
|
// in another library doesn't make this one look already-indexed.
|
||||||
|
match dao.get_exif_batch(Some(library.id), &file_paths) {
|
||||||
|
Ok(exif_records) => exif_records
|
||||||
|
.into_iter()
|
||||||
|
.map(|record| (record.file_path, record.content_hash))
|
||||||
|
.collect(),
|
||||||
|
Err(e) => {
|
||||||
|
error!("Error batch querying EXIF data: {:?}", e);
|
||||||
|
HashMap::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut new_files_found = false;
|
||||||
|
let mut files_needing_row = Vec::new();
|
||||||
|
// Files whose image_exif row was written this tick (version linking
|
||||||
|
// looks only at these; re-reading every PNG's XMP on each full scan
|
||||||
|
// would be wasted I/O on a network share).
|
||||||
|
let mut newly_registered: Vec<(PathBuf, String)> = Vec::new();
|
||||||
|
|
||||||
|
// Register every image/video file in image_exif. Rows without EXIF
|
||||||
|
// still carry library_id, rel_path, content_hash, and size_bytes so
|
||||||
|
// derivative dedup and DB-indexed sort/filter work for every file,
|
||||||
|
// not just photos with parseable EXIF.
|
||||||
|
for (file_path, relative_path) in &files {
|
||||||
|
// Check both the library-scoped legacy path (current shape) and
|
||||||
|
// the bare-legacy path (pre-multi-library shape). Either one
|
||||||
|
// existing means a thumbnail is already on disk for this file.
|
||||||
|
let scoped_thumb_path = content_hash::library_scoped_legacy_path(
|
||||||
|
thumbnail_directory,
|
||||||
|
library.id,
|
||||||
|
relative_path,
|
||||||
|
);
|
||||||
|
let bare_legacy_thumb_path = thumbnail_directory.join(relative_path);
|
||||||
|
let needs_thumbnail = !scoped_thumb_path.exists()
|
||||||
|
&& !bare_legacy_thumb_path.exists()
|
||||||
|
&& !thumbnails::unsupported_thumbnail_sentinel(&scoped_thumb_path).exists()
|
||||||
|
&& !thumbnails::unsupported_thumbnail_sentinel(&bare_legacy_thumb_path).exists();
|
||||||
|
let needs_row = !existing_exif.contains_key(relative_path);
|
||||||
|
|
||||||
|
if needs_thumbnail || needs_row {
|
||||||
|
new_files_found = true;
|
||||||
|
|
||||||
|
if needs_thumbnail {
|
||||||
|
info!("New file detected (missing thumbnail): {}", relative_path);
|
||||||
|
}
|
||||||
|
|
||||||
|
if needs_row {
|
||||||
|
files_needing_row.push((file_path.clone(), relative_path.clone()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !files_needing_row.is_empty() {
|
||||||
|
newly_registered = register_new_files(library, files_needing_row, &exif_dao);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Photo versions (#191) ──────────────────────────────────────────
|
// ── Photo versions (#191) ──────────────────────────────────────────
|
||||||
|
|||||||
Reference in new issue
Block a user