Apps/ImageApi
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

perf: DB-backed recursive /photos + watcher reconciliation

Browse Source 
Recursive listings now query image_exif instead of walking disk, taking
union-mode /photos from ~17s to sub-second on a 10k-file library. The
watcher's full scan prunes stale image_exif rows so the DB stays in
parity with the filesystem when files are deleted externally.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Cameron
2026-04-18 21:38:51 -04:00
committed by cameron
parent 4a775b5e9b
commit 3027a3ffda
3 changed files with 235 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

225
src/files.rs
View File

@@ -546,91 +546,89 @@ pub async fn list_photos<TagD: TagDao, FS: FileSystemAccess>(
.unwrap_or_else(|e| e.error_response());
}
// Walk each candidate library's root for the requested sub-path. In
// scoped mode `libraries_to_scan` has one entry (the selected library);
// in union mode we walk every configured library and intermix results.
// For the primary library we preserve the original FileSystemAccess
// path so the test-mock path (MockFileSystem) continues to work.
// In scoped mode `libraries_to_scan` has one entry (the selected library);
// in union mode we enumerate every configured library and intermix results.
//
// Recursive mode pulls rel_paths from image_exif (kept in parity with disk
// by the watcher's full-scan reconciliation) instead of walking — a ~10k
// file library drops from multi-second to ~10ms for the listing itself.
// Non-recursive mode still walks because we need directory metadata for
// the `dirs` response and listing a single directory is cheap.
let mut file_names: Vec<String> = Vec::new();
let mut file_libraries: Vec<i32> = Vec::new();
let mut dirs_set: std::collections::HashSet<String> = std::collections::HashSet::new();
let mut any_library_resolved = false;
for lib in &libraries_to_scan {
let files_result = if search_recursively {
is_valid_full_path(
&PathBuf::from(&lib.root_path),
&PathBuf::from(search_path),
false,
)
.map(|path| {
debug!("Valid path for recursive search: {:?}", path);
list_files_recursive(&path).unwrap_or_default()
})
.context("Invalid path")
} else if lib.id == app_state.primary_library().id {
file_system.get_files_for_path(search_path)
if search_recursively {
let start_db_list = std::time::Instant::now();
let lib_ids: Vec<i32> = libraries_to_scan.iter().map(|l| l.id).collect();
let trimmed = search_path.trim();
let prefix = if trimmed.is_empty() || trimmed == "/" {
None
} else {
is_valid_full_path(
&PathBuf::from(&lib.root_path),
&PathBuf::from(search_path),
false,
)
.map(|path| {
debug!("Valid path for non-recursive search: {:?}", path);
list_files(&path).unwrap_or_default()
})
.context("Invalid path")
Some(trimmed)
};
let files = match files_result {
Ok(f) => {
any_library_resolved = true;
f
}
Err(e) => {
debug!(
"Skipping library '{}' for path '{}': {:?}",
lib.name, search_path, e
);
continue;
}
let rows = {
let mut dao = exif_dao.lock().expect("Unable to get ExifDao");
dao.list_rel_paths_for_libraries(&span_context, &lib_ids, prefix)
.unwrap_or_else(|e| {
warn!("list_rel_paths_for_libraries failed: {:?}", e);
Vec::new()
})
};
info!(
"Found {:?} files in library '{}' path: {:?} (recursive: {})",
files.len(),
lib.name,
search_path,
search_recursively
"DB-backed recursive listing: {} files across {} libraries in {:?}",
rows.len(),
lib_ids.len(),
start_db_list.elapsed()
);
any_library_resolved = true;
for (lib_id, path) in rows {
file_libraries.push(lib_id);
file_names.push(path);
}
} else {
for lib in &libraries_to_scan {
let files_result = if lib.id == app_state.primary_library().id {
file_system.get_files_for_path(search_path)
} else {
is_valid_full_path(
&PathBuf::from(&lib.root_path),
&PathBuf::from(search_path),
false,
)
.map(|path| {
debug!("Valid path for non-recursive search: {:?}", path);
list_files(&path).unwrap_or_default()
})
.context("Invalid path")
};
for path in &files {
match path.metadata() {
Ok(md) => {
let relative = path.strip_prefix(&lib.root_path).unwrap_or_else(|_| {
panic!(
"Unable to strip library root {} from file path {}",
&lib.root_path,
path.display()
)
});
// Normalize separators to '/' so downstream lookups
// (tags, EXIF, insights) that store rel_paths with
// forward slashes still match on Windows.
let relative_str = relative.to_str().unwrap().replace('\\', "/");
if md.is_file() {
file_names.push(relative_str);
file_libraries.push(lib.id);
} else if md.is_dir() {
dirs_set.insert(relative_str);
}
let files = match files_result {
Ok(f) => {
any_library_resolved = true;
f
}
Err(e) => {
error!("Failed getting file metadata: {:?}", e);
// Include files without metadata if they have extensions
if path.extension().is_some() {
debug!(
"Skipping library '{}' for path '{}': {:?}",
lib.name, search_path, e
);
continue;
}
};
info!(
"Found {:?} files in library '{}' path: {:?} (recursive: {})",
files.len(),
lib.name,
search_path,
search_recursively
);
for path in &files {
match path.metadata() {
Ok(md) => {
let relative = path.strip_prefix(&lib.root_path).unwrap_or_else(|_| {
panic!(
"Unable to strip library root {} from file path {}",
@@ -638,8 +636,32 @@ pub async fn list_photos<TagD: TagDao, FS: FileSystemAccess>(
path.display()
)
});
file_names.push(relative.to_str().unwrap().replace('\\', "/"));
file_libraries.push(lib.id);
// Normalize separators to '/' so downstream lookups
// (tags, EXIF, insights) that store rel_paths with
// forward slashes still match on Windows.
let relative_str = relative.to_str().unwrap().replace('\\', "/");
if md.is_file() {
file_names.push(relative_str);
file_libraries.push(lib.id);
} else if md.is_dir() {
dirs_set.insert(relative_str);
}
}
Err(e) => {
error!("Failed getting file metadata: {:?}", e);
// Include files without metadata if they have extensions
if path.extension().is_some() {
let relative = path.strip_prefix(&lib.root_path).unwrap_or_else(|_| {
panic!(
"Unable to strip library root {} from file path {}",
&lib.root_path,
path.display()
)
});
file_names.push(relative.to_str().unwrap().replace('\\', "/"));
file_libraries.push(lib.id);
}
}
}
}
@@ -943,43 +965,6 @@ pub fn list_files(dir: &Path) -> io::Result<Vec<PathBuf>> {
Ok(files)
}
pub fn list_files_recursive(dir: &Path) -> io::Result<Vec<PathBuf>> {
let tracer = global_tracer();
let mut span = tracer.start("list_files_recursive");
let dir_name_string = dir.to_str().unwrap_or_default().to_string();
span.set_attribute(KeyValue::new("dir", dir_name_string));
info!("Recursively listing files in: {:?}", dir);
let mut result = Vec::new();
fn visit_dirs(dir: &Path, files: &mut Vec<PathBuf>) -> io::Result<()> {
if dir.is_dir() {
for entry in read_dir(dir)? {
let entry = entry?;
let path = entry.path();
if path.is_dir() {
visit_dirs(&path, files)?;
} else if is_image_or_video(&path) {
files.push(path);
}
}
}
Ok(())
}
visit_dirs(dir, &mut result)?;
span.set_attribute(KeyValue::new("file_count", result.len().to_string()));
span.set_status(Status::Ok);
info!(
"Found {:?} files recursively in directory: {:?}",
result.len(),
dir
);
Ok(result)
}
pub fn is_image_or_video(path: &Path) -> bool {
file_types::is_media_file(path)
}
@@ -1567,6 +1552,24 @@ mod tests {
) -> Result<Vec<String>, DbError> {
Ok(vec![])
}
fn list_rel_paths_for_libraries(
&mut self,
_context: &opentelemetry::Context,
_library_ids: &[i32],
_path_prefix: Option<&str>,
) -> Result<Vec<(i32, String)>, DbError> {
Ok(vec![])
}
fn delete_exif_by_library(
&mut self,
_context: &opentelemetry::Context,
_library_id: i32,
_rel_path: &str,
) -> Result<(), DbError> {
Ok(())
}
}
mod api {