Apps/ImageApi
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

perf/faces-embeddings-no-clone #72

Merged
cameron merged 2 commits from perf/faces-embeddings-no-clone into master 2026-05-01 23:09:23 +00:00
Conversation 0 Commits 2 Files Changed 7 +86 -76
7 changed files with 86 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/content_hash.rs
View File

@@ -78,9 +78,7 @@ pub fn library_scoped_legacy_path(
library_id: i32,
rel_path: impl AsRef<Path>,
) -> PathBuf {
derivative_dir
.join(library_id.to_string())
.join(rel_path)
derivative_dir.join(library_id.to_string()).join(rel_path)
}
fn shard_prefix(hash: &str) -> &str {

27
src/database/mod.rs
View File

@@ -1149,18 +1149,23 @@ impl ExifDao for SqliteExifDao {
limit: i64,
offset: i64,
) -> Result<Vec<(i32, String)>, DbError> {
trace_db_call(context, "query", "list_rel_paths_for_library_page", |_span| {
use schema::image_exif::dsl::*;
trace_db_call(
context,
"query",
"list_rel_paths_for_library_page",
|_span| {
use schema::image_exif::dsl::*;
image_exif
.filter(library_id.eq(library_id_val))
.order(id.asc())
.select((id, rel_path))
.limit(limit)
.offset(offset)
.load::<(i32, String)>(self.connection.lock().unwrap().deref_mut())
.map_err(|_| anyhow::anyhow!("Query error"))
})
image_exif
.filter(library_id.eq(library_id_val))
.order(id.asc())
.select((id, rel_path))
.limit(limit)
.offset(offset)
.load::<(i32, String)>(self.connection.lock().unwrap().deref_mut())
.map_err(|_| anyhow::anyhow!("Query error"))
},
)
.map_err(|_| DbError::new(DbErrorKind::QueryError))
}
}

33
src/database/reconcile.rs
View File

@@ -325,12 +325,10 @@ mod tests {
let stats = run(&mut conn);
assert_eq!(stats.photo_insights_hashes_filled, 1);
let row = diesel::sql_query(
"SELECT content_hash FROM photo_insights WHERE id = ?",
)
.bind::<diesel::sql_types::Integer, _>(id1)
.get_result::<HashOnly>(&mut conn)
.unwrap();
let row = diesel::sql_query("SELECT content_hash FROM photo_insights WHERE id = ?")
.bind::<diesel::sql_types::Integer, _>(id1)
.get_result::<HashOnly>(&mut conn)
.unwrap();
assert_eq!(row.content_hash.as_deref(), Some("hash-lib1"));
}
@@ -350,14 +348,15 @@ mod tests {
assert_eq!(stats.photo_insights_hashes_filled, 2);
assert_eq!(stats.photo_insights_demoted, 1);
let rows = diesel::sql_query(
"SELECT id, is_current FROM photo_insights ORDER BY id",
)
.get_results::<CurrentRow>(&mut conn)
.unwrap();
let rows = diesel::sql_query("SELECT id, is_current FROM photo_insights ORDER BY id")
.get_results::<CurrentRow>(&mut conn)
.unwrap();
let earlier_row = rows.iter().find(|r| r.id == earlier).unwrap();
let later_row = rows.iter().find(|r| r.id == later).unwrap();
assert!(earlier_row.is_current, "earlier insight should remain current");
assert!(
earlier_row.is_current,
"earlier insight should remain current"
);
assert!(!later_row.is_current, "later insight should be demoted");
// Idempotent.
@@ -374,12 +373,10 @@ mod tests {
let stats = run(&mut conn);
assert_eq!(stats.photo_insights_demoted, 0);
let row = diesel::sql_query(
"SELECT id, is_current FROM photo_insights WHERE id = ?",
)
.bind::<diesel::sql_types::Integer, _>(solo)
.get_result::<CurrentRow>(&mut conn)
.unwrap();
let row = diesel::sql_query("SELECT id, is_current FROM photo_insights WHERE id = ?")
.bind::<diesel::sql_types::Integer, _>(solo)
.get_result::<CurrentRow>(&mut conn)
.unwrap();
assert!(row.is_current);
}
}

14
src/faces.rs
View File

@@ -884,14 +884,18 @@ impl FaceDao for SqliteFaceDao {
// Pair with the base64-encoded embedding string so the handler
// doesn't need to know the wire format. Skip rows with NULL
// embedding (shouldn't happen on detected rows, but defensive).
// `embedding.take()` moves the bytes out of the row so we can
// hand the (now-empty-embedding) row plus the encoded string
// back to the caller without cloning the whole row — at 20k
// rows × 2 KB that clone was 40 MB of pointless heap traffic
// per cluster-suggest run.
use base64::Engine;
Ok(rows
.into_iter()
.filter_map(|r| {
r.embedding.as_ref().map(|bytes| {
let b64 = base64::engine::general_purpose::STANDARD.encode(bytes);
(r.clone(), b64)
})
.filter_map(|mut r| {
let bytes = r.embedding.take()?;
let b64 = base64::engine::general_purpose::STANDARD.encode(&bytes);
Some((r, b64))
})
.collect())
})

3
src/libraries.rs
View File

@@ -71,7 +71,8 @@ impl Library {
if self.excluded_dirs.is_empty() {
return globals.to_vec();
}
let mut combined: Vec<String> = Vec::with_capacity(globals.len() + self.excluded_dirs.len());
let mut combined: Vec<String> =
Vec::with_capacity(globals.len() + self.excluded_dirs.len());
combined.extend_from_slice(globals);
combined.extend(self.excluded_dirs.iter().cloned());
combined

59
src/library_maintenance.rs
View File

@@ -350,7 +350,11 @@ pub fn run_orphan_gc(
// ("revived"). Common case: a network share that briefly went
// stale comes back, image_exif gets re-populated by ingest, and
// the hash is no longer orphaned.
let revived = state.pending.difference(&orphans).cloned().collect::<Vec<_>>();
let revived = state
.pending
.difference(&orphans)
.cloned()
.collect::<Vec<_>>();
if !revived.is_empty() {
for h in &revived {
state.pending.remove(h);
@@ -438,7 +442,10 @@ pub fn run_orphan_gc(
state.pending.len(),
);
} else {
debug!("orphan-gc: no changes this tick (pending: {})", state.pending.len());
debug!(
"orphan-gc: no changes this tick (pending: {})",
state.pending.len()
);
}
stats
@@ -458,12 +465,7 @@ pub fn all_libraries_online(libs: &[Library], health: &LibraryHealthMap) -> bool
let guard = health.read().unwrap_or_else(|e| e.into_inner());
libs.iter()
.filter(|lib| lib.enabled)
.all(|lib| {
guard
.get(&lib.id)
.map(|h| h.is_online())
.unwrap_or(false)
})
.all(|lib| guard.get(&lib.id).map(|h| h.is_online()).unwrap_or(false))
}
#[derive(QueryableByName, Debug)]
@@ -504,18 +506,15 @@ fn delete_hash_keyed_rows(
use crate::database::schema::{face_detections, photo_insights, tagged_photo};
let faces = diesel::delete(
face_detections::table.filter(face_detections::content_hash.eq_any(hashes)),
)
.execute(conn)?;
let tags = diesel::delete(
tagged_photo::table.filter(tagged_photo::content_hash.eq_any(hashes)),
)
.execute(conn)?;
let insights = diesel::delete(
photo_insights::table.filter(photo_insights::content_hash.eq_any(hashes)),
)
.execute(conn)?;
let faces =
diesel::delete(face_detections::table.filter(face_detections::content_hash.eq_any(hashes)))
.execute(conn)?;
let tags =
diesel::delete(tagged_photo::table.filter(tagged_photo::content_hash.eq_any(hashes)))
.execute(conn)?;
let insights =
diesel::delete(photo_insights::table.filter(photo_insights::content_hash.eq_any(hashes)))
.execute(conn)?;
Ok((faces, tags, insights))
}
@@ -605,7 +604,10 @@ mod tests {
n: i64,
}
fn count(conn: &mut SqliteConnection, sql: &str) -> i64 {
diesel::sql_query(sql).get_result::<CountRow>(conn).unwrap().n
diesel::sql_query(sql)
.get_result::<CountRow>(conn)
.unwrap()
.n
}
#[test]
@@ -731,9 +733,18 @@ mod tests {
assert_eq!(stats.deleted_tagged_photo, 1);
assert_eq!(stats.deleted_photo_insights, 1);
assert_eq!(count(&mut conn, "SELECT COUNT(*) AS n FROM face_detections"), 0);
assert_eq!(count(&mut conn, "SELECT COUNT(*) AS n FROM tagged_photo"), 0);
assert_eq!(count(&mut conn, "SELECT COUNT(*) AS n FROM photo_insights"), 0);
assert_eq!(
count(&mut conn, "SELECT COUNT(*) AS n FROM face_detections"),
0
);
assert_eq!(
count(&mut conn, "SELECT COUNT(*) AS n FROM tagged_photo"),
0
);
assert_eq!(
count(&mut conn, "SELECT COUNT(*) AS n FROM photo_insights"),
0
);
}
#[test]

22
src/main.rs
View File

@@ -1736,7 +1736,10 @@ fn cleanup_orphaned_playlists(
info!(" Cleanup interval: {} seconds", cleanup_interval_secs);
info!(" Playlist directory: {}", video_path);
for lib in &libs {
info!(" Checking sources under '{}' at {}", lib.name, lib.root_path);
info!(
" Checking sources under '{}' at {}",
lib.name, lib.root_path
);
}
loop {
@@ -1752,12 +1755,7 @@ fn cleanup_orphaned_playlists(
let guard = library_health.read().unwrap_or_else(|e| e.into_inner());
let stale: Vec<String> = libs
.iter()
.filter(|lib| {
guard
.get(&lib.id)
.map(|h| !h.is_online())
.unwrap_or(false)
})
.filter(|lib| guard.get(&lib.id).map(|h| !h.is_online()).unwrap_or(false))
.map(|lib| lib.name.clone())
.collect();
if !stale.is_empty() {
@@ -2170,13 +2168,9 @@ fn watch_files(
// requirement is enforced inside run_orphan_gc; we
// pass the current all-online flag and the function
// tracks the previous tick's flag in OrphanGcState.
let all_online =
library_maintenance::all_libraries_online(&libs, &library_health);
let _ = library_maintenance::run_orphan_gc(
&mut conn,
&mut orphan_gc_state,
all_online,
);
let all_online = library_maintenance::all_libraries_online(&libs, &library_health);
let _ =
library_maintenance::run_orphan_gc(&mut conn, &mut orphan_gc_state, all_online);
}
if is_full_scan {