perf/faces-embeddings-no-clone #72
@@ -78,9 +78,7 @@ pub fn library_scoped_legacy_path(
|
|||||||
library_id: i32,
|
library_id: i32,
|
||||||
rel_path: impl AsRef<Path>,
|
rel_path: impl AsRef<Path>,
|
||||||
) -> PathBuf {
|
) -> PathBuf {
|
||||||
derivative_dir
|
derivative_dir.join(library_id.to_string()).join(rel_path)
|
||||||
.join(library_id.to_string())
|
|
||||||
.join(rel_path)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn shard_prefix(hash: &str) -> &str {
|
fn shard_prefix(hash: &str) -> &str {
|
||||||
|
|||||||
@@ -1149,18 +1149,23 @@ impl ExifDao for SqliteExifDao {
|
|||||||
limit: i64,
|
limit: i64,
|
||||||
offset: i64,
|
offset: i64,
|
||||||
) -> Result<Vec<(i32, String)>, DbError> {
|
) -> Result<Vec<(i32, String)>, DbError> {
|
||||||
trace_db_call(context, "query", "list_rel_paths_for_library_page", |_span| {
|
trace_db_call(
|
||||||
use schema::image_exif::dsl::*;
|
context,
|
||||||
|
"query",
|
||||||
|
"list_rel_paths_for_library_page",
|
||||||
|
|_span| {
|
||||||
|
use schema::image_exif::dsl::*;
|
||||||
|
|
||||||
image_exif
|
image_exif
|
||||||
.filter(library_id.eq(library_id_val))
|
.filter(library_id.eq(library_id_val))
|
||||||
.order(id.asc())
|
.order(id.asc())
|
||||||
.select((id, rel_path))
|
.select((id, rel_path))
|
||||||
.limit(limit)
|
.limit(limit)
|
||||||
.offset(offset)
|
.offset(offset)
|
||||||
.load::<(i32, String)>(self.connection.lock().unwrap().deref_mut())
|
.load::<(i32, String)>(self.connection.lock().unwrap().deref_mut())
|
||||||
.map_err(|_| anyhow::anyhow!("Query error"))
|
.map_err(|_| anyhow::anyhow!("Query error"))
|
||||||
})
|
},
|
||||||
|
)
|
||||||
.map_err(|_| DbError::new(DbErrorKind::QueryError))
|
.map_err(|_| DbError::new(DbErrorKind::QueryError))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -325,12 +325,10 @@ mod tests {
|
|||||||
let stats = run(&mut conn);
|
let stats = run(&mut conn);
|
||||||
assert_eq!(stats.photo_insights_hashes_filled, 1);
|
assert_eq!(stats.photo_insights_hashes_filled, 1);
|
||||||
|
|
||||||
let row = diesel::sql_query(
|
let row = diesel::sql_query("SELECT content_hash FROM photo_insights WHERE id = ?")
|
||||||
"SELECT content_hash FROM photo_insights WHERE id = ?",
|
.bind::<diesel::sql_types::Integer, _>(id1)
|
||||||
)
|
.get_result::<HashOnly>(&mut conn)
|
||||||
.bind::<diesel::sql_types::Integer, _>(id1)
|
.unwrap();
|
||||||
.get_result::<HashOnly>(&mut conn)
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(row.content_hash.as_deref(), Some("hash-lib1"));
|
assert_eq!(row.content_hash.as_deref(), Some("hash-lib1"));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -350,14 +348,15 @@ mod tests {
|
|||||||
assert_eq!(stats.photo_insights_hashes_filled, 2);
|
assert_eq!(stats.photo_insights_hashes_filled, 2);
|
||||||
assert_eq!(stats.photo_insights_demoted, 1);
|
assert_eq!(stats.photo_insights_demoted, 1);
|
||||||
|
|
||||||
let rows = diesel::sql_query(
|
let rows = diesel::sql_query("SELECT id, is_current FROM photo_insights ORDER BY id")
|
||||||
"SELECT id, is_current FROM photo_insights ORDER BY id",
|
.get_results::<CurrentRow>(&mut conn)
|
||||||
)
|
.unwrap();
|
||||||
.get_results::<CurrentRow>(&mut conn)
|
|
||||||
.unwrap();
|
|
||||||
let earlier_row = rows.iter().find(|r| r.id == earlier).unwrap();
|
let earlier_row = rows.iter().find(|r| r.id == earlier).unwrap();
|
||||||
let later_row = rows.iter().find(|r| r.id == later).unwrap();
|
let later_row = rows.iter().find(|r| r.id == later).unwrap();
|
||||||
assert!(earlier_row.is_current, "earlier insight should remain current");
|
assert!(
|
||||||
|
earlier_row.is_current,
|
||||||
|
"earlier insight should remain current"
|
||||||
|
);
|
||||||
assert!(!later_row.is_current, "later insight should be demoted");
|
assert!(!later_row.is_current, "later insight should be demoted");
|
||||||
|
|
||||||
// Idempotent.
|
// Idempotent.
|
||||||
@@ -374,12 +373,10 @@ mod tests {
|
|||||||
let stats = run(&mut conn);
|
let stats = run(&mut conn);
|
||||||
assert_eq!(stats.photo_insights_demoted, 0);
|
assert_eq!(stats.photo_insights_demoted, 0);
|
||||||
|
|
||||||
let row = diesel::sql_query(
|
let row = diesel::sql_query("SELECT id, is_current FROM photo_insights WHERE id = ?")
|
||||||
"SELECT id, is_current FROM photo_insights WHERE id = ?",
|
.bind::<diesel::sql_types::Integer, _>(solo)
|
||||||
)
|
.get_result::<CurrentRow>(&mut conn)
|
||||||
.bind::<diesel::sql_types::Integer, _>(solo)
|
.unwrap();
|
||||||
.get_result::<CurrentRow>(&mut conn)
|
|
||||||
.unwrap();
|
|
||||||
assert!(row.is_current);
|
assert!(row.is_current);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
14
src/faces.rs
14
src/faces.rs
@@ -884,14 +884,18 @@ impl FaceDao for SqliteFaceDao {
|
|||||||
// Pair with the base64-encoded embedding string so the handler
|
// Pair with the base64-encoded embedding string so the handler
|
||||||
// doesn't need to know the wire format. Skip rows with NULL
|
// doesn't need to know the wire format. Skip rows with NULL
|
||||||
// embedding (shouldn't happen on detected rows, but defensive).
|
// embedding (shouldn't happen on detected rows, but defensive).
|
||||||
|
// `embedding.take()` moves the bytes out of the row so we can
|
||||||
|
// hand the (now-empty-embedding) row plus the encoded string
|
||||||
|
// back to the caller without cloning the whole row — at 20k
|
||||||
|
// rows × 2 KB that clone was 40 MB of pointless heap traffic
|
||||||
|
// per cluster-suggest run.
|
||||||
use base64::Engine;
|
use base64::Engine;
|
||||||
Ok(rows
|
Ok(rows
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter_map(|r| {
|
.filter_map(|mut r| {
|
||||||
r.embedding.as_ref().map(|bytes| {
|
let bytes = r.embedding.take()?;
|
||||||
let b64 = base64::engine::general_purpose::STANDARD.encode(bytes);
|
let b64 = base64::engine::general_purpose::STANDARD.encode(&bytes);
|
||||||
(r.clone(), b64)
|
Some((r, b64))
|
||||||
})
|
|
||||||
})
|
})
|
||||||
.collect())
|
.collect())
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -71,7 +71,8 @@ impl Library {
|
|||||||
if self.excluded_dirs.is_empty() {
|
if self.excluded_dirs.is_empty() {
|
||||||
return globals.to_vec();
|
return globals.to_vec();
|
||||||
}
|
}
|
||||||
let mut combined: Vec<String> = Vec::with_capacity(globals.len() + self.excluded_dirs.len());
|
let mut combined: Vec<String> =
|
||||||
|
Vec::with_capacity(globals.len() + self.excluded_dirs.len());
|
||||||
combined.extend_from_slice(globals);
|
combined.extend_from_slice(globals);
|
||||||
combined.extend(self.excluded_dirs.iter().cloned());
|
combined.extend(self.excluded_dirs.iter().cloned());
|
||||||
combined
|
combined
|
||||||
|
|||||||
@@ -350,7 +350,11 @@ pub fn run_orphan_gc(
|
|||||||
// ("revived"). Common case: a network share that briefly went
|
// ("revived"). Common case: a network share that briefly went
|
||||||
// stale comes back, image_exif gets re-populated by ingest, and
|
// stale comes back, image_exif gets re-populated by ingest, and
|
||||||
// the hash is no longer orphaned.
|
// the hash is no longer orphaned.
|
||||||
let revived = state.pending.difference(&orphans).cloned().collect::<Vec<_>>();
|
let revived = state
|
||||||
|
.pending
|
||||||
|
.difference(&orphans)
|
||||||
|
.cloned()
|
||||||
|
.collect::<Vec<_>>();
|
||||||
if !revived.is_empty() {
|
if !revived.is_empty() {
|
||||||
for h in &revived {
|
for h in &revived {
|
||||||
state.pending.remove(h);
|
state.pending.remove(h);
|
||||||
@@ -438,7 +442,10 @@ pub fn run_orphan_gc(
|
|||||||
state.pending.len(),
|
state.pending.len(),
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
debug!("orphan-gc: no changes this tick (pending: {})", state.pending.len());
|
debug!(
|
||||||
|
"orphan-gc: no changes this tick (pending: {})",
|
||||||
|
state.pending.len()
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
stats
|
stats
|
||||||
@@ -458,12 +465,7 @@ pub fn all_libraries_online(libs: &[Library], health: &LibraryHealthMap) -> bool
|
|||||||
let guard = health.read().unwrap_or_else(|e| e.into_inner());
|
let guard = health.read().unwrap_or_else(|e| e.into_inner());
|
||||||
libs.iter()
|
libs.iter()
|
||||||
.filter(|lib| lib.enabled)
|
.filter(|lib| lib.enabled)
|
||||||
.all(|lib| {
|
.all(|lib| guard.get(&lib.id).map(|h| h.is_online()).unwrap_or(false))
|
||||||
guard
|
|
||||||
.get(&lib.id)
|
|
||||||
.map(|h| h.is_online())
|
|
||||||
.unwrap_or(false)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(QueryableByName, Debug)]
|
#[derive(QueryableByName, Debug)]
|
||||||
@@ -504,18 +506,15 @@ fn delete_hash_keyed_rows(
|
|||||||
|
|
||||||
use crate::database::schema::{face_detections, photo_insights, tagged_photo};
|
use crate::database::schema::{face_detections, photo_insights, tagged_photo};
|
||||||
|
|
||||||
let faces = diesel::delete(
|
let faces =
|
||||||
face_detections::table.filter(face_detections::content_hash.eq_any(hashes)),
|
diesel::delete(face_detections::table.filter(face_detections::content_hash.eq_any(hashes)))
|
||||||
)
|
.execute(conn)?;
|
||||||
.execute(conn)?;
|
let tags =
|
||||||
let tags = diesel::delete(
|
diesel::delete(tagged_photo::table.filter(tagged_photo::content_hash.eq_any(hashes)))
|
||||||
tagged_photo::table.filter(tagged_photo::content_hash.eq_any(hashes)),
|
.execute(conn)?;
|
||||||
)
|
let insights =
|
||||||
.execute(conn)?;
|
diesel::delete(photo_insights::table.filter(photo_insights::content_hash.eq_any(hashes)))
|
||||||
let insights = diesel::delete(
|
.execute(conn)?;
|
||||||
photo_insights::table.filter(photo_insights::content_hash.eq_any(hashes)),
|
|
||||||
)
|
|
||||||
.execute(conn)?;
|
|
||||||
|
|
||||||
Ok((faces, tags, insights))
|
Ok((faces, tags, insights))
|
||||||
}
|
}
|
||||||
@@ -605,7 +604,10 @@ mod tests {
|
|||||||
n: i64,
|
n: i64,
|
||||||
}
|
}
|
||||||
fn count(conn: &mut SqliteConnection, sql: &str) -> i64 {
|
fn count(conn: &mut SqliteConnection, sql: &str) -> i64 {
|
||||||
diesel::sql_query(sql).get_result::<CountRow>(conn).unwrap().n
|
diesel::sql_query(sql)
|
||||||
|
.get_result::<CountRow>(conn)
|
||||||
|
.unwrap()
|
||||||
|
.n
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -731,9 +733,18 @@ mod tests {
|
|||||||
assert_eq!(stats.deleted_tagged_photo, 1);
|
assert_eq!(stats.deleted_tagged_photo, 1);
|
||||||
assert_eq!(stats.deleted_photo_insights, 1);
|
assert_eq!(stats.deleted_photo_insights, 1);
|
||||||
|
|
||||||
assert_eq!(count(&mut conn, "SELECT COUNT(*) AS n FROM face_detections"), 0);
|
assert_eq!(
|
||||||
assert_eq!(count(&mut conn, "SELECT COUNT(*) AS n FROM tagged_photo"), 0);
|
count(&mut conn, "SELECT COUNT(*) AS n FROM face_detections"),
|
||||||
assert_eq!(count(&mut conn, "SELECT COUNT(*) AS n FROM photo_insights"), 0);
|
0
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
count(&mut conn, "SELECT COUNT(*) AS n FROM tagged_photo"),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
count(&mut conn, "SELECT COUNT(*) AS n FROM photo_insights"),
|
||||||
|
0
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
22
src/main.rs
22
src/main.rs
@@ -1736,7 +1736,10 @@ fn cleanup_orphaned_playlists(
|
|||||||
info!(" Cleanup interval: {} seconds", cleanup_interval_secs);
|
info!(" Cleanup interval: {} seconds", cleanup_interval_secs);
|
||||||
info!(" Playlist directory: {}", video_path);
|
info!(" Playlist directory: {}", video_path);
|
||||||
for lib in &libs {
|
for lib in &libs {
|
||||||
info!(" Checking sources under '{}' at {}", lib.name, lib.root_path);
|
info!(
|
||||||
|
" Checking sources under '{}' at {}",
|
||||||
|
lib.name, lib.root_path
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
@@ -1752,12 +1755,7 @@ fn cleanup_orphaned_playlists(
|
|||||||
let guard = library_health.read().unwrap_or_else(|e| e.into_inner());
|
let guard = library_health.read().unwrap_or_else(|e| e.into_inner());
|
||||||
let stale: Vec<String> = libs
|
let stale: Vec<String> = libs
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|lib| {
|
.filter(|lib| guard.get(&lib.id).map(|h| !h.is_online()).unwrap_or(false))
|
||||||
guard
|
|
||||||
.get(&lib.id)
|
|
||||||
.map(|h| !h.is_online())
|
|
||||||
.unwrap_or(false)
|
|
||||||
})
|
|
||||||
.map(|lib| lib.name.clone())
|
.map(|lib| lib.name.clone())
|
||||||
.collect();
|
.collect();
|
||||||
if !stale.is_empty() {
|
if !stale.is_empty() {
|
||||||
@@ -2170,13 +2168,9 @@ fn watch_files(
|
|||||||
// requirement is enforced inside run_orphan_gc; we
|
// requirement is enforced inside run_orphan_gc; we
|
||||||
// pass the current all-online flag and the function
|
// pass the current all-online flag and the function
|
||||||
// tracks the previous tick's flag in OrphanGcState.
|
// tracks the previous tick's flag in OrphanGcState.
|
||||||
let all_online =
|
let all_online = library_maintenance::all_libraries_online(&libs, &library_health);
|
||||||
library_maintenance::all_libraries_online(&libs, &library_health);
|
let _ =
|
||||||
let _ = library_maintenance::run_orphan_gc(
|
library_maintenance::run_orphan_gc(&mut conn, &mut orphan_gc_state, all_online);
|
||||||
&mut conn,
|
|
||||||
&mut orphan_gc_state,
|
|
||||||
all_online,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if is_full_scan {
|
if is_full_scan {
|
||||||
|
|||||||
Reference in New Issue
Block a user