feat(embeddings): record the embedding contract and refuse a mismatched config (#189)
Stored vectors are only searchable with the backend, model, width and prefixes that built them, and nothing enforced that: changing [embedding] (or backend) without reembed_embeddings silently broke similarity search, and the per-row dim check missed same-width model or prefix swaps. - embedding_contract: one row (backend, model, dim, both prefixes). The server checks it once after migrations; the src/bin tools check it on their first embed. No row yet = record the running config. - On a mismatch every embed (embed_many, so queries and documents) fails with a message naming each changed field and the fix. Mixing vector spaces corrupts the corpus for good; refusing only pauses semantic search until the config is put back or the corpus re-embedded. - reembed_embeddings bypasses the guard and records the contract after a full run (all four tables, no --limit, not --dry-run). - A DB error in the check leaves embedding allowed and is logged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3ccce38505
commit
070f20b295
8 files changed
+374
-1
No files matched your search
@@ -25,10 +25,13 @@ use clap::Parser;
|
||||
use diesel::prelude::*;
|
||||
use diesel::sql_query;
|
||||
use diesel::sqlite::SqliteConnection;
|
||||
use image_api::ai::embedding_contract::{self, EmbeddingContract};
|
||||
use image_api::ai::{LocalLlm, strip_summary_boilerplate};
|
||||
use image_api::bin_progress;
|
||||
use std::env;
|
||||
|
||||
const ALL_TABLES: [&str; 4] = ["summaries", "calendar", "search", "entities"];
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(author, version, about = "Re-embed stored corpora via the configured local backend", long_about = None)]
|
||||
struct Args {
|
||||
@@ -284,6 +287,21 @@ async fn main() -> Result<()> {
|
||||
let mut conn = SqliteConnection::establish(&database_url)
|
||||
.with_context(|| format!("connecting to {}", database_url))?;
|
||||
|
||||
// This tool exists to move the corpus to the configured contract, so it
|
||||
// embeds even while the stored one differs (#189).
|
||||
embedding_contract::bypass();
|
||||
let current = EmbeddingContract::current();
|
||||
match embedding_contract::load(&mut conn) {
|
||||
Ok(Some(stored)) if stored != current => {
|
||||
println!(
|
||||
"Stored embedding contract differs: {}",
|
||||
stored.differences(¤t).join("; ")
|
||||
);
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => println!("Couldn't read the stored embedding contract: {}", e),
|
||||
}
|
||||
|
||||
let llm = LocalLlm::from_config();
|
||||
let model_version = llm.embedding_model_version();
|
||||
println!("Embedding via '{}'", model_version);
|
||||
@@ -453,6 +471,25 @@ async fn main() -> Result<()> {
|
||||
report_similarity("entities", sims);
|
||||
}
|
||||
|
||||
// Every table re-embedded in full: the corpus now follows the configured
|
||||
// contract, so record it and the server stops refusing embeds.
|
||||
let full_run =
|
||||
!args.dry_run && args.limit.is_none() && ALL_TABLES.iter().all(|t| tables.contains(t));
|
||||
if full_run {
|
||||
embedding_contract::record(&mut conn, ¤t)
|
||||
.context("recording the embedding contract")?;
|
||||
println!(
|
||||
"Recorded the embedding contract: backend={} model={} dim={}",
|
||||
current.backend, current.model, current.dim
|
||||
);
|
||||
} else if !args.dry_run {
|
||||
println!(
|
||||
"Embedding contract not recorded: only a full run (all of {}, no --limit) \
|
||||
leaves the whole corpus on the new config",
|
||||
ALL_TABLES.join(", ")
|
||||
);
|
||||
}
|
||||
|
||||
println!(
|
||||
"\n{}",
|
||||
if args.dry_run {
|
||||
|
||||
Reference in new issue
Block a user