feat(embeddings): record the embedding contract and refuse a mismatched config (#189)

Stored vectors are only searchable with the backend, model, width and
prefixes that built them, and nothing enforced that: changing [embedding]
(or backend) without reembed_embeddings silently broke similarity search,
and the per-row dim check missed same-width model or prefix swaps.

- embedding_contract: one row (backend, model, dim, both prefixes). The
  server checks it once after migrations; the src/bin tools check it on
  their first embed. No row yet = record the running config.
- On a mismatch every embed (embed_many, so queries and documents) fails
  with a message naming each changed field and the fix. Mixing vector
  spaces corrupts the corpus for good; refusing only pauses semantic
  search until the config is put back or the corpus re-embedded.
- reembed_embeddings bypasses the guard and records the contract after a
  full run (all four tables, no --limit, not --dry-run).
- A DB error in the check leaves embedding allowed and is logged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Cameron CordesandClaude Opus 5.5 committed 2026-11-03 12:06:54 -05:00
1 parent 3ccce38505
commit 070f20b295
8 files changed
+374 -1

No files matched your search

+37
View File
@@ -25,10 +25,13 @@ use clap::Parser;
use diesel::prelude::*;
use diesel::sql_query;
use diesel::sqlite::SqliteConnection;
use image_api::ai::embedding_contract::{self, EmbeddingContract};
use image_api::ai::{LocalLlm, strip_summary_boilerplate};
use image_api::bin_progress;
use std::env;
const ALL_TABLES: [&str; 4] = ["summaries", "calendar", "search", "entities"];
#[derive(Parser, Debug)]
#[command(author, version, about = "Re-embed stored corpora via the configured local backend", long_about = None)]
struct Args {
@@ -284,6 +287,21 @@ async fn main() -> Result<()> {
let mut conn = SqliteConnection::establish(&database_url)
.with_context(|| format!("connecting to {}", database_url))?;
// This tool exists to move the corpus to the configured contract, so it
// embeds even while the stored one differs (#189).
embedding_contract::bypass();
let current = EmbeddingContract::current();
match embedding_contract::load(&mut conn) {
Ok(Some(stored)) if stored != current => {
println!(
"Stored embedding contract differs: {}",
stored.differences(&current).join("; ")
);
}
Ok(_) => {}
Err(e) => println!("Couldn't read the stored embedding contract: {}", e),
}
let llm = LocalLlm::from_config();
let model_version = llm.embedding_model_version();
println!("Embedding via '{}'", model_version);
@@ -453,6 +471,25 @@ async fn main() -> Result<()> {
report_similarity("entities", sims);
}
// Every table re-embedded in full: the corpus now follows the configured
// contract, so record it and the server stops refusing embeds.
let full_run =
!args.dry_run && args.limit.is_none() && ALL_TABLES.iter().all(|t| tables.contains(t));
if full_run {
embedding_contract::record(&mut conn, &current)
.context("recording the embedding contract")?;
println!(
"Recorded the embedding contract: backend={} model={} dim={}",
current.backend, current.model, current.dim
);
} else if !args.dry_run {
println!(
"Embedding contract not recorded: only a full run (all of {}, no --limit) \
leaves the whole corpus on the new config",
ALL_TABLES.join(", ")
);
}
println!(
"\n{}",
if args.dry_run {