knowledge: stamp model + backend on facts for audit

Adds two nullable TEXT columns to entity_facts —
`created_by_model` (LLM identifier) and `created_by_backend`
("local" / "hybrid" / "manual" / NULL) — so the curator can audit
which configurations produce good fact-keeping and which produce
noise.

photo_insights already carries model_version + backend, and
entity_facts.source_insight_id links to it, but:
  - source_insight_id is set post-loop, so chat-continuation and
    regenerated-insight facts lose the link.
  - JOINing per read is more friction than embedding provenance on
    the row itself.
  - Manual facts (POST /knowledge/facts) have no insight at all and
    need their own "manual" provenance marker.

Threading: execute_tool grows `model` + `backend` params, passed
from the three call sites (agentic insight loop, chat single-turn,
chat stream) using the loop-time `chat_backend.primary_model()` +
`effective_backend` already in scope. tool_store_fact stamps the
new fact accordingly; manual create_fact stamps backend="manual".
Legacy rows leave both NULL — pre-tracking data can't be back-
filled reliably from training_messages without burning compute.

Indexes are partial (WHERE NOT NULL) so legacy rows don't bloat
them, and "show me all facts from model X" stays fast.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Cameron Cordes
2026-05-10 20:05:14 -04:00
parent 85f3716379
commit f53338923d
8 changed files with 90 additions and 2 deletions

View File

@@ -0,0 +1,4 @@
DROP INDEX IF EXISTS idx_entity_facts_created_by_backend;
DROP INDEX IF EXISTS idx_entity_facts_created_by_model;
ALTER TABLE entity_facts DROP COLUMN created_by_backend;
ALTER TABLE entity_facts DROP COLUMN created_by_model;

View File

@@ -0,0 +1,30 @@
-- Track which model + backend generated each fact so the curator
-- can audit which configurations produce trustworthy knowledge.
--
-- photo_insights already carries `model_version` + `backend`, and
-- entity_facts.source_insight_id links to it — but:
-- 1. source_insight_id is only set after an insight is stored
-- (post-loop), so chat-continuation facts and facts whose insight
-- was regenerated lose the link.
-- 2. JOINing for every read is more friction than just embedding the
-- provenance on the fact row itself.
-- 3. Manual facts (POST /knowledge/facts) have no insight at all and
-- need to record "manual" as their provenance.
--
-- Two nullable TEXT columns are enough for the audit use case: model
-- (e.g. "qwen2.5:7b", "anthropic/claude-sonnet-4") and backend
-- ("local", "hybrid", "manual"). Pre-existing rows leave both NULL —
-- legacy facts predate this tracking and can't be back-filled
-- reliably from training_messages without burning compute.
ALTER TABLE entity_facts ADD COLUMN created_by_model TEXT;
ALTER TABLE entity_facts ADD COLUMN created_by_backend TEXT;
-- Indexes are cheap and useful for "show me all facts from model X"
-- audit queries — partial so the legacy NULL rows don't bloat them.
CREATE INDEX idx_entity_facts_created_by_model
ON entity_facts(created_by_model)
WHERE created_by_model IS NOT NULL;
CREATE INDEX idx_entity_facts_created_by_backend
ON entity_facts(created_by_backend)
WHERE created_by_backend IS NOT NULL;

View File

@@ -497,6 +497,8 @@ impl InsightChatService {
&normalized,
req.user_id,
&active_persona,
&model_used,
&effective_backend,
&loop_cx,
)
.await;
@@ -870,6 +872,8 @@ impl InsightChatService {
&normalized,
req.user_id,
&active_persona,
&model_used,
&effective_backend,
max_iterations,
&tx,
)
@@ -1059,6 +1063,8 @@ impl InsightChatService {
&normalized,
req.user_id,
&active_persona,
&model_used,
&effective_backend,
max_iterations,
&tx,
)
@@ -1210,6 +1216,10 @@ impl InsightChatService {
normalized: &str,
user_id: i32,
active_persona: &str,
// Provenance — stamped onto any store_fact tool call made
// during this loop. Mirrors the non-streaming chat path.
model_used: &str,
effective_backend: &str,
max_iterations: usize,
tx: &tokio::sync::mpsc::Sender<ChatStreamEvent>,
) -> Result<AgenticLoopOutcome> {
@@ -1290,6 +1300,8 @@ impl InsightChatService {
normalized,
user_id,
active_persona,
model_used,
effective_backend,
&cx,
)
.await;

View File

@@ -1554,6 +1554,13 @@ Return ONLY the summary, nothing else."#,
file_path: &str,
user_id: i32,
persona_id: &str,
// Provenance — written into entity_facts.created_by_* when
// the loop calls store_fact. The caller knows the actual
// chat-runtime model and backend (which may differ from
// ollama.primary_model in hybrid mode where chat lives on
// OpenRouter while Ollama still handles vision).
model: &str,
backend: &str,
cx: &opentelemetry::Context,
) -> String {
let result = match tool_name {
@@ -1574,7 +1581,9 @@ Return ONLY the summary, nothing else."#,
}
"store_entity" => self.tool_store_entity(arguments, ollama, cx).await,
"store_fact" => {
self.tool_store_fact(arguments, file_path, user_id, persona_id, cx)
self.tool_store_fact(
arguments, file_path, user_id, persona_id, model, backend, cx,
)
.await
}
"get_current_datetime" => Self::tool_get_current_datetime(),
@@ -2632,6 +2641,8 @@ Return ONLY the summary, nothing else."#,
file_path: &str,
user_id: i32,
persona_id: &str,
model: &str,
backend: &str,
cx: &opentelemetry::Context,
) -> String {
use crate::database::models::{InsertEntityFact, InsertEntityPhotoLink};
@@ -2700,6 +2711,8 @@ Return ONLY the summary, nothing else."#,
valid_from,
valid_until: None,
superseded_by: None,
created_by_model: Some(model.to_string()),
created_by_backend: Some(backend.to_string()),
};
let mut kdao = self
@@ -3730,6 +3743,8 @@ Return ONLY the summary, nothing else."#,
&file_path,
user_id,
&persona_id,
chat_backend.primary_model(),
&backend_label,
&loop_cx,
)
.await;

View File

@@ -1470,6 +1470,8 @@ mod tests {
valid_from: None,
valid_until: None,
superseded_by: None,
created_by_model: None,
created_by_backend: None,
},
)
.unwrap();
@@ -1691,6 +1693,8 @@ mod tests {
valid_from: None,
valid_until: None,
superseded_by: None,
created_by_model: None,
created_by_backend: None,
},
);
assert!(
@@ -1923,6 +1927,8 @@ mod tests {
valid_from: None,
valid_until: None,
superseded_by: None,
created_by_model: None,
created_by_backend: None,
},
)
.unwrap();

View File

@@ -262,6 +262,12 @@ pub struct InsertEntityFact {
/// the supersede endpoint; status flips to 'superseded' in the
/// same transaction. See migration 2026-05-10-000200.
pub superseded_by: Option<i32>,
/// Provenance for model audit — see migration 2026-05-10-000300.
/// `created_by_model` is the LLM identifier (e.g. "qwen2.5:7b",
/// "anthropic/claude-sonnet-4") or NULL for legacy / manual rows.
/// `created_by_backend` is "local" / "hybrid" / "manual" / NULL.
pub created_by_model: Option<String>,
pub created_by_backend: Option<String>,
}
#[derive(Serialize, Queryable, Clone, Debug)]
@@ -281,6 +287,8 @@ pub struct EntityFact {
pub valid_from: Option<i64>,
pub valid_until: Option<i64>,
pub superseded_by: Option<i32>,
pub created_by_model: Option<String>,
pub created_by_backend: Option<String>,
}
#[derive(Insertable)]

View File

@@ -62,6 +62,8 @@ diesel::table! {
valid_from -> Nullable<BigInt>,
valid_until -> Nullable<BigInt>,
superseded_by -> Nullable<Integer>,
created_by_model -> Nullable<Text>,
created_by_backend -> Nullable<Text>,
}
}

View File

@@ -119,6 +119,10 @@ pub struct FactDetail {
/// supersession, migration 2026-05-10-000200). Only set when
/// status == 'superseded'.
pub superseded_by: Option<i32>,
/// Provenance — see migration 2026-05-10-000300. NULL on legacy
/// rows. `created_by_backend` is "local" / "hybrid" / "manual".
pub created_by_model: Option<String>,
pub created_by_backend: Option<String>,
/// Set when another active fact has the same subject+predicate,
/// a different object, AND their valid-time intervals overlap.
/// Detected at read time by the get_entity handler grouping
@@ -432,6 +436,8 @@ async fn get_entity<D: KnowledgeDao + 'static>(
valid_from: f.valid_from,
valid_until: f.valid_until,
superseded_by: f.superseded_by,
created_by_model: f.created_by_model,
created_by_backend: f.created_by_backend,
in_conflict: false,
});
}
@@ -768,6 +774,11 @@ async fn create_fact<D: KnowledgeDao + 'static>(
valid_from: body.valid_from,
valid_until: body.valid_until,
superseded_by: None,
// Manual creation via curation UI — provenance recorded as
// "manual" with no model, distinguishing user-entered facts
// from agent-generated ones in the audit view.
created_by_model: None,
created_by_backend: Some("manual".to_string()),
};
match dao.upsert_fact(&cx, insert) {