knowledge: cosine dedup, fact create endpoint, recall nudge

Phase 1 of the knowledge curation work. Three small server-side changes
to support an Apollo-side curation surface and reduce the agent's near-
duplicate output rate going forward:

- upsert_entity grows an embedding-cosine fallback after the exact name
  match misses. New entities whose embedding sits above
  ENTITY_DEDUP_COSINE_THRESHOLD (default 0.92) against any same-type
  active entity collapse onto the existing row. Eliminates the Sarah /
  Sara / Sarah J. trio the FTS5 prefix check was missing.
- POST /knowledge/facts symmetric with the existing PATCH/DELETE so the
  curation UI can create facts directly. Persona-scoped via X-Persona-Id;
  validates subject (and optional object) entity existence; reuses
  KnowledgeDao::upsert_fact so corroboration semantics match the agent
  path.
- One sentence in build_system_content telling the agent to call
  recall_entities before store_entity when a name resembles something
  already known. Cheap; complements the DAO-layer guard.

Includes upsert_entity_collapses_near_duplicate_by_embedding test
covering both the collapse-on-near-match path and the don't-collapse-on-
unrelated-embedding path.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Cameron Cordes
2026-05-10 15:16:05 -04:00
parent 827a78dd79
commit d7aee4f228
3 changed files with 252 additions and 3 deletions

View File

@@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize};
use std::sync::Mutex;
use crate::data::Claims;
use crate::database::models::{Entity, EntityFact, EntityPhotoLink};
use crate::database::models::{Entity, EntityFact, EntityPhotoLink, InsertEntityFact};
use crate::database::{
EntityFilter, EntityPatch, FactFilter, FactPatch, KnowledgeDao, PersonaFilter, RecentActivity,
};
@@ -179,6 +179,16 @@ pub struct FactPatchRequest {
pub confidence: Option<f32>,
}
#[derive(Deserialize)]
pub struct FactCreateRequest {
pub subject_entity_id: i32,
pub predicate: String,
pub object_entity_id: Option<i32>,
pub object_value: Option<String>,
pub source_photo: Option<String>,
pub confidence: Option<f32>,
}
#[derive(Deserialize)]
pub struct EntityListQuery {
#[serde(rename = "type")]
@@ -222,7 +232,11 @@ where
.route(web::patch().to(patch_entity::<D>))
.route(web::delete().to(delete_entity::<D>)),
)
.service(web::resource("/facts").route(web::get().to(list_facts::<D>)))
.service(
web::resource("/facts")
.route(web::get().to(list_facts::<D>))
.route(web::post().to(create_fact::<D>)),
)
.service(
web::resource("/facts/{id}")
.route(web::patch().to(patch_fact::<D>))
@@ -535,6 +549,100 @@ async fn list_facts<D: KnowledgeDao + 'static>(
}
}
async fn create_fact<D: KnowledgeDao + 'static>(
req: HttpRequest,
claims: Claims,
body: web::Json<FactCreateRequest>,
dao: web::Data<Mutex<D>>,
persona_dao: PersonaDaoData,
) -> impl Responder {
if body.object_entity_id.is_none() && body.object_value.is_none() {
return HttpResponse::BadRequest().json(serde_json::json!({
"error": "object_entity_id or object_value is required"
}));
}
if body.predicate.trim().is_empty() {
return HttpResponse::BadRequest()
.json(serde_json::json!({"error": "predicate must not be empty"}));
}
// Persona scoping: facts are written under the active single persona.
// PersonaFilter::All is read-only ("hive-mind" view); callers should
// pin a specific persona for writes via X-Persona-Id.
let persona = resolve_persona_filter(&req, &claims, &persona_dao);
let (user_id, persona_id) = match &persona {
PersonaFilter::Single { user_id, persona_id } => (*user_id, persona_id.clone()),
PersonaFilter::All { user_id } => (*user_id, "default".to_string()),
};
let cx = opentelemetry::Context::current();
let mut dao = dao.lock().expect("Unable to lock KnowledgeDao");
// Verify subject entity exists.
match dao.get_entity_by_id(&cx, body.subject_entity_id) {
Ok(None) => {
return HttpResponse::BadRequest().json(serde_json::json!({
"error": format!("Subject entity {} not found", body.subject_entity_id)
}));
}
Err(e) => {
log::error!("create_fact subject lookup error: {:?}", e);
return HttpResponse::InternalServerError()
.json(serde_json::json!({"error": "Database error"}));
}
Ok(Some(_)) => {}
}
// Optional object entity validation when supplied.
if let Some(oid) = body.object_entity_id {
match dao.get_entity_by_id(&cx, oid) {
Ok(None) => {
return HttpResponse::BadRequest().json(serde_json::json!({
"error": format!("Object entity {} not found", oid)
}));
}
Err(e) => {
log::error!("create_fact object lookup error: {:?}", e);
return HttpResponse::InternalServerError()
.json(serde_json::json!({"error": "Database error"}));
}
Ok(Some(_)) => {}
}
}
let now = Utc::now().timestamp();
let confidence = body.confidence.unwrap_or(0.6).clamp(0.0, 0.95);
let insert = InsertEntityFact {
subject_entity_id: body.subject_entity_id,
predicate: body.predicate.trim().to_string(),
object_entity_id: body.object_entity_id,
object_value: body.object_value.clone(),
source_photo: body.source_photo.clone(),
source_insight_id: None,
confidence,
status: "active".to_string(),
created_at: now,
persona_id,
user_id,
};
match dao.upsert_fact(&cx, insert) {
Ok((fact, is_new)) => {
let status = if is_new {
actix_web::http::StatusCode::CREATED
} else {
actix_web::http::StatusCode::OK
};
HttpResponse::build(status).json(fact)
}
Err(e) => {
log::error!("create_fact upsert error: {:?}", e);
HttpResponse::InternalServerError().json(serde_json::json!({"error": "Database error"}))
}
}
}
async fn patch_fact<D: KnowledgeDao + 'static>(
_claims: Claims,
id: web::Path<i32>,