Enhanced Insights with daily summary embeddings
Bump to 0.5.0. Added daily summary generation job
This commit is contained in:
Cameron
338
src/database/daily_summary_dao.rs
Normal file
338
src/database/daily_summary_dao.rs
Normal file
@@ -0,0 +1,338 @@
|
||||
use diesel::prelude::*;
|
||||
use diesel::sqlite::SqliteConnection;
|
||||
use serde::Serialize;
|
||||
use std::ops::DerefMut;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use crate::database::{connect, DbError, DbErrorKind};
|
||||
use crate::otel::trace_db_call;
|
||||
|
||||
/// Represents a daily conversation summary
|
||||
#[derive(Serialize, Clone, Debug)]
|
||||
pub struct DailySummary {
|
||||
pub id: i32,
|
||||
pub date: String,
|
||||
pub contact: String,
|
||||
pub summary: String,
|
||||
pub message_count: i32,
|
||||
pub created_at: i64,
|
||||
pub model_version: String,
|
||||
}
|
||||
|
||||
/// Data for inserting a new daily summary
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct InsertDailySummary {
|
||||
pub date: String,
|
||||
pub contact: String,
|
||||
pub summary: String,
|
||||
pub message_count: i32,
|
||||
pub embedding: Vec<f32>,
|
||||
pub created_at: i64,
|
||||
pub model_version: String,
|
||||
}
|
||||
|
||||
pub trait DailySummaryDao: Sync + Send {
|
||||
/// Store a daily summary with its embedding
|
||||
fn store_summary(
|
||||
&mut self,
|
||||
context: &opentelemetry::Context,
|
||||
summary: InsertDailySummary,
|
||||
) -> Result<DailySummary, DbError>;
|
||||
|
||||
/// Find semantically similar daily summaries using vector similarity
|
||||
fn find_similar_summaries(
|
||||
&mut self,
|
||||
context: &opentelemetry::Context,
|
||||
query_embedding: &[f32],
|
||||
limit: usize,
|
||||
) -> Result<Vec<DailySummary>, DbError>;
|
||||
|
||||
/// Check if a summary exists for a given date and contact
|
||||
fn summary_exists(
|
||||
&mut self,
|
||||
context: &opentelemetry::Context,
|
||||
date: &str,
|
||||
contact: &str,
|
||||
) -> Result<bool, DbError>;
|
||||
|
||||
/// Get count of summaries for a contact
|
||||
fn get_summary_count(
|
||||
&mut self,
|
||||
context: &opentelemetry::Context,
|
||||
contact: &str,
|
||||
) -> Result<i64, DbError>;
|
||||
}
|
||||
|
||||
pub struct SqliteDailySummaryDao {
|
||||
connection: Arc<Mutex<SqliteConnection>>,
|
||||
}
|
||||
|
||||
impl Default for SqliteDailySummaryDao {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl SqliteDailySummaryDao {
|
||||
pub fn new() -> Self {
|
||||
SqliteDailySummaryDao {
|
||||
connection: Arc::new(Mutex::new(connect())),
|
||||
}
|
||||
}
|
||||
|
||||
fn serialize_vector(vec: &[f32]) -> Vec<u8> {
|
||||
use zerocopy::IntoBytes;
|
||||
vec.as_bytes().to_vec()
|
||||
}
|
||||
|
||||
fn deserialize_vector(bytes: &[u8]) -> Result<Vec<f32>, DbError> {
|
||||
if bytes.len() % 4 != 0 {
|
||||
return Err(DbError::new(DbErrorKind::QueryError));
|
||||
}
|
||||
|
||||
let count = bytes.len() / 4;
|
||||
let mut vec = Vec::with_capacity(count);
|
||||
|
||||
for chunk in bytes.chunks_exact(4) {
|
||||
let float = f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]);
|
||||
vec.push(float);
|
||||
}
|
||||
|
||||
Ok(vec)
|
||||
}
|
||||
|
||||
fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
|
||||
if a.len() != b.len() {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
let dot_product: f32 = a.iter().zip(b.iter()).map(|(x, y)| x * y).sum();
|
||||
let magnitude_a: f32 = a.iter().map(|x| x * x).sum::<f32>().sqrt();
|
||||
let magnitude_b: f32 = b.iter().map(|x| x * x).sum::<f32>().sqrt();
|
||||
|
||||
if magnitude_a == 0.0 || magnitude_b == 0.0 {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
dot_product / (magnitude_a * magnitude_b)
|
||||
}
|
||||
}
|
||||
|
||||
impl DailySummaryDao for SqliteDailySummaryDao {
|
||||
fn store_summary(
|
||||
&mut self,
|
||||
context: &opentelemetry::Context,
|
||||
summary: InsertDailySummary,
|
||||
) -> Result<DailySummary, DbError> {
|
||||
trace_db_call(context, "insert", "store_summary", |_span| {
|
||||
let mut conn = self.connection.lock().expect("Unable to get DailySummaryDao");
|
||||
|
||||
// Validate embedding dimensions
|
||||
if summary.embedding.len() != 768 {
|
||||
return Err(anyhow::anyhow!(
|
||||
"Invalid embedding dimensions: {} (expected 768)",
|
||||
summary.embedding.len()
|
||||
));
|
||||
}
|
||||
|
||||
let embedding_bytes = Self::serialize_vector(&summary.embedding);
|
||||
|
||||
// INSERT OR REPLACE to handle updates if summary needs regeneration
|
||||
diesel::sql_query(
|
||||
"INSERT OR REPLACE INTO daily_conversation_summaries
|
||||
(date, contact, summary, message_count, embedding, created_at, model_version)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)"
|
||||
)
|
||||
.bind::<diesel::sql_types::Text, _>(&summary.date)
|
||||
.bind::<diesel::sql_types::Text, _>(&summary.contact)
|
||||
.bind::<diesel::sql_types::Text, _>(&summary.summary)
|
||||
.bind::<diesel::sql_types::Integer, _>(summary.message_count)
|
||||
.bind::<diesel::sql_types::Binary, _>(&embedding_bytes)
|
||||
.bind::<diesel::sql_types::BigInt, _>(summary.created_at)
|
||||
.bind::<diesel::sql_types::Text, _>(&summary.model_version)
|
||||
.execute(conn.deref_mut())
|
||||
.map_err(|e| anyhow::anyhow!("Insert error: {:?}", e))?;
|
||||
|
||||
let row_id: i32 = diesel::sql_query("SELECT last_insert_rowid() as id")
|
||||
.get_result::<LastInsertRowId>(conn.deref_mut())
|
||||
.map(|r| r.id as i32)
|
||||
.map_err(|e| anyhow::anyhow!("Failed to get last insert ID: {:?}", e))?;
|
||||
|
||||
Ok(DailySummary {
|
||||
id: row_id,
|
||||
date: summary.date,
|
||||
contact: summary.contact,
|
||||
summary: summary.summary,
|
||||
message_count: summary.message_count,
|
||||
created_at: summary.created_at,
|
||||
model_version: summary.model_version,
|
||||
})
|
||||
})
|
||||
.map_err(|_| DbError::new(DbErrorKind::InsertError))
|
||||
}
|
||||
|
||||
fn find_similar_summaries(
|
||||
&mut self,
|
||||
context: &opentelemetry::Context,
|
||||
query_embedding: &[f32],
|
||||
limit: usize,
|
||||
) -> Result<Vec<DailySummary>, DbError> {
|
||||
trace_db_call(context, "query", "find_similar_summaries", |_span| {
|
||||
let mut conn = self.connection.lock().expect("Unable to get DailySummaryDao");
|
||||
|
||||
if query_embedding.len() != 768 {
|
||||
return Err(anyhow::anyhow!(
|
||||
"Invalid query embedding dimensions: {} (expected 768)",
|
||||
query_embedding.len()
|
||||
));
|
||||
}
|
||||
|
||||
// Load all summaries with embeddings
|
||||
let results = diesel::sql_query(
|
||||
"SELECT id, date, contact, summary, message_count, embedding, created_at, model_version
|
||||
FROM daily_conversation_summaries"
|
||||
)
|
||||
.load::<DailySummaryWithVectorRow>(conn.deref_mut())
|
||||
.map_err(|e| anyhow::anyhow!("Query error: {:?}", e))?;
|
||||
|
||||
log::info!("Loaded {} daily summaries for similarity comparison", results.len());
|
||||
|
||||
// Compute similarity for each summary
|
||||
let mut scored_summaries: Vec<(f32, DailySummary)> = results
|
||||
.into_iter()
|
||||
.filter_map(|row| {
|
||||
match Self::deserialize_vector(&row.embedding) {
|
||||
Ok(embedding) => {
|
||||
let similarity = Self::cosine_similarity(query_embedding, &embedding);
|
||||
Some((
|
||||
similarity,
|
||||
DailySummary {
|
||||
id: row.id,
|
||||
date: row.date,
|
||||
contact: row.contact,
|
||||
summary: row.summary,
|
||||
message_count: row.message_count,
|
||||
created_at: row.created_at,
|
||||
model_version: row.model_version,
|
||||
},
|
||||
))
|
||||
}
|
||||
Err(e) => {
|
||||
log::warn!("Failed to deserialize embedding for summary {}: {:?}", row.id, e);
|
||||
None
|
||||
}
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Sort by similarity (highest first)
|
||||
scored_summaries.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
|
||||
|
||||
// Log similarity distribution
|
||||
if !scored_summaries.is_empty() {
|
||||
log::info!(
|
||||
"Daily summary similarity - Top: {:.3}, Median: {:.3}, Count: {}",
|
||||
scored_summaries.first().map(|(s, _)| *s).unwrap_or(0.0),
|
||||
scored_summaries.get(scored_summaries.len() / 2).map(|(s, _)| *s).unwrap_or(0.0),
|
||||
scored_summaries.len()
|
||||
);
|
||||
}
|
||||
|
||||
// Take top N and log matches
|
||||
let top_results: Vec<DailySummary> = scored_summaries
|
||||
.into_iter()
|
||||
.take(limit)
|
||||
.map(|(similarity, summary)| {
|
||||
log::info!(
|
||||
"Summary match: similarity={:.3}, date={}, contact={}, summary=\"{}\"",
|
||||
similarity,
|
||||
summary.date,
|
||||
summary.contact,
|
||||
summary.summary.chars().take(100).collect::<String>()
|
||||
);
|
||||
summary
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(top_results)
|
||||
})
|
||||
.map_err(|_| DbError::new(DbErrorKind::QueryError))
|
||||
}
|
||||
|
||||
fn summary_exists(
|
||||
&mut self,
|
||||
context: &opentelemetry::Context,
|
||||
date: &str,
|
||||
contact: &str,
|
||||
) -> Result<bool, DbError> {
|
||||
trace_db_call(context, "query", "summary_exists", |_span| {
|
||||
let mut conn = self.connection.lock().expect("Unable to get DailySummaryDao");
|
||||
|
||||
let count = diesel::sql_query(
|
||||
"SELECT COUNT(*) as count FROM daily_conversation_summaries
|
||||
WHERE date = ?1 AND contact = ?2"
|
||||
)
|
||||
.bind::<diesel::sql_types::Text, _>(date)
|
||||
.bind::<diesel::sql_types::Text, _>(contact)
|
||||
.get_result::<CountResult>(conn.deref_mut())
|
||||
.map(|r| r.count)
|
||||
.map_err(|e| anyhow::anyhow!("Count query error: {:?}", e))?;
|
||||
|
||||
Ok(count > 0)
|
||||
})
|
||||
.map_err(|_| DbError::new(DbErrorKind::QueryError))
|
||||
}
|
||||
|
||||
fn get_summary_count(
|
||||
&mut self,
|
||||
context: &opentelemetry::Context,
|
||||
contact: &str,
|
||||
) -> Result<i64, DbError> {
|
||||
trace_db_call(context, "query", "get_summary_count", |_span| {
|
||||
let mut conn = self.connection.lock().expect("Unable to get DailySummaryDao");
|
||||
|
||||
diesel::sql_query(
|
||||
"SELECT COUNT(*) as count FROM daily_conversation_summaries WHERE contact = ?1"
|
||||
)
|
||||
.bind::<diesel::sql_types::Text, _>(contact)
|
||||
.get_result::<CountResult>(conn.deref_mut())
|
||||
.map(|r| r.count)
|
||||
.map_err(|e| anyhow::anyhow!("Count query error: {:?}", e).into())
|
||||
})
|
||||
.map_err(|_| DbError::new(DbErrorKind::QueryError))
|
||||
}
|
||||
}
|
||||
|
||||
// Helper structs for raw SQL queries
|
||||
|
||||
#[derive(QueryableByName)]
|
||||
struct LastInsertRowId {
|
||||
#[diesel(sql_type = diesel::sql_types::BigInt)]
|
||||
id: i64,
|
||||
}
|
||||
|
||||
#[derive(QueryableByName)]
|
||||
struct DailySummaryWithVectorRow {
|
||||
#[diesel(sql_type = diesel::sql_types::Integer)]
|
||||
id: i32,
|
||||
#[diesel(sql_type = diesel::sql_types::Text)]
|
||||
date: String,
|
||||
#[diesel(sql_type = diesel::sql_types::Text)]
|
||||
contact: String,
|
||||
#[diesel(sql_type = diesel::sql_types::Text)]
|
||||
summary: String,
|
||||
#[diesel(sql_type = diesel::sql_types::Integer)]
|
||||
message_count: i32,
|
||||
#[diesel(sql_type = diesel::sql_types::Binary)]
|
||||
embedding: Vec<u8>,
|
||||
#[diesel(sql_type = diesel::sql_types::BigInt)]
|
||||
created_at: i64,
|
||||
#[diesel(sql_type = diesel::sql_types::Text)]
|
||||
model_version: String,
|
||||
}
|
||||
|
||||
#[derive(QueryableByName)]
|
||||
struct CountResult {
|
||||
#[diesel(sql_type = diesel::sql_types::BigInt)]
|
||||
count: i64,
|
||||
}
|
||||
Reference in New Issue
Block a user