Create Insight Generation Feature

Added integration with Messages API and Ollama
This commit is contained in:
Cameron
2026-01-03 10:30:37 -05:00
parent 0dfec4c8c5
commit 1171f19845
18 changed files with 1365 additions and 34 deletions

154
src/ai/handlers.rs Normal file
View File

@@ -0,0 +1,154 @@
use actix_web::{HttpResponse, Responder, delete, get, post, web};
use serde::{Deserialize, Serialize};
use crate::ai::InsightGenerator;
use crate::data::Claims;
use crate::database::InsightDao;
#[derive(Debug, Deserialize)]
pub struct GeneratePhotoInsightRequest {
pub file_path: String,
}
#[derive(Debug, Deserialize)]
pub struct GetPhotoInsightQuery {
pub path: String,
}
#[derive(Debug, Serialize)]
pub struct PhotoInsightResponse {
pub id: i32,
pub file_path: String,
pub title: String,
pub summary: String,
pub generated_at: i64,
pub model_version: String,
}
/// POST /insights/generate - Generate insight for a specific photo
#[post("/insights/generate")]
pub async fn generate_insight_handler(
_claims: Claims,
request: web::Json<GeneratePhotoInsightRequest>,
insight_generator: web::Data<InsightGenerator>,
) -> impl Responder {
log::info!(
"Manual insight generation triggered for photo: {}",
request.file_path
);
// Generate insight
match insight_generator
.generate_insight_for_photo(&request.file_path)
.await
{
Ok(()) => HttpResponse::Ok().json(serde_json::json!({
"success": true,
"message": "Insight generated successfully"
})),
Err(e) => {
log::error!("Failed to generate insight: {:?}", e);
HttpResponse::InternalServerError().json(serde_json::json!({
"error": format!("Failed to generate insight: {:?}", e)
}))
}
}
}
/// GET /insights?path=/path/to/photo.jpg - Fetch insight for specific photo
#[get("/insights")]
pub async fn get_insight_handler(
_claims: Claims,
query: web::Query<GetPhotoInsightQuery>,
insight_dao: web::Data<std::sync::Mutex<Box<dyn InsightDao>>>,
) -> impl Responder {
log::debug!("Fetching insight for {}", query.path);
let otel_context = opentelemetry::Context::new();
let mut dao = insight_dao.lock().expect("Unable to lock InsightDao");
match dao.get_insight(&otel_context, &query.path) {
Ok(Some(insight)) => {
let response = PhotoInsightResponse {
id: insight.id,
file_path: insight.file_path,
title: insight.title,
summary: insight.summary,
generated_at: insight.generated_at,
model_version: insight.model_version,
};
HttpResponse::Ok().json(response)
}
Ok(None) => HttpResponse::NotFound().json(serde_json::json!({
"error": "Insight not found"
})),
Err(e) => {
log::error!("Failed to fetch insight ({}): {:?}", &query.path, e);
HttpResponse::InternalServerError().json(serde_json::json!({
"error": format!("Failed to fetch insight: {:?}", e)
}))
}
}
}
/// DELETE /insights?path=/path/to/photo.jpg - Remove insight (will regenerate on next request)
#[delete("/insights")]
pub async fn delete_insight_handler(
_claims: Claims,
query: web::Query<GetPhotoInsightQuery>,
insight_dao: web::Data<std::sync::Mutex<Box<dyn InsightDao>>>,
) -> impl Responder {
log::info!("Deleting insight for {}", query.path);
let otel_context = opentelemetry::Context::new();
let mut dao = insight_dao.lock().expect("Unable to lock InsightDao");
match dao.delete_insight(&otel_context, &query.path) {
Ok(()) => HttpResponse::Ok().json(serde_json::json!({
"success": true,
"message": "Insight deleted successfully"
})),
Err(e) => {
log::error!("Failed to delete insight: {:?}", e);
HttpResponse::InternalServerError().json(serde_json::json!({
"error": format!("Failed to delete insight: {:?}", e)
}))
}
}
}
/// GET /insights/all - Get all insights
#[get("/insights/all")]
pub async fn get_all_insights_handler(
_claims: Claims,
insight_dao: web::Data<std::sync::Mutex<Box<dyn InsightDao>>>,
) -> impl Responder {
log::debug!("Fetching all insights");
let otel_context = opentelemetry::Context::new();
let mut dao = insight_dao.lock().expect("Unable to lock InsightDao");
match dao.get_all_insights(&otel_context) {
Ok(insights) => {
let responses: Vec<PhotoInsightResponse> = insights
.into_iter()
.map(|insight| PhotoInsightResponse {
id: insight.id,
file_path: insight.file_path,
title: insight.title,
summary: insight.summary,
generated_at: insight.generated_at,
model_version: insight.model_version,
})
.collect();
HttpResponse::Ok().json(responses)
}
Err(e) => {
log::error!("Failed to fetch all insights: {:?}", e);
HttpResponse::InternalServerError().json(serde_json::json!({
"error": format!("Failed to fetch insights: {:?}", e)
}))
}
}
}

239
src/ai/insight_generator.rs Normal file
View File

@@ -0,0 +1,239 @@
use anyhow::Result;
use chrono::Utc;
use serde::Deserialize;
use std::sync::{Arc, Mutex};
use crate::ai::ollama::OllamaClient;
use crate::ai::sms_client::SmsApiClient;
use crate::database::models::InsertPhotoInsight;
use crate::database::{ExifDao, InsightDao};
use crate::memories::extract_date_from_filename;
#[derive(Deserialize)]
struct NominatimResponse {
display_name: Option<String>,
address: Option<NominatimAddress>,
}
#[derive(Deserialize)]
struct NominatimAddress {
city: Option<String>,
town: Option<String>,
village: Option<String>,
county: Option<String>,
state: Option<String>,
country: Option<String>,
}
#[derive(Clone)]
pub struct InsightGenerator {
ollama: OllamaClient,
sms_client: SmsApiClient,
insight_dao: Arc<Mutex<Box<dyn InsightDao>>>,
exif_dao: Arc<Mutex<Box<dyn ExifDao>>>,
}
impl InsightGenerator {
pub fn new(
ollama: OllamaClient,
sms_client: SmsApiClient,
insight_dao: Arc<Mutex<Box<dyn InsightDao>>>,
exif_dao: Arc<Mutex<Box<dyn ExifDao>>>,
) -> Self {
Self {
ollama,
sms_client,
insight_dao,
exif_dao,
}
}
/// Extract contact name from file path
/// e.g., "Sarah/img.jpeg" -> Some("Sarah")
/// e.g., "img.jpeg" -> None
fn extract_contact_from_path(file_path: &str) -> Option<String> {
use std::path::Path;
let path = Path::new(file_path);
let components: Vec<_> = path.components().collect();
// If path has at least 2 components (directory + file), extract first directory
if components.len() >= 2 {
if let Some(component) = components.first() {
if let Some(os_str) = component.as_os_str().to_str() {
return Some(os_str.to_string());
}
}
}
None
}
/// Generate AI insight for a single photo
pub async fn generate_insight_for_photo(&self, file_path: &str) -> Result<()> {
log::info!("Generating insight for photo: {}", file_path);
// 1. Get EXIF data for the photo
let otel_context = opentelemetry::Context::new();
let exif = {
let mut exif_dao = self.exif_dao.lock().expect("Unable to lock ExifDao");
exif_dao
.get_exif(&otel_context, file_path)
.map_err(|e| anyhow::anyhow!("Failed to get EXIF: {:?}", e))?
};
// Get full timestamp for proximity-based message filtering
let timestamp = if let Some(ts) = exif.as_ref().and_then(|e| e.date_taken) {
ts
} else {
log::warn!("No date_taken in EXIF for {}, trying filename", file_path);
extract_date_from_filename(file_path)
.map(|dt| dt.timestamp())
.unwrap_or_else(|| Utc::now().timestamp())
};
let date_taken = chrono::DateTime::from_timestamp(timestamp, 0)
.map(|dt| dt.date_naive())
.unwrap_or_else(|| Utc::now().date_naive());
// 3. Extract contact name from file path
let contact = Self::extract_contact_from_path(file_path);
log::info!("Extracted contact from path: {:?}", contact);
// 4. Fetch SMS messages for the contact (±1 day)
// Pass the full timestamp for proximity sorting
let sms_messages = self
.sms_client
.fetch_messages_for_contact(contact.as_deref(), timestamp)
.await
.unwrap_or_else(|e| {
log::error!("Failed to fetch SMS messages: {}", e);
Vec::new()
});
log::info!(
"Fetched {} SMS messages closest to {}",
sms_messages.len(),
chrono::DateTime::from_timestamp(timestamp, 0)
.map(|dt| dt.format("%Y-%m-%d %H:%M:%S").to_string())
.unwrap_or_else(|| "unknown time".to_string())
);
// 5. Summarize SMS context
let sms_summary = if !sms_messages.is_empty() {
match self
.sms_client
.summarize_context(&sms_messages, &self.ollama)
.await
{
Ok(summary) => Some(summary),
Err(e) => {
log::warn!("Failed to summarize SMS context: {}", e);
None
}
}
} else {
None
};
// 6. Get location name from GPS coordinates
let location = match exif {
Some(exif) => {
if let (Some(lat), Some(lon)) = (exif.gps_latitude, exif.gps_longitude) {
self.reverse_geocode(lat, lon).await
} else {
None
}
}
None => None,
};
log::info!(
"Photo context: date={}, location={:?}, sms_messages={}",
date_taken,
location,
sms_messages.len()
);
// 7. Generate title and summary with Ollama
let title = self
.ollama
.generate_photo_title(date_taken, location.as_deref(), sms_summary.as_deref())
.await?;
let summary = self
.ollama
.generate_photo_summary(date_taken, location.as_deref(), sms_summary.as_deref())
.await?;
log::info!("Generated title: {}", title);
log::info!("Generated summary: {}", summary);
// 8. Store in database
let insight = InsertPhotoInsight {
file_path: file_path.to_string(),
title,
summary,
generated_at: Utc::now().timestamp(),
model_version: self.ollama.model.clone(),
};
let mut dao = self.insight_dao.lock().expect("Unable to lock InsightDao");
dao.store_insight(&otel_context, insight)
.map_err(|e| anyhow::anyhow!("Failed to store insight: {:?}", e))?;
log::info!("Successfully stored insight for {}", file_path);
Ok(())
}
/// Reverse geocode GPS coordinates to human-readable place names
async fn reverse_geocode(&self, lat: f64, lon: f64) -> Option<String> {
let url = format!(
"https://nominatim.openstreetmap.org/reverse?format=json&lat={}&lon={}",
lat, lon
);
let client = reqwest::Client::new();
let response = client
.get(&url)
.header("User-Agent", "ImageAPI/1.0") // Nominatim requires User-Agent
.send()
.await
.ok()?;
if !response.status().is_success() {
log::warn!(
"Geocoding failed for {}, {}: {}",
lat,
lon,
response.status()
);
return None;
}
let data: NominatimResponse = response.json().await.ok()?;
// Try to build a concise location name
if let Some(addr) = data.address {
let mut parts = Vec::new();
// Prefer city/town/village
if let Some(city) = addr.city.or(addr.town).or(addr.village) {
parts.push(city);
}
// Add state if available
if let Some(state) = addr.state {
parts.push(state);
}
if !parts.is_empty() {
return Some(parts.join(", "));
}
}
// Fallback to display_name if structured address not available
data.display_name
}
}

11
src/ai/mod.rs Normal file
View File

@@ -0,0 +1,11 @@
pub mod handlers;
pub mod insight_generator;
pub mod ollama;
pub mod sms_client;
pub use handlers::{
delete_insight_handler, generate_insight_handler, get_all_insights_handler, get_insight_handler,
};
pub use insight_generator::InsightGenerator;
pub use ollama::OllamaClient;
pub use sms_client::SmsApiClient;

173
src/ai/ollama.rs Normal file
View File

@@ -0,0 +1,173 @@
use anyhow::Result;
use chrono::NaiveDate;
use reqwest::Client;
use serde::{Deserialize, Serialize};
use crate::memories::MemoryItem;
#[derive(Clone)]
pub struct OllamaClient {
client: Client,
pub base_url: String,
pub model: String,
}
impl OllamaClient {
pub fn new(base_url: String, model: String) -> Self {
Self {
client: Client::new(),
base_url,
model,
}
}
/// Extract final answer from thinking model output
/// Handles <think>...</think> tags and takes everything after
fn extract_final_answer(&self, response: &str) -> String {
let response = response.trim();
// Look for </think> tag and take everything after it
if let Some(pos) = response.find("</think>") {
let answer = response[pos + 8..].trim();
if !answer.is_empty() {
return answer.to_string();
}
}
// Fallback: return the whole response trimmed
response.to_string()
}
pub async fn generate(&self, prompt: &str, system: Option<&str>) -> Result<String> {
log::debug!("=== Ollama Request ===");
log::debug!("Model: {}", self.model);
if let Some(sys) = system {
log::debug!("System: {}", sys);
}
log::debug!("Prompt:\n{}", prompt);
log::debug!("=====================");
let request = OllamaRequest {
model: self.model.clone(),
prompt: prompt.to_string(),
stream: false,
system: system.map(|s| s.to_string()),
};
let response = self
.client
.post(&format!("{}/api/generate", self.base_url))
.json(&request)
.send()
.await?;
if !response.status().is_success() {
let status = response.status();
let error_body = response.text().await.unwrap_or_default();
log::error!("Ollama request failed: {} - {}", status, error_body);
return Err(anyhow::anyhow!(
"Ollama request failed: {} - {}",
status,
error_body
));
}
let result: OllamaResponse = response.json().await?;
log::debug!("=== Ollama Response ===");
log::debug!("Raw response: {}", result.response.trim());
log::debug!("=======================");
// Extract final answer from thinking model output
let cleaned = self.extract_final_answer(&result.response);
log::debug!("=== Cleaned Response ===");
log::debug!("Final answer: {}", cleaned);
log::debug!("========================");
Ok(cleaned)
}
/// Generate a title for a single photo based on its context
pub async fn generate_photo_title(
&self,
date: NaiveDate,
location: Option<&str>,
sms_summary: Option<&str>,
) -> Result<String> {
let location_str = location.unwrap_or("Unknown location");
let sms_str = sms_summary.unwrap_or("No messages");
let prompt = format!(
r#"Create a short title (maximum 8 words) for this photo:
Date: {}
Location: {}
Messages: {}
Use specific details from the context above. If no specific details are available, use a simple descriptive title.
Return ONLY the title, nothing else."#,
date.format("%B %d, %Y"),
location_str,
sms_str
);
let system =
"You are a memory assistant. Use only the information provided. Do not invent details.";
let title = self.generate(&prompt, Some(system)).await?;
Ok(title.trim().trim_matches('"').to_string())
}
/// Generate a summary for a single photo based on its context
pub async fn generate_photo_summary(
&self,
date: NaiveDate,
location: Option<&str>,
sms_summary: Option<&str>,
) -> Result<String> {
let location_str = location.unwrap_or("somewhere");
let sms_str = sms_summary.unwrap_or("No messages");
let prompt = format!(
r#"Write a brief 1-2 paragraph description of this moment based on the available information:
Date: {}
Location: {}
Messages: {}
Use only the specific details provided above. Mention people's names, places, or activities if they appear in the context. Write in first person as Cam in a casual but fluent tone. If limited information is available, keep it simple and factual. If the location is unknown omit it"#,
date.format("%B %d, %Y"),
location_str,
sms_str
);
let system = "You are a memory refreshing assistant. Use only the information provided. Do not invent details. Help me remember this day.";
self.generate(&prompt, Some(system)).await
}
}
pub struct MemoryContext {
pub date: NaiveDate,
pub photos: Vec<MemoryItem>,
pub sms_summary: Option<String>,
pub locations: Vec<String>,
pub cameras: Vec<String>,
}
#[derive(Serialize)]
struct OllamaRequest {
model: String,
prompt: String,
stream: bool,
#[serde(skip_serializing_if = "Option::is_none")]
system: Option<String>,
}
#[derive(Deserialize)]
struct OllamaResponse {
response: String,
}

220
src/ai/sms_client.rs Normal file
View File

@@ -0,0 +1,220 @@
use anyhow::Result;
use chrono::NaiveDate;
use reqwest::Client;
use serde::Deserialize;
use super::ollama::OllamaClient;
#[derive(Clone)]
pub struct SmsApiClient {
client: Client,
base_url: String,
token: Option<String>,
}
impl SmsApiClient {
pub fn new(base_url: String, token: Option<String>) -> Self {
Self {
client: Client::new(),
base_url,
token,
}
}
pub async fn fetch_messages_for_date(&self, date: NaiveDate) -> Result<Vec<SmsMessage>> {
// Calculate date range (midnight to midnight in local time)
let start = date
.and_hms_opt(0, 0, 0)
.ok_or_else(|| anyhow::anyhow!("Invalid start time"))?;
let end = date
.and_hms_opt(23, 59, 59)
.ok_or_else(|| anyhow::anyhow!("Invalid end time"))?;
let start_ts = start.and_utc().timestamp();
let end_ts = end.and_utc().timestamp();
self.fetch_messages(start_ts, end_ts, None, None).await
}
/// Fetch messages for a specific contact within ±1 day of the given timestamp
/// Falls back to all contacts if no messages found for the specific contact
/// Messages are sorted by proximity to the center timestamp
pub async fn fetch_messages_for_contact(
&self,
contact: Option<&str>,
center_timestamp: i64,
) -> Result<Vec<SmsMessage>> {
use chrono::Duration;
// Calculate ±1 day range around the center timestamp
let center_dt = chrono::DateTime::from_timestamp(center_timestamp, 0)
.ok_or_else(|| anyhow::anyhow!("Invalid timestamp"))?;
let start_dt = center_dt - Duration::days(1);
let end_dt = center_dt + Duration::days(1);
let start_ts = start_dt.timestamp();
let end_ts = end_dt.timestamp();
// If contact specified, try fetching for that contact first
if let Some(contact_name) = contact {
log::info!(
"Fetching SMS for contact: {} (±1 day from {})",
contact_name,
center_dt.format("%Y-%m-%d %H:%M:%S")
);
let messages = self
.fetch_messages(start_ts, end_ts, Some(contact_name), Some(center_timestamp))
.await?;
if !messages.is_empty() {
log::info!(
"Found {} messages for contact {}",
messages.len(),
contact_name
);
return Ok(messages);
}
log::info!(
"No messages found for contact {}, falling back to all contacts",
contact_name
);
}
// Fallback to all contacts
log::info!(
"Fetching all SMS messages (±1 day from {})",
center_dt.format("%Y-%m-%d %H:%M:%S")
);
self.fetch_messages(start_ts, end_ts, None, Some(center_timestamp))
.await
}
/// Internal method to fetch messages with optional contact filter and timestamp sorting
async fn fetch_messages(
&self,
start_ts: i64,
end_ts: i64,
contact: Option<&str>,
center_timestamp: Option<i64>,
) -> Result<Vec<SmsMessage>> {
// Call Django endpoint
let mut url = format!(
"{}/api/messages/by-date-range/?start_date={}&end_date={}",
self.base_url, start_ts, end_ts
);
// Add contact filter if provided
if let Some(contact_name) = contact {
url.push_str(&format!("&contact={}", urlencoding::encode(contact_name)));
}
// Add timestamp for proximity sorting if provided
if let Some(ts) = center_timestamp {
url.push_str(&format!("&timestamp={}", ts));
}
log::debug!("Fetching SMS messages from: {}", url);
let mut request = self.client.get(&url);
// Add authorization header if token exists
if let Some(token) = &self.token {
request = request.header("Authorization", format!("Bearer {}", token));
}
let response = request.send().await?;
log::debug!("SMS API response status: {}", response.status());
if !response.status().is_success() {
let status = response.status();
let error_body = response.text().await.unwrap_or_default();
log::error!("SMS API request failed: {} - {}", status, error_body);
return Err(anyhow::anyhow!(
"SMS API request failed: {} - {}",
status,
error_body
));
}
let data: SmsApiResponse = response.json().await?;
// Convert to internal format
Ok(data
.messages
.into_iter()
.map(|m| SmsMessage {
contact: m.contact_name,
body: m.body,
timestamp: m.date,
is_sent: m.type_ == 2, // type 2 = sent
})
.collect())
}
pub async fn summarize_context(
&self,
messages: &[SmsMessage],
ollama: &OllamaClient,
) -> Result<String> {
if messages.is_empty() {
return Ok(String::from("No messages on this day"));
}
// Create prompt for Ollama with sender/receiver distinction
let messages_text: String = messages
.iter()
.take(60) // Limit to avoid token overflow
.map(|m| {
if m.is_sent {
format!("Me: {}", m.body)
} else {
format!("{}: {}", m.contact, m.body)
}
})
.collect::<Vec<_>>()
.join("\n");
let prompt = format!(
r#"Summarize these messages in up to 4-5 sentences. Focus on key topics, places, people mentioned, and the overall context of the conversations.
Messages:
{}
Summary:"#,
messages_text
);
ollama
.generate(
&prompt,
// Some("You are a summarizer for the purposes of jogging my memory and highlighting events and situations."),
Some("You are the keeper of memories, ingest the context and give me a casual summary of the moment."),
)
.await
}
}
#[derive(Debug, Clone)]
pub struct SmsMessage {
pub contact: String,
pub body: String,
pub timestamp: i64,
pub is_sent: bool,
}
#[derive(Deserialize)]
struct SmsApiResponse {
messages: Vec<SmsApiMessage>,
}
#[derive(Deserialize)]
struct SmsApiMessage {
contact_name: String,
body: String,
date: i64,
#[serde(rename = "type")]
type_: i32,
}