Apps/ImageApi
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Compare commits

base: Apps/ImageApi:2ff06413c6644ec9347483cd661a0696fcd4127c
Branches Tags
Apps/ImageApi:master Apps/ImageApi:feature/ai-auto-tagging
..
compare: Apps/ImageApi:7e1c4ab3187d0c6acf597f1ccde4eef0989cbfa9
Branches Tags
Apps/ImageApi:master Apps/ImageApi:feature/ai-auto-tagging

1 Commits
2ff06413c6 .. 7e1c4ab318

Author SHA1 Message Date
Cameron Cordes 7e1c4ab318 backfill_date_taken: surface the actual diesel error in warnings 
The DAO swallowed every diesel::update failure as a flat
`anyhow!("Update error")`, then trace_db_call further reduced it to
`DbError { kind: UpdateError }`. Operators saw "update failed for lib
2 Snapchat/foo.mp4: DbError { kind: UpdateError }" with no clue why
(constraint violation? type mismatch? row vanished mid-flight? DB
locked?).

Two changes:
- Preserve the diesel error in the anyhow chain along with the input
  params (lib, rel_path, date_taken, source) so the cause is visible.
- Log the chain at warn-level inside the DAO before the trace wrapper
  collapses it to DbErrorKind::UpdateError, so the warning at the
  call site finally has something diagnosable next to it.
- Treat zero-row updates as a debug-level "row likely retired by the
  missing-file scan" rather than a hard failure — that case is benign
  and shouldn't poison the drain's error tally.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
 2026-05-07 11:07:17 -04:00
7 changed files with 155 additions and 653 deletions
Expand all files Collapse all files
src/ai/handlers.rs
-12
View File
@@ -48,11 +48,6 @@ pub struct GeneratePhotoInsightRequest {
/// falls back to `DEFAULT_FEWSHOT_INSIGHT_IDS`. /// falls back to `DEFAULT_FEWSHOT_INSIGHT_IDS`.
#[serde(default)] #[serde(default)]
pub fewshot_insight_ids: Option<Vec<i32>>, pub fewshot_insight_ids: Option<Vec<i32>>,
/// When true, drop `store_entity` / `store_fact` from the tool palette
/// for this run. Use for one-off explorations (caption-style prompts,
/// experimentation) that shouldn't pollute the persistent knowledge KB.
#[serde(default)]
pub disable_writes: bool,
} }
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
@@ -395,7 +390,6 @@ pub async fn generate_agentic_insight_handler(
request.backend.clone(), request.backend.clone(),
fewshot_examples, fewshot_examples,
fewshot_ids, fewshot_ids,
request.disable_writes,
) )
.await; .await;
@@ -648,10 +642,6 @@ pub struct ChatTurnHttpRequest {
pub max_iterations: Option<usize>, pub max_iterations: Option<usize>,
#[serde(default)] #[serde(default)]
pub amend: bool, pub amend: bool,
/// Drop store_entity / store_fact from the tool palette for this turn —
/// useful for hypothetical/exploration chats that shouldn't pollute the KB.
#[serde(default)]
pub disable_writes: bool,
} }
#[derive(Debug, Serialize)] #[derive(Debug, Serialize)]
@@ -706,7 +696,6 @@ pub async fn chat_turn_handler(
min_p: request.min_p, min_p: request.min_p,
max_iterations: request.max_iterations, max_iterations: request.max_iterations,
amend: request.amend, amend: request.amend,
disable_writes: request.disable_writes,
}; };
match app_state.insight_chat.chat_turn(chat_req).await { match app_state.insight_chat.chat_turn(chat_req).await {
@@ -921,7 +910,6 @@ pub async fn chat_stream_handler(
min_p: request.min_p, min_p: request.min_p,
max_iterations: request.max_iterations, max_iterations: request.max_iterations,
amend: request.amend, amend: request.amend,
disable_writes: request.disable_writes,
}; };
let service = app_state.insight_chat.clone(); let service = app_state.insight_chat.clone();
src/ai/insight_chat.rs
+1 -19
View File
@@ -48,10 +48,6 @@ pub struct ChatTurnRequest {
/// When true, write a new insight row (regenerating title) instead of /// When true, write a new insight row (regenerating title) instead of
/// updating training_messages on the existing row. /// updating training_messages on the existing row.
pub amend: bool, pub amend: bool,
/// When true, drop `store_entity` / `store_fact` from the tool palette
/// for this turn. Use to explore alternate phrasings or run
/// hypothetical chats without polluting the persistent KB.
pub disable_writes: bool,
} }
#[derive(Debug)] #[derive(Debug)]
@@ -366,7 +362,6 @@ impl InsightChatService {
let tools = InsightGenerator::build_tool_definitions( let tools = InsightGenerator::build_tool_definitions(
offer_describe_tool, offer_describe_tool,
self.generator.apollo_enabled(), self.generator.apollo_enabled(),
req.disable_writes,
); );
// Image base64 only needed when describe_photo is on the menu. Load // Image base64 only needed when describe_photo is on the menu. Load
@@ -402,9 +397,6 @@ impl InsightChatService {
// tighter and dispatching tools through the shared executor. // tighter and dispatching tools through the shared executor.
let loop_span = tracer.start_with_context("ai.chat.loop", &insight_cx); let loop_span = tracer.start_with_context("ai.chat.loop", &insight_cx);
let loop_cx = insight_cx.with_span(loop_span); let loop_cx = insight_cx.with_span(loop_span);
// Memoize describe_photo for this turn so repeated calls don't
// produce conflicting visual descriptions in the assistant transcript.
let describe_cache: tokio::sync::Mutex<Option<String>> = tokio::sync::Mutex::new(None);
let mut tool_calls_made = 0usize; let mut tool_calls_made = 0usize;
let mut iterations_used = 0usize; let mut iterations_used = 0usize;
let mut last_prompt_eval_count: Option<i32> = None; let mut last_prompt_eval_count: Option<i32> = None;
@@ -453,7 +445,6 @@ impl InsightChatService {
&image_base64, &image_base64,
&normalized, &normalized,
&loop_cx, &loop_cx,
Some(&describe_cache),
) )
.await; .await;
messages.push(ChatMessage::tool_result(result)); messages.push(ChatMessage::tool_result(result));
@@ -802,7 +793,6 @@ impl InsightChatService {
let tools = InsightGenerator::build_tool_definitions( let tools = InsightGenerator::build_tool_definitions(
offer_describe_tool, offer_describe_tool,
self.generator.apollo_enabled(), self.generator.apollo_enabled(),
req.disable_writes,
); );
let image_base64: Option<String> = if offer_describe_tool { let image_base64: Option<String> = if offer_describe_tool {
@@ -824,9 +814,6 @@ impl InsightChatService {
let original_system_content = annotate_system_with_budget(&mut messages, max_iterations); let original_system_content = annotate_system_with_budget(&mut messages, max_iterations);
// Per-turn describe_photo memo, same intent as the non-streaming
// path: avoid replaying conflicting visual descriptions in transcript.
let describe_cache: tokio::sync::Mutex<Option<String>> = tokio::sync::Mutex::new(None);
let mut tool_calls_made = 0usize; let mut tool_calls_made = 0usize;
let mut iterations_used = 0usize; let mut iterations_used = 0usize;
let mut last_prompt_eval_count: Option<i32> = None; let mut last_prompt_eval_count: Option<i32> = None;
@@ -902,7 +889,6 @@ impl InsightChatService {
&image_base64, &image_base64,
&normalized, &normalized,
&cx, &cx,
Some(&describe_cache),
) )
.await; .await;
let (result_preview, result_truncated) = truncate_tool_result(&result); let (result_preview, result_truncated) = truncate_tool_result(&result);
@@ -1148,12 +1134,8 @@ fn annotate_system_with_budget(
return None; return None;
} }
let original = first.content.clone(); let original = first.content.clone();
// Formatted as its own section so small models don't skim past it the
// way they tend to with parenthetical asides at the bottom of a long prompt.
// Phrasing matches the base prompt: budget = capacity, not a constraint
// to conserve. Small models otherwise tend to stop early.
first.content = format!( first.content = format!(
"{}\n\n## Budget for this chat turn\n\nYou have up to {} iterations available. Use as many as the question warrants for context-gathering, and reserve the last one for your reply.", "{}\n\n(Budget for this chat turn: up to {} tool-calling iterations. Produce your final reply before the budget is exhausted.)",
first.content, max_iterations first.content, max_iterations
); );
Some(original) Some(original)
src/ai/insight_generator.rs
+116 -577
View File
@@ -983,7 +983,7 @@ impl InsightGenerator {
// Step 1: Get FULL immediate temporal context (±4 days, ALL messages) // Step 1: Get FULL immediate temporal context (±4 days, ALL messages)
let immediate_messages = self let immediate_messages = self
.sms_client .sms_client
.fetch_messages_for_contact(contact.as_deref(), timestamp, None) .fetch_messages_for_contact(contact.as_deref(), timestamp)
.await .await
.unwrap_or_else(|e| { .unwrap_or_else(|e| {
log::error!("Failed to fetch immediate messages: {}", e); log::error!("Failed to fetch immediate messages: {}", e);
@@ -1129,7 +1129,7 @@ impl InsightGenerator {
log::info!("Using traditional time-based message retrieval (±4 days)"); log::info!("Using traditional time-based message retrieval (±4 days)");
let sms_messages = self let sms_messages = self
.sms_client .sms_client
.fetch_messages_for_contact(contact.as_deref(), timestamp, None) .fetch_messages_for_contact(contact.as_deref(), timestamp)
.await .await
.unwrap_or_else(|e| { .unwrap_or_else(|e| {
log::error!("Failed to fetch SMS messages: {}", e); log::error!("Failed to fetch SMS messages: {}", e);
@@ -1459,14 +1459,7 @@ Return ONLY the summary, nothing else."#,
// ── Tool executors for agentic loop ──────────────────────────────── // ── Tool executors for agentic loop ────────────────────────────────
/// Dispatch a tool call to the appropriate executor. /// Dispatch a tool call to the appropriate executor
///
/// `describe_photo_cache` lets the agentic loop memoize the visual
/// description across iterations — vision describes are non-deterministic
/// and re-running mid-loop produces conflicting answers in the chat
/// history. Pass `None` to skip caching (chat continuation falls through
/// to the live call each turn — the description isn't replayed in chat).
#[allow(clippy::too_many_arguments)]
pub(crate) async fn execute_tool( pub(crate) async fn execute_tool(
&self, &self,
tool_name: &str, tool_name: &str,
@@ -1475,7 +1468,6 @@ Return ONLY the summary, nothing else."#,
image_base64: &Option<String>, image_base64: &Option<String>,
file_path: &str, file_path: &str,
cx: &opentelemetry::Context, cx: &opentelemetry::Context,
describe_photo_cache: Option<&tokio::sync::Mutex<Option<String>>>,
) -> String { ) -> String {
let result = match tool_name { let result = match tool_name {
"search_rag" => self.tool_search_rag(arguments, ollama, cx).await, "search_rag" => self.tool_search_rag(arguments, ollama, cx).await,
@@ -1483,34 +1475,12 @@ Return ONLY the summary, nothing else."#,
"get_sms_messages" => self.tool_get_sms_messages(arguments, cx).await, "get_sms_messages" => self.tool_get_sms_messages(arguments, cx).await,
"get_calendar_events" => self.tool_get_calendar_events(arguments, cx).await, "get_calendar_events" => self.tool_get_calendar_events(arguments, cx).await,
"get_location_history" => self.tool_get_location_history(arguments, cx).await, "get_location_history" => self.tool_get_location_history(arguments, cx).await,
"describe_photo" => match describe_photo_cache { "get_file_tags" => self.tool_get_file_tags(arguments, cx).await,
Some(cache) => { "describe_photo" => self.tool_describe_photo(ollama, image_base64).await,
let mut guard = cache.lock().await;
if let Some(ref cached) = *guard {
log::info!(
"tool_describe_photo: returning cached description ({} chars)",
cached.len()
);
cached.clone()
} else {
let fresh = self.tool_describe_photo(ollama, image_base64).await;
// Only cache successful descriptions — error strings
// shouldn't lock in for the rest of the loop.
if !fresh.starts_with("Error") && !fresh.starts_with("No image") {
*guard = Some(fresh.clone());
}
fresh
}
}
None => self.tool_describe_photo(ollama, image_base64).await,
},
"reverse_geocode" => self.tool_reverse_geocode(arguments).await, "reverse_geocode" => self.tool_reverse_geocode(arguments).await,
"get_personal_place_at" => self.tool_get_personal_place_at(arguments).await, "get_personal_place_at" => self.tool_get_personal_place_at(arguments).await,
"recall_entities" => self.tool_recall_entities(arguments, cx).await, "recall_entities" => self.tool_recall_entities(arguments, cx).await,
"recall_facts_for_photo" => self.tool_recall_facts_for_photo(arguments, cx).await, "recall_facts_for_photo" => self.tool_recall_facts_for_photo(arguments, cx).await,
"recall_facts_for_entity" => self.tool_recall_facts_for_entity(arguments, cx).await,
"find_photos_with_entity" => self.tool_find_photos_with_entity(arguments, cx).await,
"get_exif" => self.tool_get_exif(arguments, cx).await,
"store_entity" => self.tool_store_entity(arguments, ollama, cx).await, "store_entity" => self.tool_store_entity(arguments, ollama, cx).await,
"store_fact" => self.tool_store_fact(arguments, file_path, cx).await, "store_fact" => self.tool_store_fact(arguments, file_path, cx).await,
"get_current_datetime" => Self::tool_get_current_datetime(), "get_current_datetime" => Self::tool_get_current_datetime(),
@@ -1865,7 +1835,7 @@ Return ONLY the summary, nothing else."#,
match self match self
.sms_client .sms_client
.fetch_messages_for_contact(contact.as_deref(), timestamp, Some(days_radius)) .fetch_messages_for_contact(contact.as_deref(), timestamp)
.await .await
{ {
Ok(messages) if !messages.is_empty() => { Ok(messages) if !messages.is_empty() => {
@@ -1899,8 +1869,7 @@ Return ONLY the summary, nothing else."#,
} }
} }
/// Tool: get_calendar_events — fetch calendar events near a date, /// Tool: get_calendar_events — fetch calendar events near a date
/// optionally ranked by semantic similarity to a query string.
async fn tool_get_calendar_events( async fn tool_get_calendar_events(
&self, &self,
args: &serde_json::Value, args: &serde_json::Value,
@@ -1910,11 +1879,6 @@ Return ONLY the summary, nothing else."#,
Some(d) => d, Some(d) => d,
None => return "Error: missing required parameter 'date'".to_string(), None => return "Error: missing required parameter 'date'".to_string(),
}; };
let query = args
.get("query")
.and_then(|v| v.as_str())
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty());
let days_radius = args let days_radius = args
.get("days_radius") .get("days_radius")
.and_then(|v| v.as_i64()) .and_then(|v| v.as_i64())
@@ -1932,40 +1896,18 @@ Return ONLY the summary, nothing else."#,
let timestamp = date.and_hms_opt(12, 0, 0).unwrap().and_utc().timestamp(); let timestamp = date.and_hms_opt(12, 0, 0).unwrap().and_utc().timestamp();
log::info!( log::info!(
"tool_get_calendar_events: date={}, days_radius={}, limit={}, query={:?}", "tool_get_calendar_events: date={}, days_radius={}, limit={}",
date, date,
days_radius, days_radius,
limit, limit
query
); );
// Embed the query (best-effort) so the DAO can do hybrid time + semantic ranking.
let query_embedding: Option<Vec<f32>> = match query.as_deref() {
Some(q) => match self.ollama.generate_embedding(q).await {
Ok(emb) => Some(emb),
Err(e) => {
log::warn!(
"Calendar query embedding failed, falling back to time-only: {}",
e
);
None
}
},
None => None,
};
let events = { let events = {
let mut dao = self let mut dao = self
.calendar_dao .calendar_dao
.lock() .lock()
.expect("Unable to lock CalendarEventDao"); .expect("Unable to lock CalendarEventDao");
dao.find_relevant_events_hybrid( dao.find_relevant_events_hybrid(cx, timestamp, days_radius, None, limit)
cx,
timestamp,
days_radius,
query_embedding.as_deref(),
limit,
)
.ok() .ok()
}; };
@@ -2020,11 +1962,6 @@ Return ONLY the summary, nothing else."#,
.get("days_radius") .get("days_radius")
.and_then(|v| v.as_i64()) .and_then(|v| v.as_i64())
.unwrap_or(14); .unwrap_or(14);
let limit = args
.get("limit")
.and_then(|v| v.as_i64())
.unwrap_or(20)
.clamp(1, 50) as usize;
let date = match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") { let date = match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") {
Ok(d) => d, Ok(d) => d,
@@ -2033,10 +1970,9 @@ Return ONLY the summary, nothing else."#,
let timestamp = date.and_hms_opt(12, 0, 0).unwrap().and_utc().timestamp(); let timestamp = date.and_hms_opt(12, 0, 0).unwrap().and_utc().timestamp();
log::info!( log::info!(
"tool_get_location_history: date={}, days_radius={}, limit={}", "tool_get_location_history: date={}, days_radius={}",
date, date,
days_radius, days_radius
limit
); );
let start_ts = timestamp - (days_radius * 86400); let start_ts = timestamp - (days_radius * 86400);
@@ -2054,7 +1990,7 @@ Return ONLY the summary, nothing else."#,
Some(locs) if !locs.is_empty() => { Some(locs) if !locs.is_empty() => {
let formatted: Vec<String> = locs let formatted: Vec<String> = locs
.iter() .iter()
.take(limit) .take(20)
.map(|loc| { .map(|loc| {
let dt = DateTime::from_timestamp(loc.timestamp, 0) let dt = DateTime::from_timestamp(loc.timestamp, 0)
.map(|dt| { .map(|dt| {
@@ -2090,6 +2026,34 @@ Return ONLY the summary, nothing else."#,
} }
} }
/// Tool: get_file_tags — fetch tags for a file path
async fn tool_get_file_tags(
&self,
args: &serde_json::Value,
cx: &opentelemetry::Context,
) -> String {
let file_path = match args.get("file_path").and_then(|v| v.as_str()) {
Some(p) => p.to_string(),
None => return "Error: missing required parameter 'file_path'".to_string(),
};
log::info!("tool_get_file_tags: file_path='{}'", file_path);
let tags = {
let mut dao = self.tag_dao.lock().expect("Unable to lock TagDao");
dao.get_tags_for_path(cx, &file_path).ok()
};
match tags {
Some(t) if !t.is_empty() => {
let names: Vec<String> = t.into_iter().map(|tag| tag.name).collect();
names.join(", ")
}
Some(_) => "No tags found.".to_string(),
None => "No tags found.".to_string(),
}
}
/// Tool: describe_photo — generate a visual description of the photo /// Tool: describe_photo — generate a visual description of the photo
async fn tool_describe_photo( async fn tool_describe_photo(
&self, &self,
@@ -2300,255 +2264,6 @@ Return ONLY the summary, nothing else."#,
} }
} }
/// Resolve `entity_id` directly when provided, otherwise look up by
/// `name` (+ optional `entity_type`). Returns the chosen entity, or an
/// error string suitable for use as a tool result.
fn resolve_entity_arg(
&self,
args: &serde_json::Value,
cx: &opentelemetry::Context,
) -> std::result::Result<crate::database::models::Entity, String> {
if let Some(eid) = args.get("entity_id").and_then(|v| v.as_i64()) {
let mut kdao = self
.knowledge_dao
.lock()
.expect("Unable to lock KnowledgeDao");
return match kdao.get_entity_by_id(cx, eid as i32) {
Ok(Some(e)) => Ok(e),
Ok(None) => Err(format!("No entity found with id {}.", eid)),
Err(e) => Err(format!("Error looking up entity: {:?}", e)),
};
}
let Some(name) = args.get("name").and_then(|v| v.as_str()) else {
return Err("Error: provide either 'entity_id' or 'name'".to_string());
};
let entity_type = args
.get("entity_type")
.and_then(|v| v.as_str())
.map(|s| s.to_string());
let mut kdao = self
.knowledge_dao
.lock()
.expect("Unable to lock KnowledgeDao");
match kdao.get_entity_by_name(cx, name, entity_type.as_deref()) {
Ok(matches) => {
// Prefer active rows; among active, highest confidence wins.
// Falls through to any match if no active rows exist.
let mut active: Vec<_> = matches
.iter()
.filter(|e| e.status == "active")
.cloned()
.collect();
active.sort_by(|a, b| {
b.confidence
.partial_cmp(&a.confidence)
.unwrap_or(std::cmp::Ordering::Equal)
});
if let Some(e) = active.into_iter().next() {
return Ok(e);
}
if let Some(e) = matches.into_iter().next() {
return Ok(e);
}
Err(format!(
"No entity found matching name '{}'{}.",
name,
entity_type
.map(|t| format!(" of type '{}'", t))
.unwrap_or_default()
))
}
Err(e) => Err(format!("Error looking up entity by name: {:?}", e)),
}
}
/// Tool: recall_facts_for_entity — list active facts for one entity
async fn tool_recall_facts_for_entity(
&self,
args: &serde_json::Value,
cx: &opentelemetry::Context,
) -> String {
let entity = match self.resolve_entity_arg(args, cx) {
Ok(e) => e,
Err(msg) => return msg,
};
log::info!(
"tool_recall_facts_for_entity: entity_id={}, name='{}'",
entity.id,
entity.name
);
let mut kdao = self
.knowledge_dao
.lock()
.expect("Unable to lock KnowledgeDao");
let facts = match kdao.get_facts_for_entity(cx, entity.id) {
Ok(f) => f,
Err(e) => return format!("Error fetching facts: {:?}", e),
};
let mut lines: Vec<String> = Vec::new();
lines.push(format!(
"Entity: {} (id {}, {}, confidence {:.2})",
entity.name, entity.id, entity.entity_type, entity.confidence
));
if !entity.description.is_empty() {
lines.push(format!(" description: {}", entity.description));
}
let mut wrote_any_fact = false;
for f in facts.iter().filter(|f| f.status == "active") {
wrote_any_fact = true;
let obj = if let Some(ref v) = f.object_value {
v.clone()
} else if let Some(oid) = f.object_entity_id {
kdao.get_entity_by_id(cx, oid)
.ok()
.flatten()
.map(|e| format!("{} (id {})", e.name, e.id))
.unwrap_or_else(|| format!("entity:{}", oid))
} else {
"(unknown)".to_string()
};
lines.push(format!(
" - {} {} (confidence {:.2})",
f.predicate, obj, f.confidence
));
}
if !wrote_any_fact {
lines.push(" (no active facts)".to_string());
}
lines.join("\n")
}
/// Tool: find_photos_with_entity — list photo paths linked to an entity
async fn tool_find_photos_with_entity(
&self,
args: &serde_json::Value,
cx: &opentelemetry::Context,
) -> String {
let entity = match self.resolve_entity_arg(args, cx) {
Ok(e) => e,
Err(msg) => return msg,
};
let limit = args
.get("limit")
.and_then(|v| v.as_i64())
.unwrap_or(20)
.clamp(1, 50) as usize;
log::info!(
"tool_find_photos_with_entity: entity_id={}, name='{}', limit={}",
entity.id,
entity.name,
limit
);
let links = {
let mut kdao = self
.knowledge_dao
.lock()
.expect("Unable to lock KnowledgeDao");
match kdao.get_links_for_entity(cx, entity.id) {
Ok(l) => l,
Err(e) => return format!("Error fetching links: {:?}", e),
}
};
if links.is_empty() {
return format!(
"No photos linked to entity '{}' (id {}).",
entity.name, entity.id
);
}
// Deduplicate by file_path so the same photo under multiple libraries
// shows once. Roles for the same path are unioned.
use std::collections::BTreeMap;
let mut by_path: BTreeMap<String, Vec<String>> = BTreeMap::new();
for l in links {
by_path.entry(l.file_path).or_default().push(l.role);
}
let total = by_path.len();
let mut lines = Vec::with_capacity(limit + 1);
lines.push(format!(
"Found {} photo(s) linked to '{}' (id {}); showing up to {}:",
total, entity.name, entity.id, limit
));
for (path, mut roles) in by_path.into_iter().take(limit) {
roles.sort();
roles.dedup();
lines.push(format!("- {} [roles: {}]", path, roles.join(", ")));
}
lines.join("\n")
}
/// Tool: get_exif — return the stored EXIF row for a photo file path
async fn tool_get_exif(&self, args: &serde_json::Value, cx: &opentelemetry::Context) -> String {
let file_path = match args.get("file_path").and_then(|v| v.as_str()) {
Some(p) => p.to_string(),
None => return "Error: missing required parameter 'file_path'".to_string(),
};
let normalized = normalize_path(&file_path);
log::info!("tool_get_exif: file_path='{}'", normalized);
let exif = {
let mut dao = self.exif_dao.lock().expect("Unable to lock ExifDao");
match dao.get_exif(cx, &normalized) {
Ok(e) => e,
Err(e) => return format!("Error fetching EXIF: {:?}", e),
}
};
let Some(e) = exif else {
return format!("No EXIF row found for '{}'.", normalized);
};
let mut lines: Vec<String> = Vec::new();
lines.push(format!("EXIF for {}:", normalized));
if let (Some(make), Some(model)) = (e.camera_make.as_deref(), e.camera_model.as_deref()) {
lines.push(format!(" camera: {} {}", make, model));
} else if let Some(make) = e.camera_make.as_deref() {
lines.push(format!(" camera_make: {}", make));
} else if let Some(model) = e.camera_model.as_deref() {
lines.push(format!(" camera_model: {}", model));
}
if let Some(lens) = e.lens_model.as_deref() {
lines.push(format!(" lens: {}", lens));
}
if let (Some(w), Some(h)) = (e.width, e.height) {
lines.push(format!(" dimensions: {}x{}", w, h));
}
if let Some(fl) = e.focal_length {
lines.push(format!(" focal_length: {:.1} mm", fl));
}
if let Some(ap) = e.aperture {
lines.push(format!(" aperture: f/{:.1}", ap));
}
if let Some(ref ss) = e.shutter_speed {
lines.push(format!(" shutter_speed: {}", ss));
}
if let Some(iso) = e.iso {
lines.push(format!(" iso: {}", iso));
}
if let Some(dt) = e.date_taken {
let dt_str = chrono::DateTime::from_timestamp(dt, 0)
.map(|t| t.format("%Y-%m-%d %H:%M:%S UTC").to_string())
.unwrap_or_else(|| dt.to_string());
lines.push(format!(
" date_taken: {} (source: {})",
dt_str,
e.date_taken_source.as_deref().unwrap_or("unknown")
));
}
if let (Some(lat), Some(lon)) = (e.gps_latitude, e.gps_longitude) {
lines.push(format!(" gps: {:.6}, {:.6}", lat, lon));
}
if let Some(alt) = e.gps_altitude {
lines.push(format!(" gps_altitude: {:.1} m", alt));
}
lines.join("\n")
}
/// Tool: store_entity — upsert an entity into the knowledge memory /// Tool: store_entity — upsert an entity into the knowledge memory
async fn tool_store_entity( async fn tool_store_entity(
&self, &self,
@@ -2770,30 +2485,26 @@ Return ONLY the summary, nothing else."#,
// ── Agentic insight generation ────────────────────────────────────── // ── Agentic insight generation ──────────────────────────────────────
/// Build the list of tool definitions for the agentic loop /// Build the list of tool definitions for the agentic loop
pub(crate) fn build_tool_definitions( pub(crate) fn build_tool_definitions(has_vision: bool, apollo_enabled: bool) -> Vec<Tool> {
has_vision: bool,
apollo_enabled: bool,
disable_writes: bool,
) -> Vec<Tool> {
let mut tools = vec![ let mut tools = vec![
Tool::function( Tool::function(
"search_rag", "search_rag",
"Semantic search over per-day, per-contact CONVERSATION SUMMARIES (not raw messages). Each hit is one compressed paragraph for one (date, contact) day. Use for high-level themes around a date — for specific wording, call `search_messages`. The `date` argument biases ranking toward summaries near that date and is required even when searching across all time.", "Search conversation history using semantic search. Use this to find relevant past conversations about specific topics, people, or events.",
serde_json::json!({ serde_json::json!({
"type": "object", "type": "object",
"required": ["query", "date"], "required": ["query", "date"],
"properties": { "properties": {
"query": { "query": {
"type": "string", "type": "string",
"description": "Topic / theme query (semantic). 'wedding planning', 'job search stress', 'Tahoe trip'." "description": "The search query to find relevant conversations"
}, },
"date": { "date": {
"type": "string", "type": "string",
"description": "Reference date in YYYY-MM-DD format. Used for time-decay weighting (closer summaries rank higher)." "description": "The reference date in YYYY-MM-DD format"
}, },
"contact": { "contact": {
"type": "string", "type": "string",
"description": "Optional contact name to filter results. When you know the contact, also make at least one call WITHOUT this filter to surface what else was happening that week." "description": "Optional contact name to filter results"
}, },
"limit": { "limit": {
"type": "integer", "type": "integer",
@@ -2853,7 +2564,7 @@ Return ONLY the summary, nothing else."#,
), ),
Tool::function( Tool::function(
"get_calendar_events", "get_calendar_events",
"Fetch calendar events near a specific date. Pass a `query` to rank by semantic similarity (e.g. 'wedding', 'doctor visit', 'work travel') in addition to time. Without a query, results are time-ordered.", "Fetch calendar events near a specific date. Shows scheduled events, meetings, and activities.",
serde_json::json!({ serde_json::json!({
"type": "object", "type": "object",
"required": ["date"], "required": ["date"],
@@ -2862,10 +2573,6 @@ Return ONLY the summary, nothing else."#,
"type": "string", "type": "string",
"description": "The center date in YYYY-MM-DD format" "description": "The center date in YYYY-MM-DD format"
}, },
"query": {
"type": "string",
"description": "Optional topic / theme to rank events by (semantic). Pairs well with photo-relevant cues from the visual description."
},
"days_radius": { "days_radius": {
"type": "integer", "type": "integer",
"description": "Number of days before and after the date to search (default: 7)" "description": "Number of days before and after the date to search (default: 7)"
@@ -2879,7 +2586,7 @@ Return ONLY the summary, nothing else."#,
), ),
Tool::function( Tool::function(
"get_location_history", "get_location_history",
"Fetch location-history records (lat/lon / activity / place_name) within ±days_radius days of a date. Time-ordered; no semantic ranking. Useful for reconstructing where the user was around the photo's timestamp.", "Fetch location history records near a specific date. Shows places visited and activities.",
serde_json::json!({ serde_json::json!({
"type": "object", "type": "object",
"required": ["date"], "required": ["date"],
@@ -2891,22 +2598,29 @@ Return ONLY the summary, nothing else."#,
"days_radius": { "days_radius": {
"type": "integer", "type": "integer",
"description": "Number of days before and after the date to search (default: 14)" "description": "Number of days before and after the date to search (default: 14)"
}, }
"limit": { }
"type": "integer", }),
"description": "Maximum number of records to return (default: 20, max: 50)" ),
Tool::function(
"get_file_tags",
"Get tags/labels that have been applied to a specific photo file.",
serde_json::json!({
"type": "object",
"required": ["file_path"],
"properties": {
"file_path": {
"type": "string",
"description": "The file path of the photo to get tags for"
} }
} }
}), }),
), ),
]; ];
// (`get_file_tags` was removed — the tags for the current photo are
// already inlined in the user message, and exposing both gave models
// an excuse to spend an iteration "confirming" what they already had.)
tools.push(Tool::function( tools.push(Tool::function(
"reverse_geocode", "reverse_geocode",
"Convert GPS latitude/longitude to a city/state place name via Nominatim. The photo's primary location is already pre-resolved on the user message — only call this for *other* coordinates (e.g. those returned by get_location_history). For the user's personal places (Home, Work, Cabin) prefer `get_personal_place_at`.", "Convert GPS latitude/longitude coordinates to a human-readable place name (city, state). Use this when GPS coordinates are available in the photo metadata, or to resolve coordinates returned by get_location_history.",
serde_json::json!({ serde_json::json!({
"type": "object", "type": "object",
"required": ["latitude", "longitude"], "required": ["latitude", "longitude"],
@@ -2928,7 +2642,7 @@ Return ONLY the summary, nothing else."#,
if apollo_enabled { if apollo_enabled {
tools.push(Tool::function( tools.push(Tool::function(
"get_personal_place_at", "get_personal_place_at",
"Get the user's personal, named place (e.g. Home, Work, Cabin) at a GPS coordinate, if any. Returns the place name, category, free-text description (the user's own notes about the location), and radius. More specific than reverse_geocode — prefer this when both apply. The cheap default is to call this once with the photo's GPS before any other location reasoning.", "Get the user's personal, named place (e.g. Home, Work, Cabin) at a GPS coordinate, if any. Returns the place name, category, free-text description (the user's own notes about the location), and radius. More specific than reverse_geocode — prefer this when both apply.",
serde_json::json!({ serde_json::json!({
"type": "object", "type": "object",
"required": ["latitude", "longitude"], "required": ["latitude", "longitude"],
@@ -2943,7 +2657,7 @@ Return ONLY the summary, nothing else."#,
// Knowledge memory tools // Knowledge memory tools
tools.push(Tool::function( tools.push(Tool::function(
"recall_entities", "recall_entities",
"Search the knowledge memory for people, places, events, or things previously learned from other photos. Provide at least one of `name` or `entity_type` — calling with neither dumps up to 50 entities ordered by id, which is rarely what you want.", "Search the knowledge memory for people, places, events, or things previously learned from other photos. Use this to retrieve context about subjects appearing in this photo.",
serde_json::json!({ serde_json::json!({
"type": "object", "type": "object",
"properties": { "properties": {
@@ -2954,7 +2668,7 @@ Return ONLY the summary, nothing else."#,
"entity_type": { "entity_type": {
"type": "string", "type": "string",
"enum": ["person", "place", "event", "thing"], "enum": ["person", "place", "event", "thing"],
"description": "Filter by entity type. Pass alone to enumerate everything of one kind (e.g. 'all known places')." "description": "Filter by entity type (optional)"
}, },
"limit": { "limit": {
"type": "integer", "type": "integer",
@@ -2979,80 +2693,9 @@ Return ONLY the summary, nothing else."#,
}), }),
)); ));
tools.push(Tool::function(
"recall_facts_for_entity",
"Retrieve all stored facts about one entity (person, place, event, thing) without needing a photo path. Use in chat when the user asks about a known subject (e.g. 'tell me more about Sarah') and you have the entity_id from recall_entities — or pass `name` to look it up by name.",
serde_json::json!({
"type": "object",
"properties": {
"entity_id": {
"type": "integer",
"description": "The entity's ID (preferred — exact match)."
},
"name": {
"type": "string",
"description": "Entity name (case-insensitive). Resolves to the highest-confidence active entity with that name. Provide this OR entity_id."
},
"entity_type": {
"type": "string",
"enum": ["person", "place", "event", "thing"],
"description": "Optional filter when looking up by name (e.g. disambiguate a person named 'Tahoe' from the place)."
}
}
}),
));
tools.push(Tool::function(
"find_photos_with_entity",
"List photos that have been linked to an entity in the knowledge memory. Use to answer 'when did I last see Sarah' / 'show me photos from the Tahoe trip'. Returns file paths and the role each entity played in the photo (subject / background / location).",
serde_json::json!({
"type": "object",
"properties": {
"entity_id": {
"type": "integer",
"description": "The entity's ID (preferred — exact match)."
},
"name": {
"type": "string",
"description": "Entity name (case-insensitive). Provide this OR entity_id."
},
"entity_type": {
"type": "string",
"enum": ["person", "place", "event", "thing"],
"description": "Optional filter when looking up by name."
},
"limit": {
"type": "integer",
"description": "Maximum number of photo paths to return (default: 20, max: 50)"
}
}
}),
));
tools.push(Tool::function(
"get_exif",
"Read the stored EXIF row for a photo file path. Returns camera make/model, lens, focal length, aperture, shutter speed, ISO, dimensions, and the date_taken source. Use to answer photography questions ('what camera was this on?') or to inspect technical metadata. The current photo's GPS, date, and date source are already on the user message — only call this when you need the additional fields.",
serde_json::json!({
"type": "object",
"required": ["file_path"],
"properties": {
"file_path": {
"type": "string",
"description": "The file path of the photo (defaults to the current photo if you pass its path)."
}
}
}),
));
// Knowledge-memory writes are gated by `disable_writes` — when set,
// exploration / chat continuation can run without polluting the
// persistent KB with one-off variants (e.g. caption-style prompts).
if !disable_writes {
tools.push(Tool::function( tools.push(Tool::function(
"store_entity", "store_entity",
"Store or update a person, place, event, or thing in the knowledge memory. \ "Store or update a person, place, event, or thing in the knowledge memory. Call this when you identify a subject in this photo that should be remembered for future insights.",
Returns the entity's ID. If similarly-named entities already exist, the \
response lists them — prefer using an existing ID over creating a duplicate.",
serde_json::json!({ serde_json::json!({
"type": "object", "type": "object",
"required": ["name", "entity_type"], "required": ["name", "entity_type"],
@@ -3076,12 +2719,7 @@ Return ONLY the summary, nothing else."#,
tools.push(Tool::function( tools.push(Tool::function(
"store_fact", "store_fact",
"Record a fact about an entity in the knowledge memory. Provide EITHER object_entity_id \ "Record a fact about an entity in the knowledge memory. Provide EITHER object_entity_id (when the object is a known entity whose ID you have) OR object_value (for free-text attributes). The fact will be linked to the current photo automatically.",
(when the object is a known entity whose ID you have) OR object_value (for free-text \
attributes). The fact is linked to the current photo automatically. Predicates use \
snake_case present-tense verbs/relations: is_friend_of, is_sibling_of, is_parent_of, \
located_in, works_at, attended_event, visited_in, owns. Symmetric relations (friendship, \
sibling) need only be stored once.",
serde_json::json!({ serde_json::json!({
"type": "object", "type": "object",
"required": ["subject_entity_id", "predicate"], "required": ["subject_entity_id", "predicate"],
@@ -3092,7 +2730,7 @@ Return ONLY the summary, nothing else."#,
}, },
"predicate": { "predicate": {
"type": "string", "type": "string",
"description": "snake_case verb/relation in present tense (e.g. 'is_friend_of', 'located_in', 'attended_event', 'is_sibling_of', 'works_at')" "description": "The relationship or attribute (e.g. 'is_friend_of', 'located_in', 'attended_event', 'is_sibling_of')"
}, },
"object_entity_id": { "object_entity_id": {
"type": "integer", "type": "integer",
@@ -3109,11 +2747,10 @@ Return ONLY the summary, nothing else."#,
} }
}), }),
)); ));
}
tools.push(Tool::function( tools.push(Tool::function(
"get_current_datetime", "get_current_datetime",
"Returns the current date and time. Use ONLY when you need to compute time-since-photo for phrases like 'two years ago' — the photo's date is already in the user message and re-deriving it is wasted budget.", "Get the current date and time. Useful for understanding how long ago the photo was taken.",
serde_json::json!({ serde_json::json!({
"type": "object", "type": "object",
"properties": {} "properties": {}
@@ -3320,7 +2957,6 @@ Return ONLY the summary, nothing else."#,
backend: Option<String>, backend: Option<String>,
fewshot_examples: Vec<Vec<ChatMessage>>, fewshot_examples: Vec<Vec<ChatMessage>>,
fewshot_source_ids: Vec<i32>, fewshot_source_ids: Vec<i32>,
disable_writes: bool,
) -> Result<(Option<i32>, Option<i32>)> { ) -> Result<(Option<i32>, Option<i32>)> {
let tracer = global_tracer(); let tracer = global_tracer();
let current_cx = opentelemetry::Context::current(); let current_cx = opentelemetry::Context::current();
@@ -3555,40 +3191,9 @@ Return ONLY the summary, nothing else."#,
.map(|dt| dt.date_naive()) .map(|dt| dt.date_naive())
.unwrap_or_else(|| Utc::now().date_naive()); .unwrap_or_else(|| Utc::now().date_naive());
// Date confidence comes from the canonical-date waterfall — one of
// exif/exiftool/filename/fs_time/manual. Surface in the user message
// so the model can hedge appropriately on `fs_time`-sourced dates.
let date_taken_source = exif
.as_ref()
.and_then(|e| e.date_taken_source.clone())
.unwrap_or_else(|| "unknown".to_string());
let contact = Self::extract_contact_from_path(&file_path); let contact = Self::extract_contact_from_path(&file_path);
log::info!("Agentic: date_taken={}, contact={:?}", date_taken, contact); log::info!("Agentic: date_taken={}, contact={:?}", date_taken, contact);
// Pre-resolve a human-readable location from GPS using the same
// Apollo + Nominatim composer the non-agentic flow uses. Saves the
// agent two iterations re-deriving a string we already have.
let resolved_location: Option<String> = match exif.as_ref() {
Some(e) => match (e.gps_latitude, e.gps_longitude) {
(Some(lat), Some(lon)) => {
let lat = lat as f64;
let lon = lon as f64;
let nominatim = self.reverse_geocode(lat, lon).await;
let apollo_primary = self
.apollo_client
.places_containing(lat, lon)
.await
.into_iter()
.next();
let combined = compose_location_string(apollo_primary, nominatim, lat, lon);
Some(combined)
}
_ => None,
},
None => None,
};
// 5. Fetch tags // 5. Fetch tags
let tag_names: Vec<String> = { let tag_names: Vec<String> = {
let mut dao = self.tag_dao.lock().expect("Unable to lock TagDao"); let mut dao = self.tag_dao.lock().expect("Unable to lock TagDao");
@@ -3697,139 +3302,79 @@ Return ONLY the summary, nothing else."#,
None => String::new(), None => String::new(),
}; };
let fewshot_block = Self::render_fewshot_examples(&fewshot_examples); let fewshot_block = Self::render_fewshot_examples(&fewshot_examples);
// The knowledge-write line gets rewritten when disable_writes is on
// so the model isn't told to call tools that aren't on the menu.
let knowledge_write_line = if disable_writes {
"Knowledge-memory writes are disabled for this run — do not attempt to call store_entity or store_fact (they are not available)."
} else {
"When you identify people, places, events, or notable things in this photo: use store_entity to record them and store_fact to record key facts (relationships, roles, attributes). This builds a persistent memory for future insights."
};
let base_system = format!( let base_system = format!(
"You are a personal photo memory assistant helping to reconstruct a memory from a photo.{owner_id_note}\n\n\ "You are a personal photo memory assistant helping to reconstruct a memory from a photo.{owner_id_note}\n\n\
{fewshot_block}\ {fewshot_block}\
## Tool-use guidance\n\ IMPORTANT INSTRUCTIONS:\n\
- You have up to {max_iterations} iterations available. Aim to use at least 5 of them on context-gathering tools before writing — only skip context-gathering when the photo is genuinely trivial (e.g. a screenshot of a receipt). Reserve your last 1–2 iterations for writing the final insight.\n\ 1. You MUST call multiple tools to gather context BEFORE writing any final insight. Do not produce a final answer after only one or two tool calls.\n\
- When you call get_sms_messages or search_rag and you know the contact, also make at least one call WITHOUT a contact filter so you can see what else was happening in {owner_name}'s life around this date.\n\ 2. When calling get_sms_messages and search_rag, always make at least one call WITHOUT a contact filter to capture what else was happening in {owner_name}'s life around this date — other conversations, events, and activities provide important wider context even when a specific contact is known.\n\
- Use recall_facts_for_photo and recall_entities early to load any prior knowledge about subjects in this photo.\n\ 3. Use recall_facts_for_photo to load any previously stored knowledge about subjects in this photo.\n\
- {knowledge_write_line}\n\ 4. Use recall_entities to look up known people, places, or things that appear in this photo.\n\
- If a tool returns no results, that's useful information — pivot to a different tool, don't retry the same call.", 5. When you identify people, places, events, or notable things in this photo: use store_entity to record them and store_fact to record key facts (relationships, roles, attributes). This builds a persistent memory for future insights.\n\
6. Only produce your final insight AFTER you have gathered context from at least 5 tool calls.\n\
7. If a tool returns no results, that is useful information — continue calling the remaining tools anyway.\n\
8. You have a hard budget of {max_iterations} tool-calling iterations before the loop ends. Plan your context gathering so you can write a complete final insight within that budget.",
owner_id_note = owner_id_note, owner_id_note = owner_id_note,
fewshot_block = fewshot_block, fewshot_block = fewshot_block,
owner_name = owner_name, owner_name = owner_name,
knowledge_write_line = knowledge_write_line,
max_iterations = max_iterations max_iterations = max_iterations
); );
// Custom prompts are *appended* under an explicit override heading so
// they actually beat the base instructions. Prepending was the wrong
// default — later instructions tend to win attention.
let system_content = if let Some(ref custom) = custom_system_prompt { let system_content = if let Some(ref custom) = custom_system_prompt {
format!( format!("{}\n\n{}", custom, base_system)
"{}\n\n## User overrides (these take precedence over the instructions above)\n\n{}",
base_system, custom
)
} else { } else {
base_system base_system.to_string()
}; };
// 9. Build user message // 9. Build user message
// The user message is restructured to lead with photo facts as a let gps_info = exif
// bulleted "## This photo" block (so small models can't skim past
// them), followed by an imperative "## What to do" recipe and a
// forcing line. Small models bail out of the agentic loop when the
// user message ends with "write a detailed insight" — they just
// write. The forcing line replaces the soft "aim to use 5 tools"
// floor with a hard "do not output text yet" gate.
// Date with weekday + canonical-date source so the model can hedge
// on filename- or fs_time-derived dates.
let date_bullet = format!(
"- Date: {} (source: {})",
date_taken.format("%A, %B %d, %Y"),
date_taken_source
);
// Location: full resolved string + raw coordinates when GPS is
// present, falling back to "unknown" when not.
let location_bullet = match (resolved_location.as_deref(), exif.as_ref()) {
(Some(name), Some(e)) if e.gps_latitude.is_some() && e.gps_longitude.is_some() => {
format!(
"- Location: {} (GPS {:.4}, {:.4})",
name,
e.gps_latitude.unwrap(),
e.gps_longitude.unwrap()
)
}
(None, Some(e)) if e.gps_latitude.is_some() && e.gps_longitude.is_some() => {
format!(
"- Location: GPS {:.4}, {:.4} (geocoder unavailable)",
e.gps_latitude.unwrap(),
e.gps_longitude.unwrap()
)
}
_ => "- Location: unknown".to_string(),
};
let contact_bullet = contact
.as_ref() .as_ref()
.map(|c| format!("- Contact/Person: {}", c)) .and_then(|e| {
.unwrap_or_else(|| "- Contact/Person: unknown".to_string()); if let (Some(lat), Some(lon)) = (e.gps_latitude, e.gps_longitude) {
Some(format!("GPS: {:.4}, {:.4}", lat, lon))
let tags_bullet = if tag_names.is_empty() {
"- Tags: none".to_string()
} else { } else {
format!("- Tags: {}", tag_names.join(", ")) None
}
})
.unwrap_or_else(|| "GPS: unknown".to_string());
let tags_info = if tag_names.is_empty() {
"Tags: none".to_string()
} else {
format!("Tags: {}", tag_names.join(", "))
}; };
let path_bullet = format!("- File path: {}", file_path); let contact_info = contact
.as_ref()
.map(|c| format!("Contact/Person: {}", c))
.unwrap_or_else(|| "Contact/Person: unknown".to_string());
// Hybrid: visual description is inlined as a bullet (no image bytes let visual_block = hybrid_visual_description
// reach the chat model). Local: the image is attached to this
// message, no inline description bullet — describe_photo is the tool.
let visual_bullet = hybrid_visual_description
.as_deref() .as_deref()
.map(|d| { .map(|d| format!("Visual description (from local vision model):\n{}\n\n", d))
format!(
"- Visual description (already generated — do not call describe_photo):\n {}",
d.lines().collect::<Vec<_>>().join("\n ")
)
})
.unwrap_or_default(); .unwrap_or_default();
// Compose the photo block (omit empty visual bullet to avoid stray newline).
let photo_block = if visual_bullet.is_empty() {
format!(
"## This photo\n\n{}\n{}\n{}\n{}\n{}",
path_bullet, date_bullet, contact_bullet, location_bullet, tags_bullet
)
} else {
format!(
"## This photo\n\n{}\n{}\n{}\n{}\n{}\n{}",
path_bullet,
date_bullet,
contact_bullet,
location_bullet,
tags_bullet,
visual_bullet
)
};
let user_content = format!( let user_content = format!(
"{photo_block}\n\n\ "{visual_block}Please analyze this photo and gather any relevant context from the surrounding weeks.\n\n\
## What to do\n\n\ Photo file path: {}\n\
1. First, call recall_facts_for_photo and recall_entities to load any prior knowledge about subjects in this photo.\n\ Date taken: {}\n\
2. Then call at least 3 of: search_rag, get_sms_messages (try once with the contact filter and once without), get_calendar_events, get_location_history — pick the ones most relevant to this photo's date and context.\n\ {}\n\
3. Only after you have tool results, write the final insight with a title and a detailed summary that references specific facts from the metadata above and from your tool results. Generic narration is not acceptable.\n\n\ {}\n\
YOUR FIRST RESPONSE MUST BE A TOOL CALL. Do not output any final answer text until you have called at least 5 tools." {}\n\n\
Use the available tools to gather more context about this moment (messages, calendar events, location history, etc.), \
then write a detailed insight with a title and summary.",
file_path,
date_taken.format("%B %d, %Y"),
contact_info,
gps_info,
tags_info,
visual_block = visual_block,
); );
// 10. Define tools. Hybrid mode omits `describe_photo` since the // 10. Define tools. Hybrid mode omits `describe_photo` since the
// chat model receives the visual description inline. // chat model receives the visual description inline.
let offer_describe_tool = has_vision && !is_hybrid; let offer_describe_tool = has_vision && !is_hybrid;
let tools = Self::build_tool_definitions( let tools =
offer_describe_tool, Self::build_tool_definitions(offer_describe_tool, self.apollo_client.is_enabled());
self.apollo_client.is_enabled(),
disable_writes,
);
// 11. Build initial messages. In hybrid mode images are never // 11. Build initial messages. In hybrid mode images are never
// attached to the wire message — the description is part of // attached to the wire message — the description is part of
@@ -3852,11 +3397,6 @@ Return ONLY the summary, nothing else."#,
let loop_span = tracer.start_with_context("ai.agentic.loop", &insight_cx); let loop_span = tracer.start_with_context("ai.agentic.loop", &insight_cx);
let loop_cx = insight_cx.with_span(loop_span); let loop_cx = insight_cx.with_span(loop_span);
// Memoize describe_photo for the lifetime of this loop. Vision
// describes are non-deterministic; without a cache, retries leave
// the chat history with conflicting descriptions of the same image.
let describe_cache: tokio::sync::Mutex<Option<String>> = tokio::sync::Mutex::new(None);
let mut final_content = String::new(); let mut final_content = String::new();
let mut iterations_used = 0usize; let mut iterations_used = 0usize;
let mut last_prompt_eval_count: Option<i32> = None; let mut last_prompt_eval_count: Option<i32> = None;
@@ -3910,7 +3450,6 @@ Return ONLY the summary, nothing else."#,
&image_base64, &image_base64,
&file_path, &file_path,
&loop_cx, &loop_cx,
Some(&describe_cache),
) )
.await; .await;
messages.push(ChatMessage::tool_result(result)); messages.push(ChatMessage::tool_result(result));
src/ai/sms_client.rs
+8 -12
View File
@@ -20,24 +20,22 @@ impl SmsApiClient {
} }
} }
/// Fetch messages for a specific contact within ±`days_radius` days of /// Fetch messages for a specific contact within ±4 days of the given timestamp
/// the given timestamp (defaults to ±4 days when `None`). Falls back to /// Falls back to all contacts if no messages found for the specific contact
/// all contacts if no messages are found for the specified contact. /// Messages are sorted by proximity to the center timestamp
/// Messages are sorted by proximity to the center timestamp.
pub async fn fetch_messages_for_contact( pub async fn fetch_messages_for_contact(
&self, &self,
contact: Option<&str>, contact: Option<&str>,
center_timestamp: i64, center_timestamp: i64,
days_radius: Option<i64>,
) -> Result<Vec<SmsMessage>> { ) -> Result<Vec<SmsMessage>> {
use chrono::Duration; use chrono::Duration;
let radius = days_radius.unwrap_or(4).clamp(1, 30); // Calculate ±4 days range around the center timestamp
let center_dt = chrono::DateTime::from_timestamp(center_timestamp, 0) let center_dt = chrono::DateTime::from_timestamp(center_timestamp, 0)
.ok_or_else(|| anyhow::anyhow!("Invalid timestamp"))?; .ok_or_else(|| anyhow::anyhow!("Invalid timestamp"))?;
let start_dt = center_dt - Duration::days(radius); let start_dt = center_dt - Duration::days(4);
let end_dt = center_dt + Duration::days(radius); let end_dt = center_dt + Duration::days(4);
let start_ts = start_dt.timestamp(); let start_ts = start_dt.timestamp();
let end_ts = end_dt.timestamp(); let end_ts = end_dt.timestamp();
@@ -45,9 +43,8 @@ impl SmsApiClient {
// If contact specified, try fetching for that contact first // If contact specified, try fetching for that contact first
if let Some(contact_name) = contact { if let Some(contact_name) = contact {
log::info!( log::info!(
"Fetching SMS for contact: {} (±{} days from {})", "Fetching SMS for contact: {} (±4 days from {})",
contact_name, contact_name,
radius,
center_dt.format("%Y-%m-%d %H:%M:%S") center_dt.format("%Y-%m-%d %H:%M:%S")
); );
let messages = self let messages = self
@@ -71,8 +68,7 @@ impl SmsApiClient {
// Fallback to all contacts // Fallback to all contacts
log::info!( log::info!(
"Fetching all SMS messages (±{} days from {})", "Fetching all SMS messages (±4 days from {})",
radius,
center_dt.format("%Y-%m-%d %H:%M:%S") center_dt.format("%Y-%m-%d %H:%M:%S")
); );
self.fetch_messages(start_ts, end_ts, None, Some(center_timestamp)) self.fetch_messages(start_ts, end_ts, None, Some(center_timestamp))
src/bin/populate_knowledge.rs
-1
View File
@@ -331,7 +331,6 @@ async fn main() -> anyhow::Result<()> {
None, None,
Vec::new(), Vec::new(),
Vec::new(), Vec::new(),
false, // disable_writes — keep KB writes on for the population job
) )
.await .await
{ {
src/files.rs
+1 -6
View File
@@ -1718,12 +1718,7 @@ mod tests {
// Mock — files.rs tests don't exercise the date-override endpoints. // Mock — files.rs tests don't exercise the date-override endpoints.
// Returning a synthetic row keeps the trait satisfied without // Returning a synthetic row keeps the trait satisfied without
// depending on private DbError constructors. // depending on private DbError constructors.
Ok(mock_exif_row( Ok(mock_exif_row(library_id, rel_path, Some(date_taken), Some("manual".to_string())))
library_id,
rel_path,
Some(date_taken),
Some("manual".to_string()),
))
} }
fn clear_manual_date_taken( fn clear_manual_date_taken(
src/main.rs
+6 -3
View File
@@ -995,8 +995,10 @@ async fn upload_image(
} }
}; };
let perceptual = perceptual_hash::compute(&uploaded_path); let perceptual = perceptual_hash::compute(&uploaded_path);
let resolved_date = let resolved_date = date_resolver::resolve_date_taken(
date_resolver::resolve_date_taken(&uploaded_path, exif_data.date_taken); &uploaded_path,
exif_data.date_taken,
);
let insert_exif = InsertImageExif { let insert_exif = InsertImageExif {
library_id: target_library.id, library_id: target_library.id,
file_path: relative_path.clone(), file_path: relative_path.clone(),
@@ -1020,7 +1022,8 @@ async fn upload_image(
size_bytes, size_bytes,
phash_64: perceptual.map(|h| h.phash_64), phash_64: perceptual.map(|h| h.phash_64),
dhash_64: perceptual.map(|h| h.dhash_64), dhash_64: perceptual.map(|h| h.dhash_64),
date_taken_source: resolved_date.map(|r| r.source.as_str().to_string()), date_taken_source: resolved_date
.map(|r| r.source.as_str().to_string()),
}; };
if let Ok(mut dao) = exif_dao.lock() { if let Ok(mut dao) = exif_dao.lock() {