feat(ai): few-shot exemplars + sticky Ollama preference
- Few-shot injection on /insights/generate/agentic: compresses prior training_messages into trajectory blocks (tool calls + result summaries) and injects into the system prompt. Hardcoded default ids with optional request override. - New fewshot_source_ids column on photo_insights (+ migration) to track which exemplars influenced a given row, for downstream training-set filtering. Chat amend rows stamp None with a lineage note. - Ollama client now remembers which server (primary/fallback) most recently succeeded and tries it first on the next call, via a shared Arc<AtomicBool>. Avoids re-404ing the primary on every agent iteration when the chosen model only lives on the fallback. - Demote noisy logs: daily_summary "Summary match" lines to debug; inner chat_with_tools non-2xx body log from error to warn (outer layer owns the terminal-error signal). - Drift-guard tests for summarize_tool_result covering the success / empty / error / unknown shape for every tool. - Tidy: three pre-existing clippy warnings cleaned up. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Cameron
@@ -1227,6 +1227,7 @@ impl InsightGenerator {
|
||||
is_current: true,
|
||||
training_messages: None,
|
||||
backend: "local".to_string(),
|
||||
fewshot_source_ids: None,
|
||||
};
|
||||
|
||||
let mut dao = self.insight_dao.lock().expect("Unable to lock InsightDao");
|
||||
@@ -2634,6 +2635,159 @@ Return ONLY the summary, nothing else."#,
|
||||
/// text, and runs the loop through OpenRouter (chat only — embeddings
|
||||
/// and describe calls stay local in either mode).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
/// Render a set of prior-conversation transcripts into a compact
|
||||
/// trajectory block for inclusion in the system prompt. Tool results
|
||||
/// are summarised to one line each so the prompt stays small.
|
||||
fn render_fewshot_examples(examples: &[Vec<ChatMessage>]) -> String {
|
||||
if examples.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
let mut out = String::from("## Examples of strong context-gathering\n\n");
|
||||
out.push_str(
|
||||
"The following are compressed trajectories from prior high-quality insights. \
|
||||
They show the *pattern* of tool use, not answers to copy.\n\n",
|
||||
);
|
||||
|
||||
for (i, msgs) in examples.iter().enumerate() {
|
||||
out.push_str(&format!("### Example {}\n\n", i + 1));
|
||||
out.push_str(&Self::render_single_trajectory(msgs));
|
||||
out.push('\n');
|
||||
}
|
||||
|
||||
out.push_str("---\n\n");
|
||||
out
|
||||
}
|
||||
|
||||
fn render_single_trajectory(msgs: &[ChatMessage]) -> String {
|
||||
let mut out = String::new();
|
||||
|
||||
if let Some(first_user) = msgs.iter().find(|m| m.role == "user") {
|
||||
let trimmed = first_user
|
||||
.content
|
||||
.lines()
|
||||
.filter(|l| !l.trim().is_empty())
|
||||
.take(8)
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
out.push_str(&format!("Input:\n{}\n\n", trimmed));
|
||||
}
|
||||
|
||||
out.push_str("Trajectory:\n");
|
||||
let mut step = 1;
|
||||
let mut final_content: Option<String> = None;
|
||||
|
||||
for (i, m) in msgs.iter().enumerate() {
|
||||
if m.role != "assistant" {
|
||||
continue;
|
||||
}
|
||||
if let Some(ref calls) = m.tool_calls {
|
||||
for call in calls {
|
||||
let args_brief = Self::brief_json_args(&call.function.arguments);
|
||||
let result_summary = msgs
|
||||
.get(i + 1)
|
||||
.filter(|r| r.role == "tool")
|
||||
.map(|r| Self::summarize_tool_result(&call.function.name, &r.content))
|
||||
.unwrap_or_else(|| "(no result)".to_string());
|
||||
out.push_str(&format!(
|
||||
"{}. {}({}) -> {}\n",
|
||||
step, call.function.name, args_brief, result_summary
|
||||
));
|
||||
step += 1;
|
||||
}
|
||||
} else if !m.content.is_empty() {
|
||||
final_content = Some(m.content.clone());
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(content) = final_content {
|
||||
let short: String = content.chars().take(240).collect();
|
||||
out.push_str(&format!("\nFinal insight: {}...\n", short));
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
fn brief_json_args(v: &serde_json::Value) -> String {
|
||||
let Some(obj) = v.as_object() else {
|
||||
return v.to_string();
|
||||
};
|
||||
obj.iter()
|
||||
.map(|(k, v)| {
|
||||
let rendered = match v {
|
||||
serde_json::Value::String(s) if s.len() > 40 => {
|
||||
format!("\"{}...\"", &s[..40])
|
||||
}
|
||||
_ => v.to_string(),
|
||||
};
|
||||
format!("{}={}", k, rendered)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
}
|
||||
|
||||
/// Collapse a raw tool-result string (the text the model saw) into a
|
||||
/// short phrase suitable for a few-shot trajectory. Detects the
|
||||
/// "Found N ...", "No ...", and "Error ..." idioms used by the tool
|
||||
/// implementations in this file. Unknown shapes fall back to a char
|
||||
/// count, which is deliberately visible so drift shows up in output.
|
||||
fn summarize_tool_result(tool_name: &str, raw: &str) -> String {
|
||||
if raw.starts_with("Error ") {
|
||||
return "error".to_string();
|
||||
}
|
||||
if raw.starts_with("No ") || raw.starts_with("Could not ") {
|
||||
return "empty (pivoted)".to_string();
|
||||
}
|
||||
|
||||
if let Some(rest) = raw.strip_prefix("Found ")
|
||||
&& let Some(n_str) = rest.split_whitespace().next()
|
||||
&& let Ok(n) = n_str.parse::<usize>()
|
||||
{
|
||||
let kind = match tool_name {
|
||||
"search_messages" | "get_sms_messages" => "messages",
|
||||
"get_calendar_events" => "events",
|
||||
"get_location_history" => "location records",
|
||||
_ => "results",
|
||||
};
|
||||
return format!("{} {}", n, kind);
|
||||
}
|
||||
|
||||
match tool_name {
|
||||
"search_rag" => {
|
||||
let n = raw.split("\n\n").filter(|s| !s.trim().is_empty()).count();
|
||||
format!("{} rag hits", n)
|
||||
}
|
||||
"get_file_tags" => {
|
||||
let n = raw.split(',').filter(|s| !s.trim().is_empty()).count();
|
||||
format!("{} tags", n)
|
||||
}
|
||||
"describe_photo" => {
|
||||
let short: String = raw.chars().take(80).collect();
|
||||
format!("described: \"{}...\"", short)
|
||||
}
|
||||
"reverse_geocode" => {
|
||||
let short: String = raw.chars().take(60).collect();
|
||||
format!("place: {}", short)
|
||||
}
|
||||
"recall_entities" | "recall_facts_for_photo" => {
|
||||
let n = raw.lines().skip(1).filter(|l| !l.trim().is_empty()).count();
|
||||
let kind = if tool_name == "recall_entities" {
|
||||
"entities"
|
||||
} else {
|
||||
"facts"
|
||||
};
|
||||
format!("{} {}", n, kind)
|
||||
}
|
||||
"store_entity" | "store_fact" => raw
|
||||
.split_whitespace()
|
||||
.find_map(|tok| tok.strip_prefix("ID:"))
|
||||
.map(|id| format!("stored id={}", id.trim_end_matches(',')))
|
||||
.unwrap_or_else(|| "stored".to_string()),
|
||||
"get_current_datetime" => "time noted".to_string(),
|
||||
_ => format!("{} chars", raw.len()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn generate_agentic_insight_for_photo(
|
||||
&self,
|
||||
file_path: &str,
|
||||
@@ -2646,6 +2800,8 @@ Return ONLY the summary, nothing else."#,
|
||||
min_p: Option<f32>,
|
||||
max_iterations: usize,
|
||||
backend: Option<String>,
|
||||
fewshot_examples: Vec<Vec<ChatMessage>>,
|
||||
fewshot_source_ids: Vec<i32>,
|
||||
) -> Result<(Option<i32>, Option<i32>)> {
|
||||
let tracer = global_tracer();
|
||||
let current_cx = opentelemetry::Context::current();
|
||||
@@ -2990,8 +3146,10 @@ Return ONLY the summary, nothing else."#,
|
||||
),
|
||||
None => String::new(),
|
||||
};
|
||||
let fewshot_block = Self::render_fewshot_examples(&fewshot_examples);
|
||||
let base_system = format!(
|
||||
"You are a personal photo memory assistant helping to reconstruct a memory from a photo.{owner_id_note}\n\n\
|
||||
{fewshot_block}\
|
||||
IMPORTANT INSTRUCTIONS:\n\
|
||||
1. You MUST call multiple tools to gather context BEFORE writing any final insight. Do not produce a final answer after only one or two tool calls.\n\
|
||||
2. When calling get_sms_messages and search_rag, always make at least one call WITHOUT a contact filter to capture what else was happening in {owner_name}'s life around this date — other conversations, events, and activities provide important wider context even when a specific contact is known.\n\
|
||||
@@ -3002,6 +3160,7 @@ Return ONLY the summary, nothing else."#,
|
||||
7. If a tool returns no results, that is useful information — continue calling the remaining tools anyway.\n\
|
||||
8. You have a hard budget of {max_iterations} tool-calling iterations before the loop ends. Plan your context gathering so you can write a complete final insight within that budget.",
|
||||
owner_id_note = owner_id_note,
|
||||
fewshot_block = fewshot_block,
|
||||
owner_name = owner_name,
|
||||
max_iterations = max_iterations
|
||||
);
|
||||
@@ -3153,12 +3312,10 @@ Return ONLY the summary, nothing else."#,
|
||||
"Agentic loop exhausted after {} iterations, requesting final answer",
|
||||
iterations_used
|
||||
);
|
||||
messages.push(ChatMessage::user(
|
||||
&format!(
|
||||
"Based on the context gathered, please write the final photo insight: a title and a detailed personal summary. Write in first person as {}.",
|
||||
user_display_name()
|
||||
),
|
||||
));
|
||||
messages.push(ChatMessage::user(format!(
|
||||
"Based on the context gathered, please write the final photo insight: a title and a detailed personal summary. Write in first person as {}.",
|
||||
user_display_name()
|
||||
)));
|
||||
let (final_response, prompt_tokens, eval_tokens) = chat_backend
|
||||
.chat_with_tools(messages.clone(), vec![])
|
||||
.await?;
|
||||
@@ -3204,6 +3361,11 @@ Return ONLY the summary, nothing else."#,
|
||||
|
||||
// 15. Store insight (returns the persisted row including its new id)
|
||||
let model_version = chat_backend.primary_model().to_string();
|
||||
let fewshot_source_ids_json = if fewshot_source_ids.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(serde_json::to_string(&fewshot_source_ids).unwrap_or_else(|_| "[]".to_string()))
|
||||
};
|
||||
let insight = InsertPhotoInsight {
|
||||
library_id: crate::libraries::PRIMARY_LIBRARY_ID,
|
||||
file_path: file_path.to_string(),
|
||||
@@ -3214,6 +3376,7 @@ Return ONLY the summary, nothing else."#,
|
||||
is_current: true,
|
||||
training_messages,
|
||||
backend: backend_label.clone(),
|
||||
fewshot_source_ids: fewshot_source_ids_json,
|
||||
};
|
||||
|
||||
let stored = {
|
||||
@@ -3333,6 +3496,7 @@ Return ONLY the summary, nothing else."#,
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::ai::ollama::{ToolCall, ToolCallFunction};
|
||||
|
||||
#[test]
|
||||
fn combine_contexts_includes_tags_section_when_tags_present() {
|
||||
@@ -3374,4 +3538,219 @@ mod tests {
|
||||
let result = InsightGenerator::combine_contexts(None, None, None, None, None);
|
||||
assert_eq!(result, "No additional context available");
|
||||
}
|
||||
|
||||
// These tests assert the shape of the strings returned by the tool
|
||||
// implementations above. If a tool's output format changes, update the
|
||||
// tool AND the corresponding arm of `summarize_tool_result` — these
|
||||
// tests exist to make that coupling loud.
|
||||
|
||||
#[test]
|
||||
fn summarize_errors_uniformly() {
|
||||
assert_eq!(
|
||||
InsightGenerator::summarize_tool_result("search_rag", "Error searching RAG: boom"),
|
||||
"error"
|
||||
);
|
||||
assert_eq!(
|
||||
InsightGenerator::summarize_tool_result(
|
||||
"get_sms_messages",
|
||||
"Error fetching SMS messages: timeout"
|
||||
),
|
||||
"error"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summarize_empty_results_uniformly() {
|
||||
assert_eq!(
|
||||
InsightGenerator::summarize_tool_result("search_rag", "No relevant messages found."),
|
||||
"empty (pivoted)"
|
||||
);
|
||||
assert_eq!(
|
||||
InsightGenerator::summarize_tool_result("get_sms_messages", "No messages found."),
|
||||
"empty (pivoted)"
|
||||
);
|
||||
assert_eq!(
|
||||
InsightGenerator::summarize_tool_result(
|
||||
"reverse_geocode",
|
||||
"Could not resolve coordinates to a place name."
|
||||
),
|
||||
"empty (pivoted)"
|
||||
);
|
||||
assert_eq!(
|
||||
InsightGenerator::summarize_tool_result(
|
||||
"recall_facts_for_photo",
|
||||
"No knowledge facts found for this photo."
|
||||
),
|
||||
"empty (pivoted)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summarize_found_count_per_tool() {
|
||||
assert_eq!(
|
||||
InsightGenerator::summarize_tool_result(
|
||||
"get_sms_messages",
|
||||
"Found 7 messages:\n[2023-08-15 10:00] Sarah: hi"
|
||||
),
|
||||
"7 messages"
|
||||
);
|
||||
assert_eq!(
|
||||
InsightGenerator::summarize_tool_result(
|
||||
"search_messages",
|
||||
"Found 3 messages (mode: hybrid):\n\n[2023-08-15] Sarah — hi"
|
||||
),
|
||||
"3 messages"
|
||||
);
|
||||
assert_eq!(
|
||||
InsightGenerator::summarize_tool_result(
|
||||
"get_calendar_events",
|
||||
"Found 2 calendar events:\n[2023-08-15 10:00] Wedding"
|
||||
),
|
||||
"2 events"
|
||||
);
|
||||
assert_eq!(
|
||||
InsightGenerator::summarize_tool_result(
|
||||
"get_location_history",
|
||||
"Found 5 location records:\n[2023-08-15 10:00] 39.0, -120.0"
|
||||
),
|
||||
"5 location records"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summarize_search_rag_counts_hits() {
|
||||
let raw = "[2023-08-15] Sarah: venue confirmed\n\n[2023-08-14] Mom: travel plans\n\n[2023-08-13] Dad: weather";
|
||||
assert_eq!(
|
||||
InsightGenerator::summarize_tool_result("search_rag", raw),
|
||||
"3 rag hits"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summarize_get_file_tags() {
|
||||
assert_eq!(
|
||||
InsightGenerator::summarize_tool_result("get_file_tags", "wedding, tahoe, 2023"),
|
||||
"3 tags"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summarize_describe_photo_truncates() {
|
||||
let raw = "A wedding ceremony at Lake Tahoe with about 40 guests seated in rows facing a lakeside arch decorated with white flowers.";
|
||||
let out = InsightGenerator::summarize_tool_result("describe_photo", raw);
|
||||
assert!(out.starts_with("described: \""));
|
||||
assert!(out.contains("A wedding ceremony at Lake Tahoe"));
|
||||
assert!(out.ends_with("...\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summarize_reverse_geocode_returns_place() {
|
||||
let out =
|
||||
InsightGenerator::summarize_tool_result("reverse_geocode", "South Lake Tahoe, CA, USA");
|
||||
assert_eq!(out, "place: South Lake Tahoe, CA, USA");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summarize_recall_entities_counts_lines() {
|
||||
let raw = "Known entities:\n- Sarah (person)\n- Tahoe (place)\n- Wedding 2023 (event)";
|
||||
assert_eq!(
|
||||
InsightGenerator::summarize_tool_result("recall_entities", raw),
|
||||
"3 entities"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summarize_recall_facts_counts_lines() {
|
||||
let raw = "Knowledge for this photo:\n- Sarah: college friend\n- Tahoe: vacation spot";
|
||||
assert_eq!(
|
||||
InsightGenerator::summarize_tool_result("recall_facts_for_photo", raw),
|
||||
"2 facts"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summarize_store_entity_extracts_id() {
|
||||
assert_eq!(
|
||||
InsightGenerator::summarize_tool_result(
|
||||
"store_entity",
|
||||
"Entity stored: ID:42 | person | Sarah | confidence:0.80"
|
||||
),
|
||||
"stored id=42"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summarize_store_fact_extracts_id() {
|
||||
assert_eq!(
|
||||
InsightGenerator::summarize_tool_result(
|
||||
"store_fact",
|
||||
"Stored new fact: ID:17 | confidence:0.60"
|
||||
),
|
||||
"stored id=17"
|
||||
);
|
||||
assert_eq!(
|
||||
InsightGenerator::summarize_tool_result(
|
||||
"store_fact",
|
||||
"Corroborated existing fact: ID:17 | confidence:0.85"
|
||||
),
|
||||
"stored id=17"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summarize_current_datetime() {
|
||||
assert_eq!(
|
||||
InsightGenerator::summarize_tool_result(
|
||||
"get_current_datetime",
|
||||
"Current date/time: 2024-01-15 12:00:00 PST (Monday)"
|
||||
),
|
||||
"time noted"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summarize_unknown_tool_falls_back_to_char_count() {
|
||||
let out = InsightGenerator::summarize_tool_result("never_heard_of_it", "some output");
|
||||
assert_eq!(out, "11 chars");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_fewshot_empty_returns_empty_string() {
|
||||
assert!(InsightGenerator::render_fewshot_examples(&[]).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_single_trajectory_walks_tool_calls_in_order() {
|
||||
let arguments = serde_json::json!({ "query": "wedding", "date": "2023-08-15" });
|
||||
let msgs = vec![
|
||||
ChatMessage::system("ignored"),
|
||||
ChatMessage::user("Photo file path: /photos/img.jpg\nDate taken: August 15, 2023"),
|
||||
ChatMessage {
|
||||
role: "assistant".to_string(),
|
||||
content: String::new(),
|
||||
tool_calls: Some(vec![ToolCall {
|
||||
function: ToolCallFunction {
|
||||
name: "search_rag".to_string(),
|
||||
arguments,
|
||||
},
|
||||
id: None,
|
||||
}]),
|
||||
images: None,
|
||||
},
|
||||
ChatMessage::tool_result("No relevant messages found."),
|
||||
ChatMessage {
|
||||
role: "assistant".to_string(),
|
||||
content: "Final title\n\nFinal body.".to_string(),
|
||||
tool_calls: None,
|
||||
images: None,
|
||||
},
|
||||
];
|
||||
let out = InsightGenerator::render_single_trajectory(&msgs);
|
||||
assert!(out.contains("Input:"));
|
||||
assert!(out.contains("/photos/img.jpg"));
|
||||
assert!(out.contains("1. search_rag("));
|
||||
assert!(out.contains("query=\"wedding\""));
|
||||
assert!(out.contains("-> empty (pivoted)"));
|
||||
assert!(out.contains("Final insight: Final title"));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user