insights: push sms search filters server-side, render snippets, expand fts5 docs

- Refactor search_messages_with_contact -> search_messages(query, &SmsSearchParams)
  exposing date_from / date_to / offset / is_mms / has_media; drop the over-fetch
  + client-side date post-filter that could silently drop in-window hits past
  position 100.
- Surface SMS-API's <mark>-wrapped snippet for MMS messages that only matched
  via message_parts_fts (attachment text / filename) - pre-snippet, those
  rendered as a blank body preview to the LLM.
- Expose is_mms / has_media on the search_messages tool schema; expand the
  FTS5 syntax docs with worked examples for phrase / prefix / boolean / NEAR
  / grouping so the model picks the right operator.
- Unit tests for format_search_hits (body fallback, snippet preferred, MMS
  attachment-only regression, empty-snippet fallback) and strip_mark_tags.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

src/ai/insight_generator.rs

@@ -13,7 +13,7 @@ use crate::ai::apollo_client::{ApolloClient, ApolloPlace};
 use crate::ai::llm_client::LlmClient;
 use crate::ai::ollama::{ChatMessage, OllamaClient, Tool};
 use crate::ai::openrouter::OpenRouterClient;
-use crate::ai::sms_client::SmsApiClient;
+use crate::ai::sms_client::{SmsApiClient, SmsSearchHit, SmsSearchParams};
 use crate::ai::user_display_name;
 use crate::database::models::InsertPhotoInsight;
 use crate::database::{
@@ -212,10 +212,7 @@ impl InsightGenerator {
         };
         let allow_agent_corrections = persona
             .and_then(|(uid, pid)| {
-                let mut pdao = self
+                let mut pdao = self.persona_dao.lock().expect("Unable to lock PersonaDao");
-                    .persona_dao
-                    .lock()
-                    .expect("Unable to lock PersonaDao");
                 pdao.get_persona(&cx, uid, pid).ok().flatten()
             })
             .map(|p| p.allow_agent_corrections)
@@ -1861,7 +1858,11 @@ Return ONLY the summary, nothing else."#,
     /// Tool: search_messages — keyword / semantic / hybrid search over all
     /// SMS message bodies via the Django FTS5 + embeddings pipeline. Now
     /// supports optional `contact_id`, `start_ts`, `end_ts` filters.
-    async fn tool_search_messages(&self, args: &serde_json::Value, cx: &opentelemetry::Context) -> String {
+    async fn tool_search_messages(
+        &self,
+        args: &serde_json::Value,
+        cx: &opentelemetry::Context,
+    ) -> String {
         let query = match args.get("query").and_then(|v| v.as_str()) {
             Some(q) if !q.trim().is_empty() => q.trim(),
             _ => {
@@ -1881,8 +1882,7 @@ Return ONLY the summary, nothing else."#,
                     .map(|s| !s.trim().is_empty())
                     .unwrap_or(false);
                 let has_numeric_contact = args.get("contact_id").is_some();
-                let has_ts_window =
+                let has_ts_window = args.get("start_ts").is_some() || args.get("end_ts").is_some();
-                    args.get("start_ts").is_some() || args.get("end_ts").is_some();
                 if has_date_str && !has_numeric_contact && !has_ts_window {
                     log::info!(
                         "search_messages with `date` and no `query` — routing to get_sms_messages"
@@ -1895,8 +1895,7 @@ Return ONLY the summary, nothing else."#,
                         routed
                     );
                 }
-                let has_contact_name =
+                let has_contact_name = args.get("contact").and_then(|v| v.as_str()).is_some();
-                    args.get("contact").and_then(|v| v.as_str()).is_some();
                 if has_ts_window || has_numeric_contact || has_contact_name {
                     return "Error: search_messages needs a 'query' (keywords/phrase). \
                             To fetch messages around a date or from a contact without keywords, \
@@ -1928,70 +1927,62 @@ Return ONLY the summary, nothing else."#,
         let contact_id = args.get("contact_id").and_then(|v| v.as_i64());
         let start_ts = args.get("start_ts").and_then(|v| v.as_i64());
         let end_ts = args.get("end_ts").and_then(|v| v.as_i64());
+        let is_mms = args.get("is_mms").and_then(|v| v.as_bool());
+        let has_media = args.get("has_media").and_then(|v| v.as_bool());
         let has_date_filter = start_ts.is_some() || end_ts.is_some();
 
-        // When a date filter is supplied, fetch a larger pool from SMS-API
-        // so in-window matches that ranked lower than out-of-window ones
-        // aren't lost.
-        let fetch_limit = if has_date_filter { 100 } else { user_limit };
-
         log::info!(
-            "tool_search_messages: query='{}', mode={}, contact_id={:?}, range=[{:?}, {:?}], user_limit={}, fetch_limit={}",
+            "tool_search_messages: query='{}', mode={}, contact_id={:?}, range=[{:?}, {:?}], is_mms={:?}, has_media={:?}, limit={}",
             query,
             mode,
             contact_id,
             start_ts,
             end_ts,
-            user_limit,
+            is_mms,
-            fetch_limit
+            has_media,
+            user_limit
         );
 
-        let hits = match self
+        // SMS-API applies all of date/contact/mms/media filtering and
-            .sms_client
+        // pagination server-side, so the response is already exactly the
-            .search_messages_with_contact(query, &mode, fetch_limit, contact_id)
+        // page we want — no over-fetch, no client-side post-filter.
-            .await
+        let params = SmsSearchParams {
-        {
+            mode: mode.as_str(),
+            limit: user_limit,
+            contact_id,
+            date_from: start_ts,
+            date_to: end_ts,
+            is_mms,
+            has_media,
+            offset: None,
+        };
+
+        let hits = match self.sms_client.search_messages(query, &params).await {
             Ok(h) => h,
             Err(e) => return format!("Error searching messages: {}", e),
         };
 
-        // Date-range post-filter on the client side. SMS-API's /search/
+        if hits.is_empty() {
-        // doesn't accept date params; mirroring Apollo's pattern here.
-        let filtered: Vec<_> = hits
-            .into_iter()
-            .filter(|h| {
-                if let Some(s) = start_ts
-                    && h.date < s
-                {
-                    return false;
-                }
-                if let Some(e) = end_ts
-                    && h.date > e
-                {
-                    return false;
-                }
-                true
-            })
-            .take(user_limit)
-            .collect();
-
-        if filtered.is_empty() {
             return "No messages matched.".to_string();
         }
 
+        Self::format_search_hits(&hits, &mode, has_date_filter)
+    }
+
+    /// Render a list of [`SmsSearchHit`] for the LLM. Prefers the SMS-API
+    /// snippet (which already excerpts the matched span and is the only
+    /// preview MMS-attachment-only matches have) over the full body, and
+    /// strips the `<mark>` tags the snippet ships with.
+    fn format_search_hits(hits: &[SmsSearchHit], mode: &str, date_filtered: bool) -> String {
         let user_name = user_display_name();
         let mut out = String::new();
         out.push_str(&format!(
             "Found {} messages (mode: {}{}):\n\n",
-            filtered.len(),
+            hits.len(),
             mode,
-            if has_date_filter {
+            if date_filtered { ", date-filtered" } else { "" }
-                ", date-filtered"
-            } else {
-                ""
-            }
         ));
-        for h in filtered {
+        for h in hits {
             let date = chrono::DateTime::from_timestamp(h.date, 0)
                 .map(|dt| dt.format("%Y-%m-%d").to_string())
                 .unwrap_or_else(|| h.date.to_string());
@@ -2004,14 +1995,25 @@ Return ONLY the summary, nothing else."#,
                 .similarity_score
                 .map(|s| format!(" [score {:.2}]", s))
                 .unwrap_or_default();
+            let preview = match h.snippet.as_deref() {
+                Some(s) if !s.is_empty() => Self::strip_mark_tags(s),
+                _ => h.body.clone(),
+            };
             out.push_str(&format!(
                 "[{}]{} {} — {}\n\n",
-                date, score, direction, h.body
+                date, score, direction, preview
             ));
         }
         out
     }
 
+    /// Strip the `<mark>` / `</mark>` highlight tags that SMS-API wraps
+    /// matched terms in. The tags carry no signal beyond what the query
+    /// already conveys to the LLM, and leaving them in adds prompt noise.
+    fn strip_mark_tags(s: &str) -> String {
+        s.replace("<mark>", "").replace("</mark>", "")
+    }
+
     /// Tool: get_sms_messages — fetch SMS messages near a date for a contact
     async fn tool_get_sms_messages(
         &self,
@@ -2474,11 +2476,7 @@ Return ONLY the summary, nothing else."#,
             .lock()
             .expect("Unable to lock KnowledgeDao");
         match kdao.list_entities(cx, filter) {
-            Ok((entities, _total))
+            Ok((entities, _total)) if entities.iter().all(|e| e.status == "rejected") => {
-                if entities
-                    .iter()
-                    .all(|e| e.status == "rejected") =>
-            {
                 "No known entities found matching the query.".to_string()
             }
             Ok((entities, _total)) => {
@@ -2527,10 +2525,7 @@ Return ONLY the summary, nothing else."#,
         // enforces existence on writes), fall back to the permissive
         // default of active+reviewed.
         let reviewed_only = {
-            let mut pdao = self
+            let mut pdao = self.persona_dao.lock().expect("Unable to lock PersonaDao");
-                .persona_dao
-                .lock()
-                .expect("Unable to lock PersonaDao");
             pdao.get_persona(cx, user_id, persona_id)
                 .ok()
                 .flatten()
@@ -2884,16 +2879,9 @@ Return ONLY the summary, nothing else."#,
         // omitted → None → leave alone, value → Some(Some(value)) →
         // set. The match-on-presence pattern below mirrors the HTTP
         // PATCH path's serde-helper behaviour.
-        let parse_optional_i64 =
+        let parse_optional_i64 = |v: Option<&serde_json::Value>| -> Option<Option<i64>> {
-            |v: Option<&serde_json::Value>| -> Option<Option<i64>> {
+            v.map(|val| if val.is_null() { None } else { val.as_i64() })
-                v.map(|val| {
+        };
-                    if val.is_null() {
-                        None
-                    } else {
-                        val.as_i64()
-                    }
-                })
-            };
 
         let patch = FactPatch {
             predicate: args
@@ -3038,23 +3026,39 @@ Return ONLY the summary, nothing else."#,
 
         tools.push(Tool::function(
             "search_messages",
-            "Search SMS/MMS message bodies. Modes: `fts5` (keyword + phrase + prefix + AND/OR/NOT + NEAR proximity), \
+            "Search SMS/MMS messages — bodies and (for MMS) attachment text + filenames. \
+             Modes: `fts5` (keyword + phrase + prefix + AND/OR/NOT + NEAR proximity), \
              `semantic` (embedding similarity, requires generated embeddings), `hybrid` (RRF merge, recommended; \
-             degrades to fts5 when embeddings absent). Optional `start_ts` / `end_ts` (real-UTC unix seconds) and \
+             degrades to fts5 when embeddings absent). Optional filters: `start_ts` / `end_ts` (real-UTC unix \
-             `contact_id` filters. For pure date / contact browsing without keywords, prefer `get_sms_messages`. \
+             seconds), `contact_id`, `is_mms` (true = MMS only, false = SMS only), `has_media` (true = messages \
-             Examples: `{query: \"trader joe's\"}` — phrase across all time. \
+             with image/video/audio attachments only). For pure date / contact browsing without keywords, prefer \
-             `{query: \"dinner\", contact_id: 42, start_ts: 1700000000, end_ts: 1700604800}` — keyword within a contact and a week. \
+             `get_sms_messages`. \
-             `{query: \"NEAR(meeting work, 5)\"}` — proximity search.",
+             \n\nFTS5 query syntax (works in fts5 + hybrid modes):\n\
+             - Phrase:   `\"trader joe's\"` — exact word sequence (use double quotes).\n\
+             - Prefix:   `restaur*` — matches restaurant, restaurants, restauranteur, ….\n\
+             - Boolean:  `dinner AND tahoe`, `wedding OR reception OR ceremony`, `vacation NOT work` (operators must be UPPERCASE).\n\
+             - Proximity: `NEAR(meeting work, 5)` — both terms within 5 tokens of each other.\n\
+             - Combine:  `(reception OR ceremony) AND tahoe*` — group with parens.\n\
+             Unquoted multi-word queries are treated as implicit AND. Apostrophes / hyphens / colons are safe — they no longer downgrade to a slow LIKE scan. Use `mode: \"fts5\"` when you want the operators above to be authoritative; `hybrid` still respects them but may surface semantically-similar non-keyword hits alongside.\n\n\
+             Examples:\n\
+             - `{query: \"trader joe's\"}` — phrase across all time.\n\
+             - `{query: \"dinner\", contact_id: 42, start_ts: 1700000000, end_ts: 1700604800}` — keyword within a contact and a week.\n\
+             - `{query: \"vacation\", has_media: true}` — only matches that include photos / videos.\n\
+             - `{query: \"wedding OR reception OR ceremony\", mode: \"fts5\"}` — any of several synonyms.\n\
+             - `{query: \"restaur*\", mode: \"fts5\"}` — prefix expansion for varying word forms.\n\
+             - `{query: \"NEAR(birthday cake, 5)\", mode: \"fts5\"}` — terms close together but in any order.",
             serde_json::json!({
                 "type": "object",
                 "required": ["query"],
                 "properties": {
-                    "query": { "type": "string", "description": "Search query. Min 3 chars. fts5 supports phrase (\"\"), prefix (*), AND/OR/NOT, and NEAR proximity." },
+                    "query": { "type": "string", "description": "Search query. Min 3 chars. fts5 supports phrase (\"\"), prefix (*), AND/OR/NOT, and NEAR proximity. Matches both message body and MMS attachment text/filename." },
                     "mode":  { "type": "string", "enum": ["fts5", "semantic", "hybrid"], "description": "Search strategy. Default: hybrid." },
                     "limit": { "type": "integer", "description": "Max results (default 20, max 50)." },
                     "contact_id": { "type": "integer", "description": "Optional numeric contact id to scope the search." },
                     "start_ts": { "type": "integer", "description": "Optional inclusive lower bound, real-UTC unix seconds." },
-                    "end_ts":   { "type": "integer", "description": "Optional inclusive upper bound, real-UTC unix seconds." }
+                    "end_ts":   { "type": "integer", "description": "Optional inclusive upper bound, real-UTC unix seconds." },
+                    "is_mms":   { "type": "boolean", "description": "Optional: true to restrict to MMS, false to restrict to SMS." },
+                    "has_media":{ "type": "boolean", "description": "Optional: true to restrict to messages with image / video / audio attachments." }
                 }
             }),
         ));
@@ -4469,6 +4473,92 @@ mod tests {
         );
     }
 
+    fn make_search_hit(
+        id: i64,
+        contact: &str,
+        body: &str,
+        snippet: Option<&str>,
+        type_: i32,
+    ) -> SmsSearchHit {
+        SmsSearchHit {
+            message_id: id,
+            contact_name: contact.to_string(),
+            contact_address: "+15551234567".to_string(),
+            body: body.to_string(),
+            date: 1_700_000_000 + id * 86_400,
+            type_,
+            similarity_score: None,
+            snippet: snippet.map(|s| s.to_string()),
+        }
+    }
+
+    #[test]
+    fn format_search_hits_falls_back_to_body_when_no_snippet() {
+        // fts5 mode often returns no snippet for messages that didn't need
+        // excerpting (whole body short, or semantic-only hit in hybrid).
+        let hit = make_search_hit(1, "Sarah", "see you at the lake tomorrow", None, 1);
+        let out = InsightGenerator::format_search_hits(&[hit], "fts5", false);
+
+        assert!(out.starts_with("Found 1 messages (mode: fts5):"));
+        assert!(out.contains("see you at the lake tomorrow"));
+        assert!(out.contains("Sarah —"));
+        assert!(!out.contains("date-filtered"));
+    }
+
+    #[test]
+    fn format_search_hits_prefers_snippet_over_body_and_strips_marks() {
+        let hit = make_search_hit(
+            2,
+            "Sarah",
+            "long body text that should be ignored once the server returns a focused snippet",
+            Some("…at the <mark>lake</mark> tomorrow…"),
+            1,
+        );
+        let out = InsightGenerator::format_search_hits(&[hit], "hybrid", true);
+
+        // Snippet wins over body, <mark> tags stripped.
+        assert!(out.contains("at the lake tomorrow"));
+        assert!(!out.contains("<mark>"));
+        assert!(!out.contains("</mark>"));
+        assert!(!out.contains("long body text"));
+        // Date-filter flag flows through to the header.
+        assert!(out.contains("date-filtered"));
+    }
+
+    #[test]
+    fn format_search_hits_surfaces_mms_attachment_only_matches() {
+        // Regression guard: pre-snippet, MMS rows that matched via
+        // message_parts_fts (filename / part text) had an empty `body`
+        // and rendered as a blank preview to the LLM.
+        let hit = make_search_hit(3, "Mom", "", Some("<mark>birthday_cake.jpg</mark>"), 1);
+        let out = InsightGenerator::format_search_hits(&[hit], "fts5", false);
+
+        assert!(out.contains("birthday_cake.jpg"));
+        assert!(!out.contains("<mark>"));
+        assert!(out.contains("Mom —"));
+    }
+
+    #[test]
+    fn format_search_hits_empty_snippet_falls_back_to_body() {
+        let hit = make_search_hit(4, "Dad", "fallback body", Some(""), 1);
+        let out = InsightGenerator::format_search_hits(&[hit], "fts5", false);
+        assert!(out.contains("fallback body"));
+    }
+
+    #[test]
+    fn strip_mark_tags_handles_common_patterns() {
+        assert_eq!(InsightGenerator::strip_mark_tags("plain text"), "plain text");
+        assert_eq!(
+            InsightGenerator::strip_mark_tags("…the <mark>lake</mark>…"),
+            "…the lake…"
+        );
+        // FTS5 highlights every match — multiple marks per snippet are normal.
+        assert_eq!(
+            InsightGenerator::strip_mark_tags("<mark>dinner</mark> at <mark>tahoe</mark>"),
+            "dinner at tahoe"
+        );
+    }
+
     #[test]
     fn summarize_search_rag_counts_hits() {
         let raw = "[2023-08-15] Sarah: venue confirmed\n\n[2023-08-14] Mom: travel plans\n\n[2023-08-13] Dad: weather";

src/ai/sms_client.rs

@@ -257,30 +257,45 @@ impl SmsApiClient {
     }
 
     /// Search message bodies via the Django side's FTS5 / semantic / hybrid
-    /// endpoint. `mode` selects the ranking strategy:
+    /// endpoint. `params.mode` selects the ranking strategy:
     ///   - "fts5"     keyword-only, supports phrase / prefix / boolean / NEAR
     ///   - "semantic" embedding similarity
     ///   - "hybrid"   both merged via reciprocal rank fusion (recommended)
     ///
-    /// The SMS-API endpoint accepts `contact_id` natively; date filtering is
+    /// All of `contact_id`, `date_from` / `date_to` (unix seconds), `is_mms`,
-    /// the caller's responsibility (post-filter on the returned rows).
+    /// `has_media`, and `offset` are pushed to SMS-API server-side so the
-    pub async fn search_messages_with_contact(
+    /// filtered+paginated result set is exact rather than a client-side
+    /// over-fetch.
+    pub async fn search_messages(
         &self,
         query: &str,
-        mode: &str,
+        params: &SmsSearchParams<'_>,
-        limit: usize,
-        contact_id: Option<i64>,
     ) -> Result<Vec<SmsSearchHit>> {
         let mut url = format!(
             "{}/api/messages/search/?q={}&mode={}&limit={}",
             self.base_url,
             urlencoding::encode(query),
-            urlencoding::encode(mode),
+            urlencoding::encode(params.mode),
-            limit
+            params.limit,
         );
-        if let Some(cid) = contact_id {
+        if let Some(cid) = params.contact_id {
             url.push_str(&format!("&contact_id={}", cid));
         }
+        if let Some(off) = params.offset {
+            url.push_str(&format!("&offset={}", off));
+        }
+        if let Some(from) = params.date_from {
+            url.push_str(&format!("&date_from={}", from));
+        }
+        if let Some(to) = params.date_to {
+            url.push_str(&format!("&date_to={}", to));
+        }
+        if let Some(is_mms) = params.is_mms {
+            url.push_str(&format!("&is_mms={}", is_mms));
+        }
+        if let Some(has_media) = params.has_media {
+            url.push_str(&format!("&has_media={}", has_media));
+        }
 
         let mut request = self.client.get(&url);
         if let Some(token) = &self.token {
@@ -383,6 +398,30 @@ pub struct SmsSearchHit {
     /// Present for semantic / hybrid modes; absent for fts5.
     #[serde(default)]
     pub similarity_score: Option<f32>,
+    /// SMS-API-generated excerpt around the match, wrapped in `<mark>` tags.
+    /// For MMS messages that only matched via attachment text / filename
+    /// (empty `body`), the snippet is the only meaningful preview.
+    #[serde(default)]
+    pub snippet: Option<String>,
+}
+
+/// Optional filter / paging knobs for [`SmsApiClient::search_messages`].
+/// All fields except `mode` and `limit` map 1:1 to the same-named SMS-API
+/// query params (added in the 2026-05 search-enhancements release).
+#[derive(Debug, Clone)]
+pub struct SmsSearchParams<'a> {
+    pub mode: &'a str,
+    pub limit: usize,
+    pub contact_id: Option<i64>,
+    /// Unix-seconds inclusive lower bound on `date`.
+    pub date_from: Option<i64>,
+    /// Unix-seconds inclusive upper bound on `date`.
+    pub date_to: Option<i64>,
+    /// `Some(true)` = MMS only, `Some(false)` = SMS only, `None` = both.
+    pub is_mms: Option<bool>,
+    /// `Some(true)` = only messages with image/video/audio attachments.
+    pub has_media: Option<bool>,
+    pub offset: Option<usize>,
 }
 
 #[derive(Deserialize)]