From f62d2ecfb4522f5ef1fc07cac7b728b99896b66a Mon Sep 17 00:00:00 2001 From: Cameron Cordes Date: Fri, 16 Oct 2026 23:31:54 -0400 Subject: [PATCH] feat(insights): read location context from Apollo (#177) The local location_history table is a one-time Takeout import: it ends in 2019, still has the ghost-device and coarse cell-fix samples Apollo has since excluded, and was read loosely. The per-photo line took the nearest sample within +/-3 h at any distance (17% of photo moments got a position >2 km away), and get_location_history listed the first 20 rows of a +/-14-day window sorted ascending (always ~14 days before the date). With APOLLO_API_BASE_URL set: - per-photo context comes from /api/location/at (30 min max gap; no sample -> no line). A Place stay renders as "at Home (14:02-18:30)" in the display zone; an unnamed sample is reverse-geocoded. - get_location_history returns Apollo's /api/location/day timeline for the date (+/-1 day at most), and is offered even with an empty local table. Both fall back to the local table when Apollo is unset or a call fails; the local tool path now covers the requested local day(s) and samples evenly instead of listing a window's first rows. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 14 ++ src/ai/apollo_client.rs | 171 +++++++++++++++++++- src/ai/insight_generator.rs | 313 +++++++++++++++++++++++++++++++----- 3 files changed, 460 insertions(+), 38 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 3e167a1..e9dae4b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -981,6 +981,20 @@ string. See `src/ai/apollo_client.rs` and `src/ai/insight_generator.rs`: variant; auto-enrichment covers the photo-context path and the agentic tool covers ad-hoc lat/lon questions in chat continuation. +- **Location context** (#177, Apollo #51): Apollo also owns the location + track — cleaned of ghost devices and coarse cell fixes, and current (HA + siphon, Garmin), unlike our `location_history` table, a one-time Takeout + import that ends in 2019. When configured, the per-photo location line + comes from `/api/location/at?ts=&max_gap=1800` (no sample within 30 min → + no line, instead of the old nearest-within-±3 h at any distance; a Place + stay renders as "You were at Home (14:02–18:30)" in the display zone — + `render_apollo_location`), and the `get_location_history` tool returns + Apollo's `/api/location/day` timeline (stays / stops / travel / gaps) for + the date, ±1 day at most. Either falls back to `location_history` when + Apollo is unset or the call fails; the local tool path samples evenly + across the requested local day(s) rather than listing a window's first + rows. The tool is offered whenever Apollo is configured, local table or not. + Failure modes degrade silently to the legacy Nominatim path: 5 s timeout, errors logged at `warn`, empty results returned. Apollo's routes are unauthenticated (single-user, LAN-trust); add JWT auth here + on Apollo's diff --git a/src/ai/apollo_client.rs b/src/ai/apollo_client.rs index 71a18eb..d3633ee 100644 --- a/src/ai/apollo_client.rs +++ b/src/ai/apollo_client.rs @@ -1,4 +1,5 @@ -//! Thin async HTTP client for Apollo's `/api/places/*` endpoints. +//! Thin async HTTP client for Apollo's `/api/places/*` and +//! `/api/location/*` endpoints. //! //! Apollo (the personal location-history viewer at the sibling repo) owns //! user-defined Places: `name + lat/lon + radius_m + description (+ optional @@ -10,6 +11,14 @@ //! 2. The agentic `get_personal_place_at` tool — lets the LLM ask "what //! user-defined place contains this lat/lon" during chat continuation. //! +//! Apollo also owns the location track itself — cleaned of ghost devices +//! and coarse cell fixes, and current (HA siphon, Garmin), unlike the +//! frozen 2019 Takeout import in our `location_history` table. The +//! insight path's location context (`/api/location/at`) and the +//! `get_location_history` tool (`/api/location/day`) read it from Apollo +//! when configured (ImageApi #177, Apollo #51), falling back to the local +//! table when Apollo is unset or a call fails. +//! //! Apollo does the haversine. This client is plumbing only — no geometry, //! no caching at the moment. If insight throughput ever makes per-photo //! HTTP latency a problem, swap to a small `Mutex` TTL cache here. @@ -46,6 +55,73 @@ struct PlacesResponse { places: Vec, } +/// A kept location sample from Apollo, `offset_s` = sample ts − asked ts. +#[allow(dead_code)] +#[derive(Debug, Clone, Deserialize)] +pub struct LocationSample { + pub ts: i64, + pub lat: f64, + pub lon: f64, + #[serde(default)] + pub accuracy: Option, + #[serde(default)] + pub activity: Option, + pub offset_s: i64, +} + +#[allow(dead_code)] +#[derive(Debug, Clone, Deserialize)] +pub struct LocationPlace { + pub id: i32, + pub name: String, + #[serde(default)] + pub category: Option, + pub radius_m: i32, +} + +/// The Place stay (run of samples inside one Place) covering an instant. +#[allow(dead_code)] +#[derive(Debug, Clone, Deserialize)] +pub struct LocationStay { + pub place_id: i32, + #[serde(default)] + pub name: Option, + pub t0: i64, + pub t1: i64, +} + +/// `/api/location/at`: where the owner was at an instant. `sample` is +/// `None` when no kept sample lies within `max_gap` — Apollo says "no +/// data" rather than handing back a position from hours away. +#[allow(dead_code)] +#[derive(Debug, Clone, Deserialize)] +pub struct LocationAt { + pub ts: i64, + #[serde(default)] + pub sample: Option, + #[serde(default)] + pub nearest_offset_s: Option, + #[serde(default)] + pub bracketed: bool, + #[serde(default)] + pub place: Option, + #[serde(default)] + pub stay: Option, +} + +/// `/api/location/day`: the owner's timeline for one local day. `text` is +/// Apollo's compact rendering (one "HH:MM–HH:MM at Home" line per stay / +/// dwell / travel / gap), meant to go into a tool result as-is; the +/// structured `items` aren't needed here. +#[allow(dead_code)] +#[derive(Debug, Clone, Deserialize)] +pub struct LocationDay { + pub tz: String, + pub points: i64, + pub km: f64, + pub text: String, +} + #[derive(Clone)] pub struct ApolloClient { client: Client, @@ -90,6 +166,52 @@ impl ApolloClient { } } + /// Where the owner was at `ts` (real UTC seconds). `None` when the + /// integration is disabled or the call fails — callers fall back to + /// the local table; `Some` with `sample: None` means Apollo answered + /// "no data within `max_gap`", which callers should respect. + pub async fn location_at(&self, ts: i64, max_gap: i64) -> Option { + let base = self.base_url.as_deref()?; + let url = format!("{}/api/location/at", base.trim_end_matches('/')); + let result: Result = async { + Ok(self + .client + .get(&url) + .query(&[("ts", ts), ("max_gap", max_gap)]) + .send() + .await? + .error_for_status()? + .json() + .await?) + } + .await; + result + .map_err(|err| log::warn!("apollo_client: location_at({ts}) failed: {err}")) + .ok() + } + + /// The owner's timeline for `date` (YYYY-MM-DD) as a local day in + /// `tz` (IANA). `None` when disabled or the call fails. + pub async fn location_day(&self, date: &str, tz: &str) -> Option { + let base = self.base_url.as_deref()?; + let url = format!("{}/api/location/day", base.trim_end_matches('/')); + let result: Result = async { + Ok(self + .client + .get(&url) + .query(&[("date", date), ("tz", tz)]) + .send() + .await? + .error_for_status()? + .json() + .await?) + } + .await; + result + .map_err(|err| log::warn!("apollo_client: location_day({date}, {tz}) failed: {err}")) + .ok() + } + async fn fetch_places_containing( &self, base: &str, @@ -108,3 +230,50 @@ impl ApolloClient { Ok(body.places) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn location_at_parses_hit_and_miss() { + let hit: LocationAt = serde_json::from_str( + r#"{"ts": 100, "max_gap": 1800, + "sample": {"ts": 120, "lat": 41.46, "lon": -81.93, "accuracy": 19, "activity": "STILL", "offset_s": 20}, + "nearest_offset_s": 20, "bracketed": true, + "place": {"id": 7, "name": "Work", "category": null, "radius_m": 250}, + "stay": {"place_id": 7, "name": "Work", "t0": 50, "t1": 900}}"#, + ) + .unwrap(); + assert_eq!(hit.sample.unwrap().offset_s, 20); + assert_eq!(hit.place.unwrap().name, "Work"); + assert_eq!(hit.stay.unwrap().t1, 900); + + let miss: LocationAt = serde_json::from_str( + r#"{"ts": 100, "max_gap": 1800, "sample": null, "nearest_offset_s": -7200, + "bracketed": false, "place": null, "stay": null}"#, + ) + .unwrap(); + assert!(miss.sample.is_none()); + assert_eq!(miss.nearest_offset_s, Some(-7200)); + } + + #[test] + fn location_day_ignores_items() { + let day: LocationDay = serde_json::from_str( + r#"{"start": 0, "end": 86400, "tz": "America/New_York", + "items": [{"kind": "gap", "t0": 0, "t1": 1}], "points": 3, "km": 1.5, + "text": "00:00–23:59 at Home"}"#, + ) + .unwrap(); + assert_eq!(day.text, "00:00–23:59 at Home"); + assert_eq!(day.points, 3); + } + + #[tokio::test] + async fn disabled_client_returns_none() { + let c = ApolloClient::new(None); + assert!(c.location_at(0, 1800).await.is_none()); + assert!(c.location_day("2016-01-01", "UTC").await.is_none()); + } +} diff --git a/src/ai/insight_generator.rs b/src/ai/insight_generator.rs index df18fd4..09598a9 100644 --- a/src/ai/insight_generator.rs +++ b/src/ai/insight_generator.rs @@ -11,7 +11,7 @@ use std::sync::{Arc, Mutex}; use tracing::Instrument; use tracing_opentelemetry::OpenTelemetrySpanExt; -use crate::ai::apollo_client::{ApolloClient, ApolloPlace}; +use crate::ai::apollo_client::{ApolloClient, ApolloPlace, LocationAt}; use crate::ai::backend::{BackendKind, ResolvedBackend, SamplingOverrides}; use crate::ai::llamacpp::LlamaCppClient; use crate::ai::llm_client::LlmClient; @@ -114,6 +114,105 @@ fn compose_location_string( } } +/// How far from a photo's instant Apollo may look for a location sample +/// for the always-on insight context. Past this Apollo answers "no data" +/// — the old local lookup took the nearest sample within ±3 h at any +/// distance, and 17 % of photo moments got a position > 2 km from where +/// the photo was taken. +const APOLLO_LOCATION_MAX_GAP_S: i64 = 1800; + +/// Apollo activities that say nothing about what the owner was doing. +const UNINFORMATIVE_ACTIVITIES: &[&str] = &["UNKNOWN", "TILTING", "EXITING_VEHICLE"]; + +/// Render Apollo's `/api/location/at` answer as the insight prompt's +/// location-history line, or `None` when it adds nothing: +/// +/// - no sample within the gap → `None` ("no data" beats a stale position); +/// - the photo has EXIF GPS within 500 m of the sample and Apollo knows no +/// stay or activity → `None` (the EXIF location line already says it); +/// - otherwise "You were at Home (14:02–18:30)", "at Home", or "near +/// ", plus the activity and, when the fix is more +/// than 5 min from the photo, how far off it is. +/// +/// Stay times render in the owner's display zone. Free function so it can +/// be tested without an InsightGenerator; `nominatim` is the caller's +/// reverse-geocode of the sample (only needed when there's no Place). +fn render_apollo_location( + at: &LocationAt, + exif_gps: Option<(f64, f64)>, + zone: chrono_tz::Tz, + nominatim: Option, +) -> Option { + let sample = at.sample.as_ref()?; + let activity = sample + .activity + .as_deref() + .filter(|a| !UNINFORMATIVE_ACTIVITIES.contains(a)); + if let Some((lat, lon)) = exif_gps { + let km = InsightGenerator::haversine_distance(lat, lon, sample.lat, sample.lon); + if km < 0.5 && at.stay.is_none() && activity.is_none() { + return None; + } + } + let hm = |ts: i64| { + DateTime::from_timestamp(ts, 0) + .map(|d| d.with_timezone(&zone).format("%H:%M").to_string()) + .unwrap_or_default() + }; + let place_name = at + .stay + .as_ref() + .and_then(|s| s.name.clone()) + .or_else(|| at.place.as_ref().map(|p| p.name.clone())); + let mut line = match (&at.stay, place_name) { + (Some(stay), Some(name)) => { + format!("You were at {} ({}–{})", name, hm(stay.t0), hm(stay.t1)) + } + (None, Some(name)) => format!("You were at {}", name), + (_, None) => match nominatim { + Some(n) => format!("You were near {}", n), + None => format!("You were near {:.4}, {:.4}", sample.lat, sample.lon), + }, + }; + if let Some(a) = activity { + line.push_str(&format!(" ({})", a)); + } + if sample.offset_s.abs() > 300 { + line.push_str(&format!( + " [location fix {} min {} the photo]", + sample.offset_s.abs() / 60, + if sample.offset_s < 0 { + "before" + } else { + "after" + } + )); + } + Some(format!("Location history: {}", line)) +} + +/// One day of Apollo's timeline as a tool-result block. +fn format_apollo_day(date: &str, day: &crate::ai::apollo_client::LocationDay) -> String { + format!( + "{} ({} location points, {:.1} km):\n{}", + date, day.points, day.km, day.text + ) +} + +/// Up to `limit` items spaced evenly through `items`, first and last +/// included, in their original order. +fn spread_evenly(items: &[T], limit: usize) -> Vec<&T> { + if items.len() <= limit { + return items.iter().collect(); + } + if limit < 2 { + return items.iter().take(limit).collect(); + } + (0..limit) + .map(|k| &items[k * (items.len() - 1) / (limit - 1)]) + .collect() +} + #[derive(Deserialize)] struct NominatimResponse { display_name: Option, @@ -352,7 +451,8 @@ impl InsightGenerator { .expect("Unable to lock CalendarEventDao"); dao.get_event_count().map(|n| n > 0).unwrap_or(false) }; - let location_history_present = { + // Apollo serves the tool when configured, local table or not. + let location_history_present = self.apollo_enabled() || { let mut dao = self .location_dao .lock() @@ -827,8 +927,10 @@ impl InsightGenerator { } } - /// Gather location context for photo timestamp - /// Finds nearest location record (±30 minutes) + /// Gather location context for photo timestamp. Asks Apollo + /// (`/api/location/at`, cleaned + current track, ±30 min) when it's + /// configured; falls back to the nearest local `location_history` + /// record within ±3 h when it isn't, or when the call fails. #[tracing::instrument(name = "ai.context.location", skip_all)] async fn gather_location_context( &self, @@ -837,6 +939,27 @@ impl InsightGenerator { ) -> Result> { let span = tracing::Span::current(); + if let Some(at) = self + .apollo_client + .location_at(timestamp, APOLLO_LOCATION_MAX_GAP_S) + .await + { + span.set_attribute("location_source", "apollo"); + // Reverse-geocode only an unnamed sample on a photo without + // EXIF GPS — with EXIF, the location line already has it. + let nominatim = match (&at.sample, &at.place, &at.stay, exif_gps) { + (Some(s), None, None, None) => self.reverse_geocode(s.lat, s.lon).await, + _ => None, + }; + span.set_status(Status::Ok); + return Ok(render_apollo_location( + &at, + exif_gps, + self.display_zone(), + nominatim, + )); + } + let nearest = { let mut dao = self .location_dao @@ -2339,53 +2462,85 @@ Return ONLY the summary, nothing else."#, } } - /// Tool: get_location_history — fetch location records near a date + /// Tool: get_location_history — where the owner was on a date. + /// + /// With Apollo configured: Apollo's day timeline (`/api/location/day`: + /// Place stays, stops, travel with km, gaps) for the date's local day + /// in the owner's display zone, or for date±1 at most. Otherwise, or + /// if every Apollo call fails: up to `LOCATION_HISTORY_DISPLAY_LIMIT` + /// raw local records spaced evenly across the requested local days. + /// (The local path used to list the first 20 rows of a ±14-day window + /// sorted ascending — always ~14 days before the date asked about.) async fn tool_get_location_history(&self, args: &serde_json::Value) -> String { let date_str = match args.get("date").and_then(|v| v.as_str()) { Some(d) => d, None => return "Error: missing required parameter 'date'".to_string(), }; - let days_radius = args - .get("days_radius") - .and_then(|v| v.as_i64()) - .unwrap_or(14) - .clamp(1, 30); - let date = match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") { Ok(d) => d, Err(e) => return format!("Error: failed to parse date '{}': {}", date_str, e), }; - let timestamp = date.and_hms_opt(12, 0, 0).unwrap().and_utc().timestamp(); + let days_radius = args + .get("days_radius") + .and_then(|v| v.as_i64()) + .unwrap_or(0); + let zone = self.display_zone(); log::info!( - "tool_get_location_history: date={}, days_radius={}", + "tool_get_location_history: date={}, days_radius={}, apollo={}", date, - days_radius + days_radius, + self.apollo_client.is_enabled() ); - let start_ts = timestamp - (days_radius * 86400); - let end_ts = timestamp + (days_radius * 86400); + if self.apollo_client.is_enabled() { + let r = days_radius.clamp(0, 1); + let mut days = Vec::new(); + for d in -r..=r { + let day = date + chrono::Duration::days(d); + let day_str = day.format("%Y-%m-%d").to_string(); + if let Some(t) = self.apollo_client.location_day(&day_str, zone.name()).await { + days.push(format_apollo_day(&day_str, &t)); + } + } + if !days.is_empty() { + return format!( + "Location timeline ({}):\n\n{}", + zone.name(), + days.join("\n\n") + ); + } + // Every call failed — fall through to the local table. + } + + let r = days_radius.clamp(0, 30); + let midnight = |d: NaiveDate| { + zone.from_local_datetime(&d.and_hms_opt(0, 0, 0).unwrap()) + .earliest() + .map(|t| t.timestamp()) + }; + let (Some(start_ts), Some(end_ts)) = ( + midnight(date - chrono::Duration::days(r)), + midnight(date + chrono::Duration::days(r + 1)), + ) else { + return format!("Error: could not resolve local midnight for {}", date); + }; let locations = { let mut dao = self .location_dao .lock() .expect("Unable to lock LocationHistoryDao"); - dao.find_locations_in_range(start_ts, end_ts).ok() + dao.find_locations_in_range(start_ts, end_ts - 1).ok() }; match locations { Some(locs) if !locs.is_empty() => { - let formatted: Vec = locs - .iter() - .take(LOCATION_HISTORY_DISPLAY_LIMIT) + let formatted: Vec = spread_evenly(&locs, LOCATION_HISTORY_DISPLAY_LIMIT) + .into_iter() .map(|loc| { let dt = DateTime::from_timestamp(loc.timestamp, 0) - .map(|dt| { - dt.with_timezone(&Local) - .format("%Y-%m-%d %H:%M") - .to_string() - }) + .map(|dt| dt.with_timezone(&zone).format("%Y-%m-%d %H:%M").to_string()) .unwrap_or_else(|| "unknown".to_string()); let activity = loc .activity @@ -2403,14 +2558,18 @@ Return ONLY the summary, nothing else."#, ) }) .collect(); - format!( - "{}\n{}", - Self::found_header(locs.len(), formatted.len(), "location records"), - formatted.join("\n") - ) + let header = if formatted.len() < locs.len() { + format!( + "Found {} location records (showing {}, evenly spaced across the window):", + locs.len(), + formatted.len() + ) + } else { + Self::found_header(locs.len(), formatted.len(), "location records") + }; + format!("{}\n{}", header, formatted.join("\n")) } - Some(_) => "No location history found.".to_string(), - None => "No location history found.".to_string(), + _ => "No location history found.".to_string(), } } @@ -3406,15 +3565,16 @@ Return ONLY the summary, nothing else."#, if opts.location_history_present { tools.push(Tool::function( "get_location_history", - "Fetch raw location records (lat/lon/timestamp/activity) near a date. The default 14-day radius is \ - wide because location density varies; tighten to ±1 day for a single-trip query. For a coordinate's \ - named place, use `reverse_geocode` (or `get_personal_place_at` when Apollo is enabled).", + "Where the user was on a date: a timeline of named places (Home, Work, …) with arrival/departure \ + times, stops, travel with distances, and gaps with no data — or, without Apollo, raw location \ + records spread across the day. Times are local. For a coordinate's named place, use \ + `get_personal_place_at` (Apollo) or `reverse_geocode`.", serde_json::json!({ "type": "object", "required": ["date"], "properties": { - "date": { "type": "string", "description": "Center date, YYYY-MM-DD." }, - "days_radius": { "type": "integer", "description": "Days before and after to include (default 14)." } + "date": { "type": "string", "description": "The day, YYYY-MM-DD (local)." }, + "days_radius": { "type": "integer", "description": "Also include this many days before and after (default 0; at most 1 with Apollo)." } } }), )); @@ -4948,6 +5108,85 @@ mod tests { ); } + fn apollo_at(json: &str) -> LocationAt { + serde_json::from_str(json).unwrap() + } + + const NY: chrono_tz::Tz = chrono_tz::Tz::America__New_York; + + #[test] + fn apollo_location_no_sample_says_nothing() { + let at = apollo_at(r#"{"ts": 0, "sample": null, "nearest_offset_s": 7200}"#); + assert_eq!( + render_apollo_location(&at, None, NY, Some("Medina, OH".into())), + None + ); + } + + #[test] + fn apollo_location_renders_stay_in_display_zone() { + // 2017-01-17 08:07 / 16:19 EST + let at = apollo_at( + r#"{"ts": 1484669220, + "sample": {"ts": 1484669240, "lat": 41.469, "lon": -81.934, "activity": "TILTING", "offset_s": 20}, + "place": {"id": 7, "name": "Hyland Software", "radius_m": 250}, + "stay": {"place_id": 7, "name": "Hyland Software", "t0": 1484658450, "t1": 1484687985}}"#, + ); + assert_eq!( + render_apollo_location(&at, None, NY, None).as_deref(), + Some("Location history: You were at Hyland Software (08:07–16:19)") + ); + } + + #[test] + fn apollo_location_unnamed_uses_nominatim_activity_and_offset() { + let at = apollo_at( + r#"{"ts": 1000, + "sample": {"ts": 100, "lat": 41.1, "lon": -81.8, "activity": "IN_VEHICLE", "offset_s": -900}}"#, + ); + assert_eq!( + render_apollo_location(&at, None, NY, Some("Medina, OH".into())).as_deref(), + Some( + "Location history: You were near Medina, OH (IN_VEHICLE) [location fix 15 min before the photo]" + ) + ); + assert_eq!( + render_apollo_location(&at, None, NY, None).as_deref(), + Some( + "Location history: You were near 41.1000, -81.8000 (IN_VEHICLE) [location fix 15 min before the photo]" + ) + ); + } + + #[test] + fn apollo_location_skips_what_exif_already_says() { + let plain = apollo_at( + r#"{"ts": 0, "sample": {"ts": 0, "lat": 41.1, "lon": -81.8, "activity": "UNKNOWN", "offset_s": 0}, + "place": {"id": 1, "name": "Park", "radius_m": 300}}"#, + ); + // EXIF 100 m away, no stay, no informative activity → nothing new. + assert_eq!( + render_apollo_location(&plain, Some((41.1009, -81.8)), NY, None), + None + ); + // Without EXIF the same answer is worth a line. + assert_eq!( + render_apollo_location(&plain, None, NY, None).as_deref(), + Some("Location history: You were at Park") + ); + } + + #[test] + fn spread_evenly_keeps_ends_and_order() { + let v: Vec = (0..100).collect(); + let picked: Vec = spread_evenly(&v, 20).into_iter().copied().collect(); + assert_eq!(picked.len(), 20); + assert_eq!((picked[0], picked[19]), (0, 99)); + assert!(picked.windows(2).all(|w| w[0] < w[1])); + assert_eq!(spread_evenly(&v[..5], 20).len(), 5); + assert_eq!(spread_evenly(&v, 1), vec![&0]); + } + #[test] fn found_header_labels_truncation_honestly() { // Truncated: total exceeds what's listed below the header.