feat(insights): read location context from Apollo (#177)

The local location_history table is a one-time Takeout import: it ends
in 2019, still has the ghost-device and coarse cell-fix samples Apollo
has since excluded, and was read loosely. The per-photo line took the
nearest sample within +/-3 h at any distance (17% of photo moments got a
position >2 km away), and get_location_history listed the first 20 rows
of a +/-14-day window sorted ascending (always ~14 days before the date).

With APOLLO_API_BASE_URL set:
- per-photo context comes from /api/location/at (30 min max gap; no
  sample -> no line). A Place stay renders as "at Home (14:02-18:30)" in
  the display zone; an unnamed sample is reverse-geocoded.
- get_location_history returns Apollo's /api/location/day timeline for
  the date (+/-1 day at most), and is offered even with an empty local
  table.

Both fall back to the local table when Apollo is unset or a call fails;
the local tool path now covers the requested local day(s) and samples
evenly instead of listing a window's first rows.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Cameron CordesandClaude Opus 5.5 authored and cameron committed 2026-10-17 04:21:49 +00:00
1 parent bc27aea81a
commit f62d2ecfb4
3 files changed
+460 -38

No files matched your search

+14
View File
@@ -981,6 +981,20 @@ string. See `src/ai/apollo_client.rs` and `src/ai/insight_generator.rs`:
variant; auto-enrichment covers the photo-context path and the agentic
tool covers ad-hoc lat/lon questions in chat continuation.
- **Location context** (#177, Apollo #51): Apollo also owns the location
track — cleaned of ghost devices and coarse cell fixes, and current (HA
siphon, Garmin), unlike our `location_history` table, a one-time Takeout
import that ends in 2019. When configured, the per-photo location line
comes from `/api/location/at?ts=&max_gap=1800` (no sample within 30 min →
no line, instead of the old nearest-within-±3 h at any distance; a Place
stay renders as "You were at Home (14:02–18:30)" in the display zone —
`render_apollo_location`), and the `get_location_history` tool returns
Apollo's `/api/location/day` timeline (stays / stops / travel / gaps) for
the date, ±1 day at most. Either falls back to `location_history` when
Apollo is unset or the call fails; the local tool path samples evenly
across the requested local day(s) rather than listing a window's first
rows. The tool is offered whenever Apollo is configured, local table or not.
Failure modes degrade silently to the legacy Nominatim path: 5 s timeout,
errors logged at `warn`, empty results returned. Apollo's routes are
unauthenticated (single-user, LAN-trust); add JWT auth here + on Apollo's
+170 -1
View File
@@ -1,4 +1,5 @@
//! Thin async HTTP client for Apollo's `/api/places/*` endpoints.
//! Thin async HTTP client for Apollo's `/api/places/*` and
//! `/api/location/*` endpoints.
//!
//! Apollo (the personal location-history viewer at the sibling repo) owns
//! user-defined Places: `name + lat/lon + radius_m + description (+ optional
@@ -10,6 +11,14 @@
//! 2. The agentic `get_personal_place_at` tool — lets the LLM ask "what
//! user-defined place contains this lat/lon" during chat continuation.
//!
//! Apollo also owns the location track itself — cleaned of ghost devices
//! and coarse cell fixes, and current (HA siphon, Garmin), unlike the
//! frozen 2019 Takeout import in our `location_history` table. The
//! insight path's location context (`/api/location/at`) and the
//! `get_location_history` tool (`/api/location/day`) read it from Apollo
//! when configured (ImageApi #177, Apollo #51), falling back to the local
//! table when Apollo is unset or a call fails.
//!
//! Apollo does the haversine. This client is plumbing only — no geometry,
//! no caching at the moment. If insight throughput ever makes per-photo
//! HTTP latency a problem, swap to a small `Mutex<HashMap>` TTL cache here.
@@ -46,6 +55,73 @@ struct PlacesResponse {
places: Vec<ApolloPlace>,
}
/// A kept location sample from Apollo, `offset_s` = sample ts − asked ts.
#[allow(dead_code)]
#[derive(Debug, Clone, Deserialize)]
pub struct LocationSample {
pub ts: i64,
pub lat: f64,
pub lon: f64,
#[serde(default)]
pub accuracy: Option<i32>,
#[serde(default)]
pub activity: Option<String>,
pub offset_s: i64,
}
#[allow(dead_code)]
#[derive(Debug, Clone, Deserialize)]
pub struct LocationPlace {
pub id: i32,
pub name: String,
#[serde(default)]
pub category: Option<String>,
pub radius_m: i32,
}
/// The Place stay (run of samples inside one Place) covering an instant.
#[allow(dead_code)]
#[derive(Debug, Clone, Deserialize)]
pub struct LocationStay {
pub place_id: i32,
#[serde(default)]
pub name: Option<String>,
pub t0: i64,
pub t1: i64,
}
/// `/api/location/at`: where the owner was at an instant. `sample` is
/// `None` when no kept sample lies within `max_gap` — Apollo says "no
/// data" rather than handing back a position from hours away.
#[allow(dead_code)]
#[derive(Debug, Clone, Deserialize)]
pub struct LocationAt {
pub ts: i64,
#[serde(default)]
pub sample: Option<LocationSample>,
#[serde(default)]
pub nearest_offset_s: Option<i64>,
#[serde(default)]
pub bracketed: bool,
#[serde(default)]
pub place: Option<LocationPlace>,
#[serde(default)]
pub stay: Option<LocationStay>,
}
/// `/api/location/day`: the owner's timeline for one local day. `text` is
/// Apollo's compact rendering (one "HH:MM–HH:MM at Home" line per stay /
/// dwell / travel / gap), meant to go into a tool result as-is; the
/// structured `items` aren't needed here.
#[allow(dead_code)]
#[derive(Debug, Clone, Deserialize)]
pub struct LocationDay {
pub tz: String,
pub points: i64,
pub km: f64,
pub text: String,
}
#[derive(Clone)]
pub struct ApolloClient {
client: Client,
@@ -90,6 +166,52 @@ impl ApolloClient {
}
}
/// Where the owner was at `ts` (real UTC seconds). `None` when the
/// integration is disabled or the call fails — callers fall back to
/// the local table; `Some` with `sample: None` means Apollo answered
/// "no data within `max_gap`", which callers should respect.
pub async fn location_at(&self, ts: i64, max_gap: i64) -> Option<LocationAt> {
let base = self.base_url.as_deref()?;
let url = format!("{}/api/location/at", base.trim_end_matches('/'));
let result: Result<LocationAt> = async {
Ok(self
.client
.get(&url)
.query(&[("ts", ts), ("max_gap", max_gap)])
.send()
.await?
.error_for_status()?
.json()
.await?)
}
.await;
result
.map_err(|err| log::warn!("apollo_client: location_at({ts}) failed: {err}"))
.ok()
}
/// The owner's timeline for `date` (YYYY-MM-DD) as a local day in
/// `tz` (IANA). `None` when disabled or the call fails.
pub async fn location_day(&self, date: &str, tz: &str) -> Option<LocationDay> {
let base = self.base_url.as_deref()?;
let url = format!("{}/api/location/day", base.trim_end_matches('/'));
let result: Result<LocationDay> = async {
Ok(self
.client
.get(&url)
.query(&[("date", date), ("tz", tz)])
.send()
.await?
.error_for_status()?
.json()
.await?)
}
.await;
result
.map_err(|err| log::warn!("apollo_client: location_day({date}, {tz}) failed: {err}"))
.ok()
}
async fn fetch_places_containing(
&self,
base: &str,
@@ -108,3 +230,50 @@ impl ApolloClient {
Ok(body.places)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn location_at_parses_hit_and_miss() {
let hit: LocationAt = serde_json::from_str(
r#"{"ts": 100, "max_gap": 1800,
"sample": {"ts": 120, "lat": 41.46, "lon": -81.93, "accuracy": 19, "activity": "STILL", "offset_s": 20},
"nearest_offset_s": 20, "bracketed": true,
"place": {"id": 7, "name": "Work", "category": null, "radius_m": 250},
"stay": {"place_id": 7, "name": "Work", "t0": 50, "t1": 900}}"#,
)
.unwrap();
assert_eq!(hit.sample.unwrap().offset_s, 20);
assert_eq!(hit.place.unwrap().name, "Work");
assert_eq!(hit.stay.unwrap().t1, 900);
let miss: LocationAt = serde_json::from_str(
r#"{"ts": 100, "max_gap": 1800, "sample": null, "nearest_offset_s": -7200,
"bracketed": false, "place": null, "stay": null}"#,
)
.unwrap();
assert!(miss.sample.is_none());
assert_eq!(miss.nearest_offset_s, Some(-7200));
}
#[test]
fn location_day_ignores_items() {
let day: LocationDay = serde_json::from_str(
r#"{"start": 0, "end": 86400, "tz": "America/New_York",
"items": [{"kind": "gap", "t0": 0, "t1": 1}], "points": 3, "km": 1.5,
"text": "00:00–23:59 at Home"}"#,
)
.unwrap();
assert_eq!(day.text, "00:00–23:59 at Home");
assert_eq!(day.points, 3);
}
#[tokio::test]
async fn disabled_client_returns_none() {
let c = ApolloClient::new(None);
assert!(c.location_at(0, 1800).await.is_none());
assert!(c.location_day("2016-01-01", "UTC").await.is_none());
}
}
+276 -37
View File
@@ -11,7 +11,7 @@ use std::sync::{Arc, Mutex};
use tracing::Instrument;
use tracing_opentelemetry::OpenTelemetrySpanExt;
use crate::ai::apollo_client::{ApolloClient, ApolloPlace};
use crate::ai::apollo_client::{ApolloClient, ApolloPlace, LocationAt};
use crate::ai::backend::{BackendKind, ResolvedBackend, SamplingOverrides};
use crate::ai::llamacpp::LlamaCppClient;
use crate::ai::llm_client::LlmClient;
@@ -114,6 +114,105 @@ fn compose_location_string(
}
}
/// How far from a photo's instant Apollo may look for a location sample
/// for the always-on insight context. Past this Apollo answers "no data"
/// — the old local lookup took the nearest sample within ±3 h at any
/// distance, and 17 % of photo moments got a position > 2 km from where
/// the photo was taken.
const APOLLO_LOCATION_MAX_GAP_S: i64 = 1800;
/// Apollo activities that say nothing about what the owner was doing.
const UNINFORMATIVE_ACTIVITIES: &[&str] = &["UNKNOWN", "TILTING", "EXITING_VEHICLE"];
/// Render Apollo's `/api/location/at` answer as the insight prompt's
/// location-history line, or `None` when it adds nothing:
///
/// - no sample within the gap → `None` ("no data" beats a stale position);
/// - the photo has EXIF GPS within 500 m of the sample and Apollo knows no
/// stay or activity → `None` (the EXIF location line already says it);
/// - otherwise "You were at Home (14:02–18:30)", "at Home", or "near
/// <nominatim or coords>", plus the activity and, when the fix is more
/// than 5 min from the photo, how far off it is.
///
/// Stay times render in the owner's display zone. Free function so it can
/// be tested without an InsightGenerator; `nominatim` is the caller's
/// reverse-geocode of the sample (only needed when there's no Place).
fn render_apollo_location(
at: &LocationAt,
exif_gps: Option<(f64, f64)>,
zone: chrono_tz::Tz,
nominatim: Option<String>,
) -> Option<String> {
let sample = at.sample.as_ref()?;
let activity = sample
.activity
.as_deref()
.filter(|a| !UNINFORMATIVE_ACTIVITIES.contains(a));
if let Some((lat, lon)) = exif_gps {
let km = InsightGenerator::haversine_distance(lat, lon, sample.lat, sample.lon);
if km < 0.5 && at.stay.is_none() && activity.is_none() {
return None;
}
}
let hm = |ts: i64| {
DateTime::from_timestamp(ts, 0)
.map(|d| d.with_timezone(&zone).format("%H:%M").to_string())
.unwrap_or_default()
};
let place_name = at
.stay
.as_ref()
.and_then(|s| s.name.clone())
.or_else(|| at.place.as_ref().map(|p| p.name.clone()));
let mut line = match (&at.stay, place_name) {
(Some(stay), Some(name)) => {
format!("You were at {} ({}–{})", name, hm(stay.t0), hm(stay.t1))
}
(None, Some(name)) => format!("You were at {}", name),
(_, None) => match nominatim {
Some(n) => format!("You were near {}", n),
None => format!("You were near {:.4}, {:.4}", sample.lat, sample.lon),
},
};
if let Some(a) = activity {
line.push_str(&format!(" ({})", a));
}
if sample.offset_s.abs() > 300 {
line.push_str(&format!(
" [location fix {} min {} the photo]",
sample.offset_s.abs() / 60,
if sample.offset_s < 0 {
"before"
} else {
"after"
}
));
}
Some(format!("Location history: {}", line))
}
/// One day of Apollo's timeline as a tool-result block.
fn format_apollo_day(date: &str, day: &crate::ai::apollo_client::LocationDay) -> String {
format!(
"{} ({} location points, {:.1} km):\n{}",
date, day.points, day.km, day.text
)
}
/// Up to `limit` items spaced evenly through `items`, first and last
/// included, in their original order.
fn spread_evenly<T>(items: &[T], limit: usize) -> Vec<&T> {
if items.len() <= limit {
return items.iter().collect();
}
if limit < 2 {
return items.iter().take(limit).collect();
}
(0..limit)
.map(|k| &items[k * (items.len() - 1) / (limit - 1)])
.collect()
}
#[derive(Deserialize)]
struct NominatimResponse {
display_name: Option<String>,
@@ -352,7 +451,8 @@ impl InsightGenerator {
.expect("Unable to lock CalendarEventDao");
dao.get_event_count().map(|n| n > 0).unwrap_or(false)
};
let location_history_present = {
// Apollo serves the tool when configured, local table or not.
let location_history_present = self.apollo_enabled() || {
let mut dao = self
.location_dao
.lock()
@@ -827,8 +927,10 @@ impl InsightGenerator {
}
}
/// Gather location context for photo timestamp
/// Finds nearest location record (±30 minutes)
/// Gather location context for photo timestamp. Asks Apollo
/// (`/api/location/at`, cleaned + current track, ±30 min) when it's
/// configured; falls back to the nearest local `location_history`
/// record within ±3 h when it isn't, or when the call fails.
#[tracing::instrument(name = "ai.context.location", skip_all)]
async fn gather_location_context(
&self,
@@ -837,6 +939,27 @@ impl InsightGenerator {
) -> Result<Option<String>> {
let span = tracing::Span::current();
if let Some(at) = self
.apollo_client
.location_at(timestamp, APOLLO_LOCATION_MAX_GAP_S)
.await
{
span.set_attribute("location_source", "apollo");
// Reverse-geocode only an unnamed sample on a photo without
// EXIF GPS — with EXIF, the location line already has it.
let nominatim = match (&at.sample, &at.place, &at.stay, exif_gps) {
(Some(s), None, None, None) => self.reverse_geocode(s.lat, s.lon).await,
_ => None,
};
span.set_status(Status::Ok);
return Ok(render_apollo_location(
&at,
exif_gps,
self.display_zone(),
nominatim,
));
}
let nearest = {
let mut dao = self
.location_dao
@@ -2339,53 +2462,85 @@ Return ONLY the summary, nothing else."#,
}
}
/// Tool: get_location_history — fetch location records near a date
/// Tool: get_location_history — where the owner was on a date.
///
/// With Apollo configured: Apollo's day timeline (`/api/location/day`:
/// Place stays, stops, travel with km, gaps) for the date's local day
/// in the owner's display zone, or for date±1 at most. Otherwise, or
/// if every Apollo call fails: up to `LOCATION_HISTORY_DISPLAY_LIMIT`
/// raw local records spaced evenly across the requested local days.
/// (The local path used to list the first 20 rows of a ±14-day window
/// sorted ascending — always ~14 days before the date asked about.)
async fn tool_get_location_history(&self, args: &serde_json::Value) -> String {
let date_str = match args.get("date").and_then(|v| v.as_str()) {
Some(d) => d,
None => return "Error: missing required parameter 'date'".to_string(),
};
let days_radius = args
.get("days_radius")
.and_then(|v| v.as_i64())
.unwrap_or(14)
.clamp(1, 30);
let date = match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") {
Ok(d) => d,
Err(e) => return format!("Error: failed to parse date '{}': {}", date_str, e),
};
let timestamp = date.and_hms_opt(12, 0, 0).unwrap().and_utc().timestamp();
let days_radius = args
.get("days_radius")
.and_then(|v| v.as_i64())
.unwrap_or(0);
let zone = self.display_zone();
log::info!(
"tool_get_location_history: date={}, days_radius={}",
"tool_get_location_history: date={}, days_radius={}, apollo={}",
date,
days_radius
days_radius,
self.apollo_client.is_enabled()
);
let start_ts = timestamp - (days_radius * 86400);
let end_ts = timestamp + (days_radius * 86400);
if self.apollo_client.is_enabled() {
let r = days_radius.clamp(0, 1);
let mut days = Vec::new();
for d in -r..=r {
let day = date + chrono::Duration::days(d);
let day_str = day.format("%Y-%m-%d").to_string();
if let Some(t) = self.apollo_client.location_day(&day_str, zone.name()).await {
days.push(format_apollo_day(&day_str, &t));
}
}
if !days.is_empty() {
return format!(
"Location timeline ({}):\n\n{}",
zone.name(),
days.join("\n\n")
);
}
// Every call failed — fall through to the local table.
}
let r = days_radius.clamp(0, 30);
let midnight = |d: NaiveDate| {
zone.from_local_datetime(&d.and_hms_opt(0, 0, 0).unwrap())
.earliest()
.map(|t| t.timestamp())
};
let (Some(start_ts), Some(end_ts)) = (
midnight(date - chrono::Duration::days(r)),
midnight(date + chrono::Duration::days(r + 1)),
) else {
return format!("Error: could not resolve local midnight for {}", date);
};
let locations = {
let mut dao = self
.location_dao
.lock()
.expect("Unable to lock LocationHistoryDao");
dao.find_locations_in_range(start_ts, end_ts).ok()
dao.find_locations_in_range(start_ts, end_ts - 1).ok()
};
match locations {
Some(locs) if !locs.is_empty() => {
let formatted: Vec<String> = locs
.iter()
.take(LOCATION_HISTORY_DISPLAY_LIMIT)
let formatted: Vec<String> = spread_evenly(&locs, LOCATION_HISTORY_DISPLAY_LIMIT)
.into_iter()
.map(|loc| {
let dt = DateTime::from_timestamp(loc.timestamp, 0)
.map(|dt| {
dt.with_timezone(&Local)
.format("%Y-%m-%d %H:%M")
.to_string()
})
.map(|dt| dt.with_timezone(&zone).format("%Y-%m-%d %H:%M").to_string())
.unwrap_or_else(|| "unknown".to_string());
let activity = loc
.activity
@@ -2403,14 +2558,18 @@ Return ONLY the summary, nothing else."#,
)
})
.collect();
format!(
"{}\n{}",
Self::found_header(locs.len(), formatted.len(), "location records"),
formatted.join("\n")
)
let header = if formatted.len() < locs.len() {
format!(
"Found {} location records (showing {}, evenly spaced across the window):",
locs.len(),
formatted.len()
)
} else {
Self::found_header(locs.len(), formatted.len(), "location records")
};
format!("{}\n{}", header, formatted.join("\n"))
}
Some(_) => "No location history found.".to_string(),
None => "No location history found.".to_string(),
_ => "No location history found.".to_string(),
}
}
@@ -3406,15 +3565,16 @@ Return ONLY the summary, nothing else."#,
if opts.location_history_present {
tools.push(Tool::function(
"get_location_history",
"Fetch raw location records (lat/lon/timestamp/activity) near a date. The default 14-day radius is \
wide because location density varies; tighten to ±1 day for a single-trip query. For a coordinate's \
named place, use `reverse_geocode` (or `get_personal_place_at` when Apollo is enabled).",
"Where the user was on a date: a timeline of named places (Home, Work, …) with arrival/departure \
times, stops, travel with distances, and gaps with no data — or, without Apollo, raw location \
records spread across the day. Times are local. For a coordinate's named place, use \
`get_personal_place_at` (Apollo) or `reverse_geocode`.",
serde_json::json!({
"type": "object",
"required": ["date"],
"properties": {
"date": { "type": "string", "description": "Center date, YYYY-MM-DD." },
"days_radius": { "type": "integer", "description": "Days before and after to include (default 14)." }
"date": { "type": "string", "description": "The day, YYYY-MM-DD (local)." },
"days_radius": { "type": "integer", "description": "Also include this many days before and after (default 0; at most 1 with Apollo)." }
}
}),
));
@@ -4948,6 +5108,85 @@ mod tests {
);
}
fn apollo_at(json: &str) -> LocationAt {
serde_json::from_str(json).unwrap()
}
const NY: chrono_tz::Tz = chrono_tz::Tz::America__New_York;
#[test]
fn apollo_location_no_sample_says_nothing() {
let at = apollo_at(r#"{"ts": 0, "sample": null, "nearest_offset_s": 7200}"#);
assert_eq!(
render_apollo_location(&at, None, NY, Some("Medina, OH".into())),
None
);
}
#[test]
fn apollo_location_renders_stay_in_display_zone() {
// 2017-01-17 08:07 / 16:19 EST
let at = apollo_at(
r#"{"ts": 1484669220,
"sample": {"ts": 1484669240, "lat": 41.469, "lon": -81.934, "activity": "TILTING", "offset_s": 20},
"place": {"id": 7, "name": "Hyland Software", "radius_m": 250},
"stay": {"place_id": 7, "name": "Hyland Software", "t0": 1484658450, "t1": 1484687985}}"#,
);
assert_eq!(
render_apollo_location(&at, None, NY, None).as_deref(),
Some("Location history: You were at Hyland Software (08:07–16:19)")
);
}
#[test]
fn apollo_location_unnamed_uses_nominatim_activity_and_offset() {
let at = apollo_at(
r#"{"ts": 1000,
"sample": {"ts": 100, "lat": 41.1, "lon": -81.8, "activity": "IN_VEHICLE", "offset_s": -900}}"#,
);
assert_eq!(
render_apollo_location(&at, None, NY, Some("Medina, OH".into())).as_deref(),
Some(
"Location history: You were near Medina, OH (IN_VEHICLE) [location fix 15 min before the photo]"
)
);
assert_eq!(
render_apollo_location(&at, None, NY, None).as_deref(),
Some(
"Location history: You were near 41.1000, -81.8000 (IN_VEHICLE) [location fix 15 min before the photo]"
)
);
}
#[test]
fn apollo_location_skips_what_exif_already_says() {
let plain = apollo_at(
r#"{"ts": 0, "sample": {"ts": 0, "lat": 41.1, "lon": -81.8, "activity": "UNKNOWN", "offset_s": 0},
"place": {"id": 1, "name": "Park", "radius_m": 300}}"#,
);
// EXIF 100 m away, no stay, no informative activity → nothing new.
assert_eq!(
render_apollo_location(&plain, Some((41.1009, -81.8)), NY, None),
None
);
// Without EXIF the same answer is worth a line.
assert_eq!(
render_apollo_location(&plain, None, NY, None).as_deref(),
Some("Location history: You were at Park")
);
}
#[test]
fn spread_evenly_keeps_ends_and_order() {
let v: Vec<i32> = (0..100).collect();
let picked: Vec<i32> = spread_evenly(&v, 20).into_iter().copied().collect();
assert_eq!(picked.len(), 20);
assert_eq!((picked[0], picked[19]), (0, 99));
assert!(picked.windows(2).all(|w| w[0] < w[1]));
assert_eq!(spread_evenly(&v[..5], 20).len(), 5);
assert_eq!(spread_evenly(&v, 1), vec![&0]);
}
#[test]
fn found_header_labels_truncation_honestly() {
// Truncated: total exceeds what's listed below the header.