fix(chat): budget chat history against the model's served context window

Long insight/persona chats on llama-swap failed with "exceeds the
available context size" even after raising Context in the app:

- llama-server fixes its window at launch (-c) and ignores num_ctx, but
  the history budget trusted the requested num_ctx, so raising it kept
  MORE history than the server could hold. LlmClient::context_window()
  now reports the served window (llama-swap: /upstream/<model>/props,
  cached 10 min) and the budget uses min(requested, served).
- apply_context_budget could only drop tool-call pairs, so plain Q&A
  chats never shrank. It now also drops whole earlier exchanges (keeping
  user/assistant alternation) and, last, the current turn's tool results,
  never the in-progress question.
- The budget only ran once, before the tool loop. ContextFitter now
  fits every LLM call (on a copy; the stored transcript is untouched)
  and, on a context-overflow error, shrinks the budget from the server's
  reported token counts and retries once.

GET /insights/models reports each llama-swap model's context_window when
it's known without loading the model (cached, or currently running), for
the app's Context picker.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/ai/llamacpp.rs

@@ -27,6 +27,7 @@ use opentelemetry::trace::Status;
 use reqwest::Client;
 use serde::Deserialize;
 use serde_json::{Value, json};
+use std::collections::HashMap;
 use std::time::Duration;
 use tracing_opentelemetry::OpenTelemetrySpanExt as _;
 
@@ -41,6 +42,18 @@ const DEFAULT_VISION_MODEL: &str = "vision";
 const DEFAULT_EMBEDDING_MODEL: &str = "embed";
 const DEFAULT_TTS_MODEL: &str = "chatterbox";
 const DEFAULT_REQUEST_TIMEOUT_SECS: u64 = 180;
+/// How long a served-context-window lookup stays cached per model.
+const CONTEXT_WINDOW_CACHE_TTL: Duration = Duration::from_secs(600);
+
+/// `"<swap root>|<model>"` -> (n_ctx, fetched at). Process-wide because
+/// clients are cloned per request.
+fn context_window_cache() -> &'static std::sync::Mutex<HashMap<String, (usize, std::time::Instant)>>
+{
+    static CACHE: std::sync::OnceLock<
+        std::sync::Mutex<HashMap<String, (usize, std::time::Instant)>>,
+    > = std::sync::OnceLock::new();
+    CACHE.get_or_init(Default::default)
+}
 
 /// OpenAI-compatible client targeting a llama-swap proxy in front of one or
 /// more llama-server processes. See the module doc-comment for the slot model.
@@ -145,6 +158,86 @@ impl LlamaCppClient {
     // upstream server directly, reached via llama-swap's passthrough at
     // `{swap_root}/upstream/<tts_model>/voices`.
 
+    /// Per-request context window `model` is actually served with, read from
+    /// llama-server's `/props` (`default_generation_settings.n_ctx`, the
+    /// per-slot window) through llama-swap's passthrough. That request loads
+    /// the model if it isn't already, which is fine: callers ask right before
+    /// chatting with it. Successful lookups are cached for
+    /// `CONTEXT_WINDOW_CACHE_TTL` so a config.yaml edit is picked up without
+    /// a restart; failures aren't cached and return `None`.
+    pub async fn served_context_window(&self, model: &str) -> Option<usize> {
+        if let Some(n_ctx) = self.cached_context_window(model) {
+            return Some(n_ctx);
+        }
+        let key = format!("{}|{}", self.swap_root(), model);
+        let url = format!("{}/upstream/{}/props", self.swap_root(), model);
+        let props: Value = match self.client.get(&url).send().await {
+            Ok(resp) if resp.status().is_success() => resp.json().await.ok()?,
+            Ok(resp) => {
+                tracing::warn!(model, status = %resp.status(), "llama-swap props lookup failed");
+                return None;
+            }
+            Err(e) => {
+                tracing::warn!(model, error = %e, "llama-swap props lookup failed");
+                return None;
+            }
+        };
+        let n_ctx = props
+            .pointer("/default_generation_settings/n_ctx")
+            .or_else(|| props.get("n_ctx"))
+            .and_then(Value::as_u64)
+            .filter(|n| *n > 0)? as usize;
+        if let Ok(mut cache) = context_window_cache().lock() {
+            cache.insert(key, (n_ctx, std::time::Instant::now()));
+        }
+        Some(n_ctx)
+    }
+
+    /// Served context windows for `models`, for display, WITHOUT loading
+    /// anything: a model counts only if its window is cached from an
+    /// earlier lookup or llama-swap reports it as currently loaded (then
+    /// `/props` is answered by the running process). Others are absent.
+    pub async fn known_context_windows(&self, models: &[String]) -> HashMap<String, usize> {
+        let loaded: std::collections::HashSet<String> = async {
+            let url = format!("{}/models", self.base_url);
+            let parsed: Value = self.client.get(&url).send().await.ok()?.json().await.ok()?;
+            Some(
+                parsed
+                    .get("data")?
+                    .as_array()?
+                    .iter()
+                    .filter(|m| {
+                        m.pointer("/status/value").and_then(Value::as_str) == Some("loaded")
+                    })
+                    .filter_map(|m| m.get("id")?.as_str().map(str::to_string))
+                    .collect(),
+            )
+        }
+        .await
+        .unwrap_or_default();
+
+        let mut out = HashMap::new();
+        for model in models {
+            let cached = self.cached_context_window(model);
+            let n_ctx = match cached {
+                Some(n) => Some(n),
+                None if loaded.contains(model) => self.served_context_window(model).await,
+                None => None,
+            };
+            if let Some(n) = n_ctx {
+                out.insert(model.clone(), n);
+            }
+        }
+        out
+    }
+
+    fn cached_context_window(&self, model: &str) -> Option<usize> {
+        let key = format!("{}|{}", self.swap_root(), model);
+        let cache = context_window_cache().lock().ok()?;
+        let &(n_ctx, at) = cache.get(&key)?;
+        (at.elapsed() < CONTEXT_WINDOW_CACHE_TTL).then_some(n_ctx)
+    }
+
     /// Root of the llama-swap proxy: `base_url` with a trailing `/v1` removed.
     /// The `/upstream/...` passthrough lives here, not under `/v1`.
     fn swap_root(&self) -> &str {
@@ -472,7 +565,8 @@ impl LlamaCppClient {
         }
         // num_ctx isn't an OpenAI param; llama-server bakes ctx in at launch
         // via -c, so we silently drop the override here. The config.yaml
-        // entry is the source of truth for context size.
+        // entry is the source of truth for context size; chat budgets read
+        // it back via `context_window()`.
         let _ = self.num_ctx;
         // Reasoning toggle for thinking-capable templates (Qwen3 et al.).
         // llama-server forwards chat_template_kwargs into the Jinja render
@@ -939,6 +1033,13 @@ impl LlmClient for LlamaCppClient {
     fn primary_model(&self) -> &str {
         &self.primary_model
     }
+
+    /// llama-server fixes its window at launch (`-c`), so ask it via
+    /// llama-swap's `/upstream/<model>/props`. Cached per model; see
+    /// [`LlamaCppClient::served_context_window`].
+    async fn context_window(&self) -> Option<usize> {
+        self.served_context_window(&self.primary_model).await
+    }
 }
 
 impl LlamaCppClient {
@@ -955,6 +1056,7 @@ impl LlamaCppClient {
             name,
             has_vision: true,
             has_tool_calling: true,
+            context_window: None,
         }
     }
 }
@@ -1113,6 +1215,52 @@ fn image_to_data_url(img: &str) -> String {
 mod tests {
     use super::*;
 
+    /// One-shot HTTP server answering any request with `body`; returns its
+    /// base URL and a handle yielding the request line it received.
+    async fn serve_once(body: &'static str) -> (String, tokio::task::JoinHandle<String>) {
+        use tokio::io::{AsyncReadExt, AsyncWriteExt};
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let addr = listener.local_addr().unwrap();
+        let handle = tokio::spawn(async move {
+            let (mut sock, _) = listener.accept().await.unwrap();
+            let mut buf = vec![0u8; 4096];
+            let n = sock.read(&mut buf).await.unwrap();
+            let req = String::from_utf8_lossy(&buf[..n])
+                .lines()
+                .next()
+                .unwrap_or("")
+                .to_string();
+            let resp = format!(
+                "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
+                body.len(),
+                body
+            );
+            sock.write_all(resp.as_bytes()).await.unwrap();
+            req
+        });
+        (format!("http://{addr}/v1"), handle)
+    }
+
+    #[tokio::test]
+    async fn served_context_window_reads_props_through_upstream() {
+        let (base, req) =
+            serve_once(r#"{"default_generation_settings":{"n_ctx":32768},"total_slots":1}"#).await;
+        let client = LlamaCppClient::new(Some(base), Some("gemma4-31b-qat-uncensored".into()));
+        assert_eq!(client.context_window().await, Some(32768));
+        assert_eq!(
+            req.await.unwrap(),
+            "GET /upstream/gemma4-31b-qat-uncensored/props HTTP/1.1"
+        );
+        // Cached: a second lookup makes no request (the server is gone).
+        assert_eq!(client.context_window().await, Some(32768));
+    }
+
+    #[tokio::test]
+    async fn served_context_window_is_none_when_unreachable() {
+        let client = LlamaCppClient::new(Some("http://127.0.0.1:9/v1".into()), Some("nope".into()));
+        assert_eq!(client.context_window().await, None);
+    }
+
     #[test]
     fn tool_call_arguments_stringified_on_send() {
         let msg = ChatMessage {