feat(ai): discover the llama-swap picker and vision from model metadata

/insights/models now lists llama-swap's models live from /v1/models
instead of LLAMA_SWAP_ALLOWED_MODELS: every kind=chat model, with
setParamsByID aliases (:sub / :fast) hidden, filtered by
[llamacpp.picker] exclude and the optional include allowlist. Ids the
include list names but llama-swap no longer serves drop out instead of
showing as dead picker entries.

has_vision and context_window come from each model's config.yaml metadata
(read from meta.llamaswap, where the deployed llama-swap merges it, or from
meta as upstream documents). A model without metadata is text-only chat.
A live served window still wins over the configured ctx.

Agentic insights on llama-swap now ask the chat model's capabilities:
text-only models (e.g. qwen3.8-27b-code, --no-mmproj) get the local vision
model's description inlined, like hybrid, instead of an image they can't
read. If llama-swap can't be asked, images are sent as before.

Drops AppState.llamacpp_allowed_models.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/ai/handlers.rs

@@ -983,17 +983,15 @@ pub async fn generate_agentic_insight_handler(
 }
 
 /// GET /insights/models - Local-backend models with capabilities. Returns
-/// Ollama servers when `LLM_BACKEND=ollama` (default), or llama-swap slots
+/// Ollama servers when `backend = "ollama"` (default), or llama-swap models
-/// when `LLM_BACKEND=llamacpp`. Same envelope shape either way so the
+/// when `backend = "llamacpp"`. Same envelope shape either way so the
 /// client picker doesn't have to branch on backend kind.
 ///
-/// For llama-swap: `models` comes verbatim from `LLAMA_SWAP_ALLOWED_MODELS`
+/// For llama-swap: `models` is discovered live from `/v1/models` — every
-/// (no live `/v1/models` probe), `has_vision` is true only for the
+/// `kind: chat` model (aliases hidden) filtered by `[llamacpp.picker]` —
-/// configured `LLAMA_SWAP_VISION_MODEL` slot id, and `has_tool_calling` is
+/// with `has_vision` / `context_window` from each model's config.yaml
-/// reported as true for every slot (llama-server is launched with `--jinja`
+/// `metadata`. A served window known without loading the model (cached from
-/// by convention — a misconfigured slot surfaces as a chat-call error).
+/// a recent chat, or the model is running) takes precedence over `ctx`.
-/// `context_window` is each slot's served `-c` window when known without
-/// loading the model (cached from a recent chat, or the model is running).
 #[get("/insights/models")]
 pub async fn get_available_models_handler(
     _claims: Claims,
@@ -1004,20 +1002,30 @@ pub async fn get_available_models_handler(
     if crate::ai::local_backend_is_llamacpp()
         && let Some(lc) = app_state.llamacpp.as_ref()
     {
+        let picker = &crate::ai::config::llm_config().llamacpp.picker;
+        let mut models = match lc.discover_models().await {
+            Ok(found) => crate::ai::llamacpp::picker_models(found, picker),
+            Err(e) => {
+                // llama-swap is down, so chat would fail anyway; still list
+                // the configured model so the picker isn't empty when it
+                // comes back.
+                log::warn!("llama-swap model discovery failed: {e:#}");
+                vec![ModelCapabilities {
+                    name: lc.primary_model.clone(),
+                    has_vision: true,
+                    has_tool_calling: true,
+                    context_window: None,
+                }]
+            }
+        };
         // Served windows for the context picker; never loads a model.
-        let windows = lc
+        let names: Vec<String> = models.iter().map(|m| m.name.clone()).collect();
-            .known_context_windows(&app_state.llamacpp_allowed_models)
+        let windows = lc.known_context_windows(&names).await;
-            .await;
+        for m in &mut models {
-        let models: Vec<ModelCapabilities> = app_state
+            if let Some(&n) = windows.get(&m.name) {
-            .llamacpp_allowed_models
+                m.context_window = Some(n);
-            .iter()
+            }
-            .map(|name| ModelCapabilities {
+        }
-                name: name.clone(),
-                has_vision: true,
-                has_tool_calling: true,
-                context_window: windows.get(name).copied(),
-            })
-            .collect();
         let primary = ServerModels {
             url: lc.base_url.clone(),
             models,

src/ai/insight_generator.rs

@@ -4447,7 +4447,7 @@ Return ONLY the summary, nothing else."#,
         } else if local_via_llamacpp {
             // Local via llama-swap.
             let arc = self.llamacpp.as_ref().ok_or_else(|| {
-                anyhow::anyhow!("LLM_BACKEND=llamacpp but LLAMA_SWAP_URL not configured")
+                anyhow::anyhow!("backend = \"llamacpp\" but llamacpp.url not configured")
             })?;
             let mut c: LlamaCppClient = (**arc).clone();
             if let Some(ref m) = overrides.model {
@@ -4515,11 +4515,22 @@ Return ONLY the summary, nothing else."#,
             // Hybrid: chat model never sees images — describe-then-inject.
             false
         } else if local_via_llamacpp {
-            // llama-swap models receive images directly via OpenAI content
+            // Vision models get images directly via OpenAI content parts;
-            // parts. Capability probing isn't available (no `/api/show`),
+            // text-only ones (metadata `vision: false`) get the local vision
-            // so assume vision support; a misconfigured model surfaces as
+            // model's description inlined instead, like hybrid. If llama-swap
-            // a chat-call error.
+            // can't be asked, assume vision — the chat call will say why.
-            true
+            let lc = self
+                .llamacpp
+                .as_deref()
+                .expect("checked by local_via_llamacpp");
+            let model = overrides.model.as_deref().unwrap_or(&lc.primary_model);
+            match lc.model_capabilities(model).await {
+                Ok(caps) => caps.has_vision,
+                Err(e) => {
+                    log::warn!("vision capability unknown for '{model}', sending images: {e:#}");
+                    true
+                }
+            }
         } else {
             // Pure Ollama: probe model capabilities.
             let ollama_for_caps = if let Some(ref model) = overrides.model {

src/ai/llamacpp.rs

@@ -10,11 +10,11 @@
 //   `generate_embeddings` issue requests with the appropriate slot id in the
 //   `model` field, which is how llama-swap selects which backend process to
 //   run.
-// - `/v1/models` returns only the configured slot ids — capabilities aren't
+// - Capabilities come from each model's `metadata:` block in llama-swap's
-//   reported by the API, so we infer `has_vision` from a single config field
+//   config.yaml, which `/v1/models` passes through (`kind`, `vision`, `ctx`;
-//   (`vision_model`, defaulting to `"vision"`) and assume `has_tool_calling`
+//   see `DiscoveredModel`). A model without metadata is treated as a
-//   is true for every slot, since llama-swap entries default to launching
+//   text-only chat model. `has_tool_calling` is assumed true for every model,
-//   llama-server with `--jinja`.
+//   since llama-swap entries launch llama-server with `--jinja`.
 //
 // First consumer lands alongside the three-way backend dispatch in
 // insight_generator / insight_chat.
@@ -31,6 +31,7 @@ use std::collections::HashMap;
 use std::time::Duration;
 use tracing_opentelemetry::OpenTelemetrySpanExt as _;
 
+use crate::ai::config::PickerConfig;
 use crate::ai::llm_client::{
     ChatMessage, LlmClient, LlmStreamEvent, ModelCapabilities, Tool, ToolCall, ToolCallFunction,
 };
@@ -987,32 +988,12 @@ impl LlmClient for LlamaCppClient {
     }
 
     async fn list_models(&self) -> Result<Vec<ModelCapabilities>> {
-        let url = format!("{}/models", self.base_url);
+        Ok(self
-        let resp = self
+            .discover_models()
-            .client
+            .await?
-            .get(&url)
+            .into_iter()
-            .send()
+            .map(|m| m.caps)
-            .await
+            .collect())
-            .with_context(|| format!("GET {} failed", url))?;
-
-        if !resp.status().is_success() {
-            let status = resp.status();
-            let body = resp.text().await.unwrap_or_default();
-            bail!("llama-swap list_models failed: {} — {}", status, body);
-        }
-
-        let parsed: Value = resp.json().await.context("parsing models response")?;
-        let data = parsed
-            .get("data")
-            .and_then(|v| v.as_array())
-            .ok_or_else(|| anyhow!("models response missing data[]"))?;
-
-        let caps: Vec<ModelCapabilities> = data
-            .iter()
-            .map(|m| self.parse_model_capabilities(m))
-            .collect();
-
-        Ok(caps)
     }
 
     async fn model_capabilities(&self, model: &str) -> Result<ModelCapabilities> {
@@ -1034,25 +1015,97 @@ impl LlmClient for LlamaCppClient {
     }
 }
 
-impl LlamaCppClient {
+/// One entry of llama-swap's `/v1/models`, with the per-model `metadata`
-    fn parse_model_capabilities(&self, m: &Value) -> ModelCapabilities {
+/// from config.yaml decoded.
+#[derive(Debug, Clone)]
+pub struct DiscoveredModel {
+    pub caps: ModelCapabilities,
+    /// `metadata.kind`: `chat` | `embed` | `tts`. `None` when the model has
+    /// no metadata (treated as chat).
+    pub kind: Option<String>,
+    /// A `setParamsByID` alias like `qwen3.8-27b-code:sub` — a parameter
+    /// profile on another model, not a model of its own.
+    pub is_alias: bool,
+}
+
+impl DiscoveredModel {
+    pub fn is_chat(&self) -> bool {
+        matches!(self.kind.as_deref(), None | Some("chat"))
+    }
+
+    /// Parse one `/v1/models` row. The deployed llama-swap merges config
+    /// `metadata` into `meta.llamaswap` (next to its own `type` / `aliases`);
+    /// upstream docs describe it at `meta` itself, so both are read.
+    fn from_json(m: &Value) -> Self {
+        let field = |key: &str| {
+            m.pointer(&format!("/meta/llamaswap/{key}"))
+                .or_else(|| m.pointer(&format!("/meta/{key}")))
+        };
         let name = m
             .get("id")
-            .and_then(|v| v.as_str())
+            .and_then(Value::as_str)
             .unwrap_or_default()
             .to_string();
-        // llama-swap doesn't expose per-model modality flags. Assume all
+        Self {
-        // configured models support vision and tool calling; a model
+            caps: ModelCapabilities {
-        // without multimodal support surfaces as a chat-call error.
+                name,
-        ModelCapabilities {
+                has_vision: field("vision").and_then(Value::as_bool).unwrap_or(false),
-            name,
+                has_tool_calling: true,
-            has_vision: true,
+                context_window: field("ctx").and_then(Value::as_u64).map(|n| n as usize),
-            has_tool_calling: true,
+            },
-            context_window: None,
+            kind: field("kind").and_then(Value::as_str).map(str::to_string),
+            is_alias: field("type").and_then(Value::as_str) == Some("alias"),
         }
     }
 }
 
+/// The `/insights/models` picker: every non-alias chat model, minus
+/// `picker.exclude`, restricted to `picker.include` when that's set.
+/// llama-swap's order is kept.
+pub fn picker_models(
+    models: Vec<DiscoveredModel>,
+    picker: &PickerConfig,
+) -> Vec<ModelCapabilities> {
+    models
+        .into_iter()
+        .filter(|m| !m.is_alias && m.is_chat())
+        .filter(|m| !picker.exclude.contains(&m.caps.name))
+        .filter(|m| {
+            picker
+                .include
+                .as_ref()
+                .is_none_or(|inc| inc.contains(&m.caps.name))
+        })
+        .map(|m| m.caps)
+        .collect()
+}
+
+impl LlamaCppClient {
+    /// Every model llama-swap serves, aliases included, with metadata.
+    pub async fn discover_models(&self) -> Result<Vec<DiscoveredModel>> {
+        let url = format!("{}/models", self.base_url);
+        let resp = self
+            .client
+            .get(&url)
+            .send()
+            .await
+            .with_context(|| format!("GET {} failed", url))?;
+
+        if !resp.status().is_success() {
+            let status = resp.status();
+            let body = resp.text().await.unwrap_or_default();
+            bail!("llama-swap list_models failed: {} — {}", status, body);
+        }
+
+        let parsed: Value = resp.json().await.context("parsing models response")?;
+        let data = parsed
+            .get("data")
+            .and_then(|v| v.as_array())
+            .ok_or_else(|| anyhow!("models response missing data[]"))?;
+        Ok(data.iter().map(DiscoveredModel::from_json).collect())
+    }
+}
+
 /// Extract a diagnostic fragment from a llama-swap / llama-server response
 /// that doesn't match the expected `{choices: [...]}` shape. llama-server
 /// returns errors as `{"error": {"message": "...", "code": N, "type": "..."}}`;
@@ -1440,22 +1493,86 @@ mod tests {
     }
 
     #[test]
-    fn all_models_report_vision_capable() {
+    fn metadata_drives_capabilities() {
-        let mut c = LlamaCppClient::new(None, Some("chat".into()));
+        // Shape of the deployed llama-swap: config metadata merged into
-        c.set_vision_model("vision".into());
+        // meta.llamaswap.
+        let m = DiscoveredModel::from_json(&json!({
+            "id": "qwen3.8-27b-code-vision",
+            "meta": { "llamaswap": {
+                "type": "model", "kind": "chat", "vision": true, "ctx": 90000
+            }}
+        }));
+        assert_eq!(m.caps.name, "qwen3.8-27b-code-vision");
+        assert!(m.caps.has_vision);
+        assert!(m.caps.has_tool_calling);
+        assert_eq!(m.caps.context_window, Some(90000));
+        assert!(m.is_chat() && !m.is_alias);
+    }
 
-        let m_chat = json!({ "id": "chat" });
+    #[test]
-        let m_vision = json!({ "id": "vision" });
+    fn top_level_meta_is_read_too() {
-        let m_other = json!({ "id": "embed" });
+        // Upstream docs put metadata at `meta` itself.
+        let m = DiscoveredModel::from_json(&json!({
+            "id": "embed", "meta": { "kind": "embed", "dim": 768 }
+        }));
+        assert_eq!(m.kind.as_deref(), Some("embed"));
+        assert!(!m.is_chat());
+    }
 
-        let chat = c.parse_model_capabilities(&m_chat);
+    #[test]
-        let vision = c.parse_model_capabilities(&m_vision);
+    fn model_without_metadata_is_text_only_chat() {
-        let other = c.parse_model_capabilities(&m_other);
+        let m = DiscoveredModel::from_json(&json!({ "id": "mystery" }));
+        assert!(m.is_chat());
+        assert!(!m.caps.has_vision);
+        assert_eq!(m.caps.context_window, None);
+    }
 
-        assert!(chat.has_vision);
+    fn row(id: &str, kind: &str, kind_type: &str) -> DiscoveredModel {
-        assert!(chat.has_tool_calling);
+        DiscoveredModel::from_json(&json!({
-        assert!(vision.has_vision);
+            "id": id, "meta": { "llamaswap": { "type": kind_type, "kind": kind } }
-        assert!(other.has_vision);
+        }))
+    }
+
+    fn names(models: Vec<ModelCapabilities>) -> Vec<String> {
+        models.into_iter().map(|m| m.name).collect()
+    }
+
+    #[test]
+    fn picker_shows_chat_models_only_and_hides_aliases() {
+        let models = vec![
+            row("chatterbox", "tts", "model"),
+            row("embed", "embed", "model"),
+            row("q-code", "chat", "model"),
+            row("q-code:sub", "chat", "alias"),
+            row("gemma", "chat", "model"),
+        ];
+        assert_eq!(
+            names(picker_models(models, &PickerConfig::default())),
+            ["q-code", "gemma"]
+        );
+    }
+
+    #[test]
+    fn picker_applies_exclude_and_include() {
+        let models = || {
+            vec![
+                row("a", "chat", "model"),
+                row("b", "chat", "model"),
+                row("c", "chat", "model"),
+            ]
+        };
+        let exclude = PickerConfig {
+            include: None,
+            exclude: vec!["b".into()],
+        };
+        assert_eq!(names(picker_models(models(), &exclude)), ["a", "c"]);
+        // include keeps llama-swap's order and silently drops ids it doesn't
+        // serve (the stale-allowlist case).
+        let include = PickerConfig {
+            include: Some(vec!["c".into(), "a".into(), "gone".into()]),
+            exclude: vec!["a".into()],
+        };
+        assert_eq!(names(picker_models(models(), &include)), ["c"]);
     }
 
     #[test]

src/state.rs

@@ -78,10 +78,6 @@ pub struct AppState {
     /// threading through the generator.
     #[allow(dead_code)]
     pub llamacpp: Option<Arc<LlamaCppClient>>,
-    /// Curated list of llama-swap model ids exposed to clients. Sourced from
-    /// `LLAMA_SWAP_ALLOWED_MODELS` (comma-separated). Empty when unset; the
-    /// server then falls back to `LLAMA_SWAP_PRIMARY_MODEL`.
-    pub llamacpp_allowed_models: Vec<String>,
     pub sms_client: SmsApiClient,
     pub insight_generator: InsightGenerator,
     /// Chat continuation service. Hold an Arc so handlers can clone cheaply.
@@ -134,7 +130,6 @@ impl AppState {
         openrouter: Option<Arc<OpenRouterClient>>,
         openrouter_allowed_models: Vec<String>,
         llamacpp: Option<Arc<LlamaCppClient>>,
-        llamacpp_allowed_models: Vec<String>,
         sms_client: SmsApiClient,
         insight_generator: InsightGenerator,
         insight_chat: Arc<InsightChatService>,
@@ -196,7 +191,6 @@ impl AppState {
             openrouter,
             openrouter_allowed_models,
             llamacpp,
-            llamacpp_allowed_models,
             sms_client,
             insight_generator,
             insight_chat,
@@ -232,7 +226,6 @@ impl Default for AppState {
         let openrouter_allowed_models = llm.openrouter.allowed.clone();
 
         let llamacpp = build_llamacpp(llm);
-        let llamacpp_allowed_models = llm.llamacpp.picker.include.clone().unwrap_or_default();
 
         let sms_api_url =
             env::var("SMS_API_URL").unwrap_or_else(|_| "http://localhost:8000".to_string());
@@ -369,7 +362,6 @@ impl Default for AppState {
             openrouter,
             openrouter_allowed_models,
             llamacpp,
-            llamacpp_allowed_models,
             sms_client,
             insight_generator,
             insight_chat,
@@ -548,7 +540,6 @@ impl AppState {
             None,
             Vec::new(),
             None,
-            Vec::new(),
             sms_client,
             insight_generator,
             insight_chat,