From e10fc26ca7f0b7b03442d6751e6a0395af49ac74 Mon Sep 17 00:00:00 2001 From: Cameron Cordes Date: Mon, 26 Oct 2026 17:34:28 -0400 Subject: [PATCH] feat(ai): discover the llama-swap picker and vision from model metadata /insights/models now lists llama-swap's models live from /v1/models instead of LLAMA_SWAP_ALLOWED_MODELS: every kind=chat model, with setParamsByID aliases (:sub / :fast) hidden, filtered by [llamacpp.picker] exclude and the optional include allowlist. Ids the include list names but llama-swap no longer serves drop out instead of showing as dead picker entries. has_vision and context_window come from each model's config.yaml metadata (read from meta.llamaswap, where the deployed llama-swap merges it, or from meta as upstream documents). A model without metadata is text-only chat. A live served window still wins over the configured ctx. Agentic insights on llama-swap now ask the chat model's capabilities: text-only models (e.g. qwen3.8-27b-code, --no-mmproj) get the local vision model's description inlined, like hybrid, instead of an image they can't read. If llama-swap can't be asked, images are sent as before. Drops AppState.llamacpp_allowed_models. Co-Authored-By: Claude Opus 5.5 --- src/ai/handlers.rs | 52 +++++---- src/ai/insight_generator.rs | 23 +++- src/ai/llamacpp.rs | 227 +++++++++++++++++++++++++++--------- src/state.rs | 9 -- 4 files changed, 219 insertions(+), 92 deletions(-) diff --git a/src/ai/handlers.rs b/src/ai/handlers.rs index b3a5b53..d587be6 100644 --- a/src/ai/handlers.rs +++ b/src/ai/handlers.rs @@ -983,17 +983,15 @@ pub async fn generate_agentic_insight_handler( } /// GET /insights/models - Local-backend models with capabilities. Returns -/// Ollama servers when `LLM_BACKEND=ollama` (default), or llama-swap slots -/// when `LLM_BACKEND=llamacpp`. Same envelope shape either way so the +/// Ollama servers when `backend = "ollama"` (default), or llama-swap models +/// when `backend = "llamacpp"`. Same envelope shape either way so the /// client picker doesn't have to branch on backend kind. /// -/// For llama-swap: `models` comes verbatim from `LLAMA_SWAP_ALLOWED_MODELS` -/// (no live `/v1/models` probe), `has_vision` is true only for the -/// configured `LLAMA_SWAP_VISION_MODEL` slot id, and `has_tool_calling` is -/// reported as true for every slot (llama-server is launched with `--jinja` -/// by convention — a misconfigured slot surfaces as a chat-call error). -/// `context_window` is each slot's served `-c` window when known without -/// loading the model (cached from a recent chat, or the model is running). +/// For llama-swap: `models` is discovered live from `/v1/models` — every +/// `kind: chat` model (aliases hidden) filtered by `[llamacpp.picker]` — +/// with `has_vision` / `context_window` from each model's config.yaml +/// `metadata`. A served window known without loading the model (cached from +/// a recent chat, or the model is running) takes precedence over `ctx`. #[get("/insights/models")] pub async fn get_available_models_handler( _claims: Claims, @@ -1004,20 +1002,30 @@ pub async fn get_available_models_handler( if crate::ai::local_backend_is_llamacpp() && let Some(lc) = app_state.llamacpp.as_ref() { + let picker = &crate::ai::config::llm_config().llamacpp.picker; + let mut models = match lc.discover_models().await { + Ok(found) => crate::ai::llamacpp::picker_models(found, picker), + Err(e) => { + // llama-swap is down, so chat would fail anyway; still list + // the configured model so the picker isn't empty when it + // comes back. + log::warn!("llama-swap model discovery failed: {e:#}"); + vec![ModelCapabilities { + name: lc.primary_model.clone(), + has_vision: true, + has_tool_calling: true, + context_window: None, + }] + } + }; // Served windows for the context picker; never loads a model. - let windows = lc - .known_context_windows(&app_state.llamacpp_allowed_models) - .await; - let models: Vec = app_state - .llamacpp_allowed_models - .iter() - .map(|name| ModelCapabilities { - name: name.clone(), - has_vision: true, - has_tool_calling: true, - context_window: windows.get(name).copied(), - }) - .collect(); + let names: Vec = models.iter().map(|m| m.name.clone()).collect(); + let windows = lc.known_context_windows(&names).await; + for m in &mut models { + if let Some(&n) = windows.get(&m.name) { + m.context_window = Some(n); + } + } let primary = ServerModels { url: lc.base_url.clone(), models, diff --git a/src/ai/insight_generator.rs b/src/ai/insight_generator.rs index 1792a03..eb95370 100644 --- a/src/ai/insight_generator.rs +++ b/src/ai/insight_generator.rs @@ -4447,7 +4447,7 @@ Return ONLY the summary, nothing else."#, } else if local_via_llamacpp { // Local via llama-swap. let arc = self.llamacpp.as_ref().ok_or_else(|| { - anyhow::anyhow!("LLM_BACKEND=llamacpp but LLAMA_SWAP_URL not configured") + anyhow::anyhow!("backend = \"llamacpp\" but llamacpp.url not configured") })?; let mut c: LlamaCppClient = (**arc).clone(); if let Some(ref m) = overrides.model { @@ -4515,11 +4515,22 @@ Return ONLY the summary, nothing else."#, // Hybrid: chat model never sees images — describe-then-inject. false } else if local_via_llamacpp { - // llama-swap models receive images directly via OpenAI content - // parts. Capability probing isn't available (no `/api/show`), - // so assume vision support; a misconfigured model surfaces as - // a chat-call error. - true + // Vision models get images directly via OpenAI content parts; + // text-only ones (metadata `vision: false`) get the local vision + // model's description inlined instead, like hybrid. If llama-swap + // can't be asked, assume vision — the chat call will say why. + let lc = self + .llamacpp + .as_deref() + .expect("checked by local_via_llamacpp"); + let model = overrides.model.as_deref().unwrap_or(&lc.primary_model); + match lc.model_capabilities(model).await { + Ok(caps) => caps.has_vision, + Err(e) => { + log::warn!("vision capability unknown for '{model}', sending images: {e:#}"); + true + } + } } else { // Pure Ollama: probe model capabilities. let ollama_for_caps = if let Some(ref model) = overrides.model { diff --git a/src/ai/llamacpp.rs b/src/ai/llamacpp.rs index 07d1afa..e6f135d 100644 --- a/src/ai/llamacpp.rs +++ b/src/ai/llamacpp.rs @@ -10,11 +10,11 @@ // `generate_embeddings` issue requests with the appropriate slot id in the // `model` field, which is how llama-swap selects which backend process to // run. -// - `/v1/models` returns only the configured slot ids — capabilities aren't -// reported by the API, so we infer `has_vision` from a single config field -// (`vision_model`, defaulting to `"vision"`) and assume `has_tool_calling` -// is true for every slot, since llama-swap entries default to launching -// llama-server with `--jinja`. +// - Capabilities come from each model's `metadata:` block in llama-swap's +// config.yaml, which `/v1/models` passes through (`kind`, `vision`, `ctx`; +// see `DiscoveredModel`). A model without metadata is treated as a +// text-only chat model. `has_tool_calling` is assumed true for every model, +// since llama-swap entries launch llama-server with `--jinja`. // // First consumer lands alongside the three-way backend dispatch in // insight_generator / insight_chat. @@ -31,6 +31,7 @@ use std::collections::HashMap; use std::time::Duration; use tracing_opentelemetry::OpenTelemetrySpanExt as _; +use crate::ai::config::PickerConfig; use crate::ai::llm_client::{ ChatMessage, LlmClient, LlmStreamEvent, ModelCapabilities, Tool, ToolCall, ToolCallFunction, }; @@ -987,32 +988,12 @@ impl LlmClient for LlamaCppClient { } async fn list_models(&self) -> Result> { - let url = format!("{}/models", self.base_url); - let resp = self - .client - .get(&url) - .send() - .await - .with_context(|| format!("GET {} failed", url))?; - - if !resp.status().is_success() { - let status = resp.status(); - let body = resp.text().await.unwrap_or_default(); - bail!("llama-swap list_models failed: {} — {}", status, body); - } - - let parsed: Value = resp.json().await.context("parsing models response")?; - let data = parsed - .get("data") - .and_then(|v| v.as_array()) - .ok_or_else(|| anyhow!("models response missing data[]"))?; - - let caps: Vec = data - .iter() - .map(|m| self.parse_model_capabilities(m)) - .collect(); - - Ok(caps) + Ok(self + .discover_models() + .await? + .into_iter() + .map(|m| m.caps) + .collect()) } async fn model_capabilities(&self, model: &str) -> Result { @@ -1034,25 +1015,97 @@ impl LlmClient for LlamaCppClient { } } -impl LlamaCppClient { - fn parse_model_capabilities(&self, m: &Value) -> ModelCapabilities { +/// One entry of llama-swap's `/v1/models`, with the per-model `metadata` +/// from config.yaml decoded. +#[derive(Debug, Clone)] +pub struct DiscoveredModel { + pub caps: ModelCapabilities, + /// `metadata.kind`: `chat` | `embed` | `tts`. `None` when the model has + /// no metadata (treated as chat). + pub kind: Option, + /// A `setParamsByID` alias like `qwen3.8-27b-code:sub` — a parameter + /// profile on another model, not a model of its own. + pub is_alias: bool, +} + +impl DiscoveredModel { + pub fn is_chat(&self) -> bool { + matches!(self.kind.as_deref(), None | Some("chat")) + } + + /// Parse one `/v1/models` row. The deployed llama-swap merges config + /// `metadata` into `meta.llamaswap` (next to its own `type` / `aliases`); + /// upstream docs describe it at `meta` itself, so both are read. + fn from_json(m: &Value) -> Self { + let field = |key: &str| { + m.pointer(&format!("/meta/llamaswap/{key}")) + .or_else(|| m.pointer(&format!("/meta/{key}"))) + }; let name = m .get("id") - .and_then(|v| v.as_str()) + .and_then(Value::as_str) .unwrap_or_default() .to_string(); - // llama-swap doesn't expose per-model modality flags. Assume all - // configured models support vision and tool calling; a model - // without multimodal support surfaces as a chat-call error. - ModelCapabilities { - name, - has_vision: true, - has_tool_calling: true, - context_window: None, + Self { + caps: ModelCapabilities { + name, + has_vision: field("vision").and_then(Value::as_bool).unwrap_or(false), + has_tool_calling: true, + context_window: field("ctx").and_then(Value::as_u64).map(|n| n as usize), + }, + kind: field("kind").and_then(Value::as_str).map(str::to_string), + is_alias: field("type").and_then(Value::as_str) == Some("alias"), } } } +/// The `/insights/models` picker: every non-alias chat model, minus +/// `picker.exclude`, restricted to `picker.include` when that's set. +/// llama-swap's order is kept. +pub fn picker_models( + models: Vec, + picker: &PickerConfig, +) -> Vec { + models + .into_iter() + .filter(|m| !m.is_alias && m.is_chat()) + .filter(|m| !picker.exclude.contains(&m.caps.name)) + .filter(|m| { + picker + .include + .as_ref() + .is_none_or(|inc| inc.contains(&m.caps.name)) + }) + .map(|m| m.caps) + .collect() +} + +impl LlamaCppClient { + /// Every model llama-swap serves, aliases included, with metadata. + pub async fn discover_models(&self) -> Result> { + let url = format!("{}/models", self.base_url); + let resp = self + .client + .get(&url) + .send() + .await + .with_context(|| format!("GET {} failed", url))?; + + if !resp.status().is_success() { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + bail!("llama-swap list_models failed: {} — {}", status, body); + } + + let parsed: Value = resp.json().await.context("parsing models response")?; + let data = parsed + .get("data") + .and_then(|v| v.as_array()) + .ok_or_else(|| anyhow!("models response missing data[]"))?; + Ok(data.iter().map(DiscoveredModel::from_json).collect()) + } +} + /// Extract a diagnostic fragment from a llama-swap / llama-server response /// that doesn't match the expected `{choices: [...]}` shape. llama-server /// returns errors as `{"error": {"message": "...", "code": N, "type": "..."}}`; @@ -1440,22 +1493,86 @@ mod tests { } #[test] - fn all_models_report_vision_capable() { - let mut c = LlamaCppClient::new(None, Some("chat".into())); - c.set_vision_model("vision".into()); + fn metadata_drives_capabilities() { + // Shape of the deployed llama-swap: config metadata merged into + // meta.llamaswap. + let m = DiscoveredModel::from_json(&json!({ + "id": "qwen3.8-27b-code-vision", + "meta": { "llamaswap": { + "type": "model", "kind": "chat", "vision": true, "ctx": 90000 + }} + })); + assert_eq!(m.caps.name, "qwen3.8-27b-code-vision"); + assert!(m.caps.has_vision); + assert!(m.caps.has_tool_calling); + assert_eq!(m.caps.context_window, Some(90000)); + assert!(m.is_chat() && !m.is_alias); + } - let m_chat = json!({ "id": "chat" }); - let m_vision = json!({ "id": "vision" }); - let m_other = json!({ "id": "embed" }); + #[test] + fn top_level_meta_is_read_too() { + // Upstream docs put metadata at `meta` itself. + let m = DiscoveredModel::from_json(&json!({ + "id": "embed", "meta": { "kind": "embed", "dim": 768 } + })); + assert_eq!(m.kind.as_deref(), Some("embed")); + assert!(!m.is_chat()); + } - let chat = c.parse_model_capabilities(&m_chat); - let vision = c.parse_model_capabilities(&m_vision); - let other = c.parse_model_capabilities(&m_other); + #[test] + fn model_without_metadata_is_text_only_chat() { + let m = DiscoveredModel::from_json(&json!({ "id": "mystery" })); + assert!(m.is_chat()); + assert!(!m.caps.has_vision); + assert_eq!(m.caps.context_window, None); + } - assert!(chat.has_vision); - assert!(chat.has_tool_calling); - assert!(vision.has_vision); - assert!(other.has_vision); + fn row(id: &str, kind: &str, kind_type: &str) -> DiscoveredModel { + DiscoveredModel::from_json(&json!({ + "id": id, "meta": { "llamaswap": { "type": kind_type, "kind": kind } } + })) + } + + fn names(models: Vec) -> Vec { + models.into_iter().map(|m| m.name).collect() + } + + #[test] + fn picker_shows_chat_models_only_and_hides_aliases() { + let models = vec![ + row("chatterbox", "tts", "model"), + row("embed", "embed", "model"), + row("q-code", "chat", "model"), + row("q-code:sub", "chat", "alias"), + row("gemma", "chat", "model"), + ]; + assert_eq!( + names(picker_models(models, &PickerConfig::default())), + ["q-code", "gemma"] + ); + } + + #[test] + fn picker_applies_exclude_and_include() { + let models = || { + vec![ + row("a", "chat", "model"), + row("b", "chat", "model"), + row("c", "chat", "model"), + ] + }; + let exclude = PickerConfig { + include: None, + exclude: vec!["b".into()], + }; + assert_eq!(names(picker_models(models(), &exclude)), ["a", "c"]); + // include keeps llama-swap's order and silently drops ids it doesn't + // serve (the stale-allowlist case). + let include = PickerConfig { + include: Some(vec!["c".into(), "a".into(), "gone".into()]), + exclude: vec!["a".into()], + }; + assert_eq!(names(picker_models(models(), &include)), ["c"]); } #[test] diff --git a/src/state.rs b/src/state.rs index da6b997..a5da3f1 100644 --- a/src/state.rs +++ b/src/state.rs @@ -78,10 +78,6 @@ pub struct AppState { /// threading through the generator. #[allow(dead_code)] pub llamacpp: Option>, - /// Curated list of llama-swap model ids exposed to clients. Sourced from - /// `LLAMA_SWAP_ALLOWED_MODELS` (comma-separated). Empty when unset; the - /// server then falls back to `LLAMA_SWAP_PRIMARY_MODEL`. - pub llamacpp_allowed_models: Vec, pub sms_client: SmsApiClient, pub insight_generator: InsightGenerator, /// Chat continuation service. Hold an Arc so handlers can clone cheaply. @@ -134,7 +130,6 @@ impl AppState { openrouter: Option>, openrouter_allowed_models: Vec, llamacpp: Option>, - llamacpp_allowed_models: Vec, sms_client: SmsApiClient, insight_generator: InsightGenerator, insight_chat: Arc, @@ -196,7 +191,6 @@ impl AppState { openrouter, openrouter_allowed_models, llamacpp, - llamacpp_allowed_models, sms_client, insight_generator, insight_chat, @@ -232,7 +226,6 @@ impl Default for AppState { let openrouter_allowed_models = llm.openrouter.allowed.clone(); let llamacpp = build_llamacpp(llm); - let llamacpp_allowed_models = llm.llamacpp.picker.include.clone().unwrap_or_default(); let sms_api_url = env::var("SMS_API_URL").unwrap_or_else(|_| "http://localhost:8000".to_string()); @@ -369,7 +362,6 @@ impl Default for AppState { openrouter, openrouter_allowed_models, llamacpp, - llamacpp_allowed_models, sms_client, insight_generator, insight_chat, @@ -548,7 +540,6 @@ impl AppState { None, Vec::new(), None, - Vec::new(), sms_client, insight_generator, insight_chat,