Apps/ImageApi
1
0 Fork 0
Code Issues 7 Pull Requests Releases Wiki Activity

feat(ai): discover the llama-swap picker and vision from model metadata

Browse Source 
/insights/models now lists llama-swap's models live from /v1/models
instead of LLAMA_SWAP_ALLOWED_MODELS: every kind=chat model, with
setParamsByID aliases (:sub / :fast) hidden, filtered by
[llamacpp.picker] exclude and the optional include allowlist. Ids the
include list names but llama-swap no longer serves drop out instead of
showing as dead picker entries.

has_vision and context_window come from each model's config.yaml metadata
(read from meta.llamaswap, where the deployed llama-swap merges it, or from
meta as upstream documents). A model without metadata is text-only chat.
A live served window still wins over the configured ctx.

Agentic insights on llama-swap now ask the chat model's capabilities:
text-only models (e.g. qwen3.8-27b-code, --no-mmproj) get the local vision
model's description inlined, like hybrid, instead of an image they can't
read. If llama-swap can't be asked, images are sent as before.

Drops AppState.llamacpp_allowed_models.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Cameron CordesandClaude Opus 5.5 committed 2026-10-26 17:34:28 -04:00
1 parent aa3536afee
commit e10fc26ca7
4 files changed
+219 -92
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/ai/handlers.rs
+30 -22
View File
@@ -983,17 +983,15 @@ pub async fn generate_agentic_insight_handler(
}
/// GET /insights/models - Local-backend models with capabilities. Returns
/// Ollama servers when `LLM_BACKEND=ollama` (default), or llama-swap slots
/// when `LLM_BACKEND=llamacpp`. Same envelope shape either way so the
/// Ollama servers when `backend = "ollama"` (default), or llama-swap models
/// when `backend = "llamacpp"`. Same envelope shape either way so the
/// client picker doesn't have to branch on backend kind.
///
/// For llama-swap: `models` comes verbatim from `LLAMA_SWAP_ALLOWED_MODELS`
/// (no live `/v1/models` probe), `has_vision` is true only for the
/// configured `LLAMA_SWAP_VISION_MODEL` slot id, and `has_tool_calling` is
/// reported as true for every slot (llama-server is launched with `--jinja`
/// by convention — a misconfigured slot surfaces as a chat-call error).
/// `context_window` is each slot's served `-c` window when known without
/// loading the model (cached from a recent chat, or the model is running).
/// For llama-swap: `models` is discovered live from `/v1/models` — every
/// `kind: chat` model (aliases hidden) filtered by `[llamacpp.picker]` —
/// with `has_vision` / `context_window` from each model's config.yaml
/// `metadata`. A served window known without loading the model (cached from
/// a recent chat, or the model is running) takes precedence over `ctx`.
#[get("/insights/models")]
pub async fn get_available_models_handler(
_claims: Claims,
@@ -1004,20 +1002,30 @@ pub async fn get_available_models_handler(
if crate::ai::local_backend_is_llamacpp()
&& let Some(lc) = app_state.llamacpp.as_ref()
{
let picker = &crate::ai::config::llm_config().llamacpp.picker;
let mut models = match lc.discover_models().await {
Ok(found) => crate::ai::llamacpp::picker_models(found, picker),
Err(e) => {
// llama-swap is down, so chat would fail anyway; still list
// the configured model so the picker isn't empty when it
// comes back.
log::warn!("llama-swap model discovery failed: {e:#}");
vec![ModelCapabilities {
name: lc.primary_model.clone(),
has_vision: true,
has_tool_calling: true,
context_window: None,
}]
}
};
// Served windows for the context picker; never loads a model.
let windows = lc
.known_context_windows(&app_state.llamacpp_allowed_models)
.await;
let models: Vec<ModelCapabilities> = app_state
.llamacpp_allowed_models
.iter()
.map(|name| ModelCapabilities {
name: name.clone(),
has_vision: true,
has_tool_calling: true,
context_window: windows.get(name).copied(),
})
.collect();
let names: Vec<String> = models.iter().map(|m| m.name.clone()).collect();
let windows = lc.known_context_windows(&names).await;
for m in &mut models {
if let Some(&n) = windows.get(&m.name) {
m.context_window = Some(n);
}
}
let primary = ServerModels {
url: lc.base_url.clone(),
models,
src/ai/insight_generator.rs
+17 -6
View File
@@ -4447,7 +4447,7 @@ Return ONLY the summary, nothing else."#,
} else if local_via_llamacpp {
// Local via llama-swap.
let arc = self.llamacpp.as_ref().ok_or_else(|| {
anyhow::anyhow!("LLM_BACKEND=llamacpp but LLAMA_SWAP_URL not configured")
anyhow::anyhow!("backend = \"llamacpp\" but llamacpp.url not configured")
})?;
let mut c: LlamaCppClient = (**arc).clone();
if let Some(ref m) = overrides.model {
@@ -4515,11 +4515,22 @@ Return ONLY the summary, nothing else."#,
// Hybrid: chat model never sees images — describe-then-inject.
false
} else if local_via_llamacpp {
// llama-swap models receive images directly via OpenAI content
// parts. Capability probing isn't available (no `/api/show`),
// so assume vision support; a misconfigured model surfaces as
// a chat-call error.
true
// Vision models get images directly via OpenAI content parts;
// text-only ones (metadata `vision: false`) get the local vision
// model's description inlined instead, like hybrid. If llama-swap
// can't be asked, assume vision — the chat call will say why.
let lc = self
.llamacpp
.as_deref()
.expect("checked by local_via_llamacpp");
let model = overrides.model.as_deref().unwrap_or(&lc.primary_model);
match lc.model_capabilities(model).await {
Ok(caps) => caps.has_vision,
Err(e) => {
log::warn!("vision capability unknown for '{model}', sending images: {e:#}");
true
}
}
} else {
// Pure Ollama: probe model capabilities.
let ollama_for_caps = if let Some(ref model) = overrides.model {
src/ai/llamacpp.rs
+172 -55
View File
@@ -10,11 +10,11 @@
// `generate_embeddings` issue requests with the appropriate slot id in the
// `model` field, which is how llama-swap selects which backend process to
// run.
// - `/v1/models` returns only the configured slot ids — capabilities aren't
// reported by the API, so we infer `has_vision` from a single config field
// (`vision_model`, defaulting to `"vision"`) and assume `has_tool_calling`
// is true for every slot, since llama-swap entries default to launching
// llama-server with `--jinja`.
// - Capabilities come from each model's `metadata:` block in llama-swap's
// config.yaml, which `/v1/models` passes through (`kind`, `vision`, `ctx`;
// see `DiscoveredModel`). A model without metadata is treated as a
// text-only chat model. `has_tool_calling` is assumed true for every model,
// since llama-swap entries launch llama-server with `--jinja`.
//
// First consumer lands alongside the three-way backend dispatch in
// insight_generator / insight_chat.
@@ -31,6 +31,7 @@ use std::collections::HashMap;
use std::time::Duration;
use tracing_opentelemetry::OpenTelemetrySpanExt as _;
use crate::ai::config::PickerConfig;
use crate::ai::llm_client::{
ChatMessage, LlmClient, LlmStreamEvent, ModelCapabilities, Tool, ToolCall, ToolCallFunction,
};
@@ -987,32 +988,12 @@ impl LlmClient for LlamaCppClient {
}
async fn list_models(&self) -> Result<Vec<ModelCapabilities>> {
let url = format!("{}/models", self.base_url);
let resp = self
.client
.get(&url)
.send()
.await
.with_context(|| format!("GET {} failed", url))?;
if !resp.status().is_success() {
let status = resp.status();
let body = resp.text().await.unwrap_or_default();
bail!("llama-swap list_models failed: {} — {}", status, body);
}
let parsed: Value = resp.json().await.context("parsing models response")?;
let data = parsed
.get("data")
.and_then(|v| v.as_array())
.ok_or_else(|| anyhow!("models response missing data[]"))?;
let caps: Vec<ModelCapabilities> = data
.iter()
.map(|m| self.parse_model_capabilities(m))
.collect();
Ok(caps)
Ok(self
.discover_models()
.await?
.into_iter()
.map(|m| m.caps)
.collect())
}
async fn model_capabilities(&self, model: &str) -> Result<ModelCapabilities> {
@@ -1034,25 +1015,97 @@ impl LlmClient for LlamaCppClient {
}
}
impl LlamaCppClient {
fn parse_model_capabilities(&self, m: &Value) -> ModelCapabilities {
/// One entry of llama-swap's `/v1/models`, with the per-model `metadata`
/// from config.yaml decoded.
#[derive(Debug, Clone)]
pub struct DiscoveredModel {
pub caps: ModelCapabilities,
/// `metadata.kind`: `chat` | `embed` | `tts`. `None` when the model has
/// no metadata (treated as chat).
pub kind: Option<String>,
/// A `setParamsByID` alias like `qwen3.8-27b-code:sub` — a parameter
/// profile on another model, not a model of its own.
pub is_alias: bool,
}
impl DiscoveredModel {
pub fn is_chat(&self) -> bool {
matches!(self.kind.as_deref(), None | Some("chat"))
}
/// Parse one `/v1/models` row. The deployed llama-swap merges config
/// `metadata` into `meta.llamaswap` (next to its own `type` / `aliases`);
/// upstream docs describe it at `meta` itself, so both are read.
fn from_json(m: &Value) -> Self {
let field = |key: &str| {
m.pointer(&format!("/meta/llamaswap/{key}"))
.or_else(|| m.pointer(&format!("/meta/{key}")))
};
let name = m
.get("id")
.and_then(|v| v.as_str())
.and_then(Value::as_str)
.unwrap_or_default()
.to_string();
// llama-swap doesn't expose per-model modality flags. Assume all
// configured models support vision and tool calling; a model
// without multimodal support surfaces as a chat-call error.
ModelCapabilities {
name,
has_vision: true,
has_tool_calling: true,
context_window: None,
Self {
caps: ModelCapabilities {
name,
has_vision: field("vision").and_then(Value::as_bool).unwrap_or(false),
has_tool_calling: true,
context_window: field("ctx").and_then(Value::as_u64).map(|n| n as usize),
},
kind: field("kind").and_then(Value::as_str).map(str::to_string),
is_alias: field("type").and_then(Value::as_str) == Some("alias"),
}
}
}
/// The `/insights/models` picker: every non-alias chat model, minus
/// `picker.exclude`, restricted to `picker.include` when that's set.
/// llama-swap's order is kept.
pub fn picker_models(
models: Vec<DiscoveredModel>,
picker: &PickerConfig,
) -> Vec<ModelCapabilities> {
models
.into_iter()
.filter(|m| !m.is_alias && m.is_chat())
.filter(|m| !picker.exclude.contains(&m.caps.name))
.filter(|m| {
picker
.include
.as_ref()
.is_none_or(|inc| inc.contains(&m.caps.name))
})
.map(|m| m.caps)
.collect()
}
impl LlamaCppClient {
/// Every model llama-swap serves, aliases included, with metadata.
pub async fn discover_models(&self) -> Result<Vec<DiscoveredModel>> {
let url = format!("{}/models", self.base_url);
let resp = self
.client
.get(&url)
.send()
.await
.with_context(|| format!("GET {} failed", url))?;
if !resp.status().is_success() {
let status = resp.status();
let body = resp.text().await.unwrap_or_default();
bail!("llama-swap list_models failed: {} — {}", status, body);
}
let parsed: Value = resp.json().await.context("parsing models response")?;
let data = parsed
.get("data")
.and_then(|v| v.as_array())
.ok_or_else(|| anyhow!("models response missing data[]"))?;
Ok(data.iter().map(DiscoveredModel::from_json).collect())
}
}
/// Extract a diagnostic fragment from a llama-swap / llama-server response
/// that doesn't match the expected `{choices: [...]}` shape. llama-server
/// returns errors as `{"error": {"message": "...", "code": N, "type": "..."}}`;
@@ -1440,22 +1493,86 @@ mod tests {
}
#[test]
fn all_models_report_vision_capable() {
let mut c = LlamaCppClient::new(None, Some("chat".into()));
c.set_vision_model("vision".into());
fn metadata_drives_capabilities() {
// Shape of the deployed llama-swap: config metadata merged into
// meta.llamaswap.
let m = DiscoveredModel::from_json(&json!({
"id": "qwen3.8-27b-code-vision",
"meta": { "llamaswap": {
"type": "model", "kind": "chat", "vision": true, "ctx": 90000
}}
}));
assert_eq!(m.caps.name, "qwen3.8-27b-code-vision");
assert!(m.caps.has_vision);
assert!(m.caps.has_tool_calling);
assert_eq!(m.caps.context_window, Some(90000));
assert!(m.is_chat() && !m.is_alias);
}
let m_chat = json!({ "id": "chat" });
let m_vision = json!({ "id": "vision" });
let m_other = json!({ "id": "embed" });
#[test]
fn top_level_meta_is_read_too() {
// Upstream docs put metadata at `meta` itself.
let m = DiscoveredModel::from_json(&json!({
"id": "embed", "meta": { "kind": "embed", "dim": 768 }
}));
assert_eq!(m.kind.as_deref(), Some("embed"));
assert!(!m.is_chat());
}
let chat = c.parse_model_capabilities(&m_chat);
let vision = c.parse_model_capabilities(&m_vision);
let other = c.parse_model_capabilities(&m_other);
#[test]
fn model_without_metadata_is_text_only_chat() {
let m = DiscoveredModel::from_json(&json!({ "id": "mystery" }));
assert!(m.is_chat());
assert!(!m.caps.has_vision);
assert_eq!(m.caps.context_window, None);
}
assert!(chat.has_vision);
assert!(chat.has_tool_calling);
assert!(vision.has_vision);
assert!(other.has_vision);
fn row(id: &str, kind: &str, kind_type: &str) -> DiscoveredModel {
DiscoveredModel::from_json(&json!({
"id": id, "meta": { "llamaswap": { "type": kind_type, "kind": kind } }
}))
}
fn names(models: Vec<ModelCapabilities>) -> Vec<String> {
models.into_iter().map(|m| m.name).collect()
}
#[test]
fn picker_shows_chat_models_only_and_hides_aliases() {
let models = vec![
row("chatterbox", "tts", "model"),
row("embed", "embed", "model"),
row("q-code", "chat", "model"),
row("q-code:sub", "chat", "alias"),
row("gemma", "chat", "model"),
];
assert_eq!(
names(picker_models(models, &PickerConfig::default())),
["q-code", "gemma"]
);
}
#[test]
fn picker_applies_exclude_and_include() {
let models = || {
vec![
row("a", "chat", "model"),
row("b", "chat", "model"),
row("c", "chat", "model"),
]
};
let exclude = PickerConfig {
include: None,
exclude: vec!["b".into()],
};
assert_eq!(names(picker_models(models(), &exclude)), ["a", "c"]);
// include keeps llama-swap's order and silently drops ids it doesn't
// serve (the stale-allowlist case).
let include = PickerConfig {
include: Some(vec!["c".into(), "a".into(), "gone".into()]),
exclude: vec!["a".into()],
};
assert_eq!(names(picker_models(models(), &include)), ["c"]);
}
#[test]
src/state.rs
-9
View File
@@ -78,10 +78,6 @@ pub struct AppState {
/// threading through the generator.
#[allow(dead_code)]
pub llamacpp: Option<Arc<LlamaCppClient>>,
/// Curated list of llama-swap model ids exposed to clients. Sourced from
/// `LLAMA_SWAP_ALLOWED_MODELS` (comma-separated). Empty when unset; the
/// server then falls back to `LLAMA_SWAP_PRIMARY_MODEL`.
pub llamacpp_allowed_models: Vec<String>,
pub sms_client: SmsApiClient,
pub insight_generator: InsightGenerator,
/// Chat continuation service. Hold an Arc so handlers can clone cheaply.
@@ -134,7 +130,6 @@ impl AppState {
openrouter: Option<Arc<OpenRouterClient>>,
openrouter_allowed_models: Vec<String>,
llamacpp: Option<Arc<LlamaCppClient>>,
llamacpp_allowed_models: Vec<String>,
sms_client: SmsApiClient,
insight_generator: InsightGenerator,
insight_chat: Arc<InsightChatService>,
@@ -196,7 +191,6 @@ impl AppState {
openrouter,
openrouter_allowed_models,
llamacpp,
llamacpp_allowed_models,
sms_client,
insight_generator,
insight_chat,
@@ -232,7 +226,6 @@ impl Default for AppState {
let openrouter_allowed_models = llm.openrouter.allowed.clone();
let llamacpp = build_llamacpp(llm);
let llamacpp_allowed_models = llm.llamacpp.picker.include.clone().unwrap_or_default();
let sms_api_url =
env::var("SMS_API_URL").unwrap_or_else(|_| "http://localhost:8000".to_string());
@@ -369,7 +362,6 @@ impl Default for AppState {
openrouter,
openrouter_allowed_models,
llamacpp,
llamacpp_allowed_models,
sms_client,
insight_generator,
insight_chat,
@@ -548,7 +540,6 @@ impl AppState {
None,
Vec::new(),
None,
Vec::new(),
sms_client,
insight_generator,
insight_chat,