feat(ai): discover the llama-swap picker and vision from model metadata
/insights/models now lists llama-swap's models live from /v1/models instead of LLAMA_SWAP_ALLOWED_MODELS: every kind=chat model, with setParamsByID aliases (:sub / :fast) hidden, filtered by [llamacpp.picker] exclude and the optional include allowlist. Ids the include list names but llama-swap no longer serves drop out instead of showing as dead picker entries. has_vision and context_window come from each model's config.yaml metadata (read from meta.llamaswap, where the deployed llama-swap merges it, or from meta as upstream documents). A model without metadata is text-only chat. A live served window still wins over the configured ctx. Agentic insights on llama-swap now ask the chat model's capabilities: text-only models (e.g. qwen3.8-27b-code, --no-mmproj) get the local vision model's description inlined, like hybrid, instead of an image they can't read. If llama-swap can't be asked, images are sent as before. Drops AppState.llamacpp_allowed_models. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
aa3536afee
commit
e10fc26ca7
4 files changed
+219
-92
No files matched your search
+30
-22
@@ -983,17 +983,15 @@ pub async fn generate_agentic_insight_handler(
|
||||
}
|
||||
|
||||
/// GET /insights/models - Local-backend models with capabilities. Returns
|
||||
/// Ollama servers when `LLM_BACKEND=ollama` (default), or llama-swap slots
|
||||
/// when `LLM_BACKEND=llamacpp`. Same envelope shape either way so the
|
||||
/// Ollama servers when `backend = "ollama"` (default), or llama-swap models
|
||||
/// when `backend = "llamacpp"`. Same envelope shape either way so the
|
||||
/// client picker doesn't have to branch on backend kind.
|
||||
///
|
||||
/// For llama-swap: `models` comes verbatim from `LLAMA_SWAP_ALLOWED_MODELS`
|
||||
/// (no live `/v1/models` probe), `has_vision` is true only for the
|
||||
/// configured `LLAMA_SWAP_VISION_MODEL` slot id, and `has_tool_calling` is
|
||||
/// reported as true for every slot (llama-server is launched with `--jinja`
|
||||
/// by convention — a misconfigured slot surfaces as a chat-call error).
|
||||
/// `context_window` is each slot's served `-c` window when known without
|
||||
/// loading the model (cached from a recent chat, or the model is running).
|
||||
/// For llama-swap: `models` is discovered live from `/v1/models` — every
|
||||
/// `kind: chat` model (aliases hidden) filtered by `[llamacpp.picker]` —
|
||||
/// with `has_vision` / `context_window` from each model's config.yaml
|
||||
/// `metadata`. A served window known without loading the model (cached from
|
||||
/// a recent chat, or the model is running) takes precedence over `ctx`.
|
||||
#[get("/insights/models")]
|
||||
pub async fn get_available_models_handler(
|
||||
_claims: Claims,
|
||||
@@ -1004,20 +1002,30 @@ pub async fn get_available_models_handler(
|
||||
if crate::ai::local_backend_is_llamacpp()
|
||||
&& let Some(lc) = app_state.llamacpp.as_ref()
|
||||
{
|
||||
let picker = &crate::ai::config::llm_config().llamacpp.picker;
|
||||
let mut models = match lc.discover_models().await {
|
||||
Ok(found) => crate::ai::llamacpp::picker_models(found, picker),
|
||||
Err(e) => {
|
||||
// llama-swap is down, so chat would fail anyway; still list
|
||||
// the configured model so the picker isn't empty when it
|
||||
// comes back.
|
||||
log::warn!("llama-swap model discovery failed: {e:#}");
|
||||
vec![ModelCapabilities {
|
||||
name: lc.primary_model.clone(),
|
||||
has_vision: true,
|
||||
has_tool_calling: true,
|
||||
context_window: None,
|
||||
}]
|
||||
}
|
||||
};
|
||||
// Served windows for the context picker; never loads a model.
|
||||
let windows = lc
|
||||
.known_context_windows(&app_state.llamacpp_allowed_models)
|
||||
.await;
|
||||
let models: Vec<ModelCapabilities> = app_state
|
||||
.llamacpp_allowed_models
|
||||
.iter()
|
||||
.map(|name| ModelCapabilities {
|
||||
name: name.clone(),
|
||||
has_vision: true,
|
||||
has_tool_calling: true,
|
||||
context_window: windows.get(name).copied(),
|
||||
})
|
||||
.collect();
|
||||
let names: Vec<String> = models.iter().map(|m| m.name.clone()).collect();
|
||||
let windows = lc.known_context_windows(&names).await;
|
||||
for m in &mut models {
|
||||
if let Some(&n) = windows.get(&m.name) {
|
||||
m.context_window = Some(n);
|
||||
}
|
||||
}
|
||||
let primary = ServerModels {
|
||||
url: lc.base_url.clone(),
|
||||
models,
|
||||
|
||||
Reference in new issue
Block a user