diff --git a/src/ai/llamacpp.rs b/src/ai/llamacpp.rs index e6f135d..0e7de7a 100644 --- a/src/ai/llamacpp.rs +++ b/src/ai/llamacpp.rs @@ -31,7 +31,7 @@ use std::collections::HashMap; use std::time::Duration; use tracing_opentelemetry::OpenTelemetrySpanExt as _; -use crate::ai::config::PickerConfig; +use crate::ai::config::{LlmConfig, LocalBackend, PickerConfig}; use crate::ai::llm_client::{ ChatMessage, LlmClient, LlmStreamEvent, ModelCapabilities, Tool, ToolCall, ToolCallFunction, }; @@ -1026,6 +1026,8 @@ pub struct DiscoveredModel { /// A `setParamsByID` alias like `qwen3.8-27b-code:sub` — a parameter /// profile on another model, not a model of its own. pub is_alias: bool, + /// `metadata.dim` on embed models: the vector width they produce. + pub dim: Option, } impl DiscoveredModel { @@ -1055,6 +1057,7 @@ impl DiscoveredModel { }, kind: field("kind").and_then(Value::as_str).map(str::to_string), is_alias: field("type").and_then(Value::as_str) == Some("alias"), + dim: field("dim").and_then(Value::as_u64).map(|n| n as usize), } } } @@ -1080,7 +1083,90 @@ pub fn picker_models( .collect() } +/// Ways `cfg` disagrees with what llama-swap serves: role models that don't +/// exist or lack the capability the role needs, picker ids that don't exist, +/// models without metadata, and an embed `dim` that differs from +/// `embedding.dim` (stored vectors would stop matching). +pub fn catalog_warnings(cfg: &LlmConfig, models: &[DiscoveredModel]) -> Vec { + let find = |id: &str| models.iter().find(|m| m.caps.name == id); + let mut out = Vec::new(); + let mut require = + |role: &str, id: &str, check: &dyn Fn(&DiscoveredModel) -> Option| match find(id) { + None => out.push(format!("{role} model '{id}' is not served by llama-swap")), + Some(m) => { + if let Some(problem) = check(m) { + out.push(format!("{role} model '{id}' {problem}")); + } + } + }; + + let chat_check = |m: &DiscoveredModel| { + (!m.is_chat()).then(|| format!("has kind '{}', not chat", m.kind.as_deref().unwrap_or(""))) + }; + let lc = &cfg.llamacpp; + if cfg.backend == LocalBackend::Llamacpp { + require("chat", &lc.chat, &chat_check); + require("vision", lc.vision_model(), &|m| { + (!m.caps.has_vision).then(|| "has no vision (metadata vision: true missing)".into()) + }); + let want_dim = cfg.embedding.dim; + require("embedding", &cfg.embedding.model, &|m| { + if m.kind.as_deref() != Some("embed") { + Some("isn't kind: embed".into()) + } else { + match m.dim { + Some(d) if d != want_dim => Some(format!( + "produces {d}-dim vectors but embedding.dim is {want_dim}; \ + stored vectors won't match" + )), + _ => None, + } + } + }); + if let Some(id) = &cfg.unified_search_model { + require("unified_search", id, &chat_check); + } + } + require("tts", &cfg.tts.model, &|m| { + (m.kind.as_deref() != Some("tts")).then(|| "isn't kind: tts".into()) + }); + + let picker_ids = lc.picker.include.iter().flatten().chain(&lc.picker.exclude); + for id in picker_ids { + if find(id).is_none() { + out.push(format!( + "picker lists '{id}', which llama-swap doesn't serve" + )); + } + } + let unlabeled: Vec<&str> = models + .iter() + .filter(|m| !m.is_alias && m.kind.is_none()) + .map(|m| m.caps.name.as_str()) + .collect(); + if !unlabeled.is_empty() { + out.push(format!( + "no metadata in llama-swap config.yaml for: {} (treated as text-only chat)", + unlabeled.join(", ") + )); + } + out +} + impl LlamaCppClient { + /// Log [`catalog_warnings`] once at startup. Never fails: llama-swap may + /// simply not be up yet. + pub async fn log_catalog_warnings(&self, cfg: &LlmConfig) { + match self.discover_models().await { + Ok(models) => { + for w in catalog_warnings(cfg, &models) { + log::warn!("LLM config: {w}"); + } + } + Err(e) => log::warn!("LLM config not checked, llama-swap unreachable: {e:#}"), + } + } + /// Every model llama-swap serves, aliases included, with metadata. pub async fn discover_models(&self) -> Result> { let url = format!("{}/models", self.base_url); @@ -1552,6 +1638,79 @@ mod tests { ); } + fn served() -> Vec { + let m = |v: Value| DiscoveredModel::from_json(&v); + vec![ + m( + json!({"id": "chat", "meta": {"llamaswap": {"type": "model", "kind": "chat", "vision": false}}}), + ), + m( + json!({"id": "eyes", "meta": {"llamaswap": {"type": "model", "kind": "chat", "vision": true}}}), + ), + m( + json!({"id": "embed", "meta": {"llamaswap": {"type": "model", "kind": "embed", "dim": 768}}}), + ), + m(json!({"id": "chatterbox", "meta": {"llamaswap": {"type": "model", "kind": "tts"}}})), + m(json!({"id": "chat:sub", "meta": {"llamaswap": {"type": "alias", "kind": "chat"}}})), + ] + } + + fn llamacpp_cfg() -> LlmConfig { + let mut cfg = LlmConfig { + backend: LocalBackend::Llamacpp, + ..Default::default() + }; + cfg.llamacpp.url = Some("http://x/v1".into()); + cfg.llamacpp.vision = Some("eyes".into()); + cfg + } + + #[test] + fn matching_catalog_has_no_warnings() { + assert!(catalog_warnings(&llamacpp_cfg(), &served()).is_empty()); + } + + #[test] + fn catalog_mismatches_are_reported() { + let mut cfg = llamacpp_cfg(); + cfg.llamacpp.chat = "gone".into(); + cfg.llamacpp.vision = None; // falls back to "gone" too + cfg.embedding.dim = 1024; + cfg.tts.model = "embed".into(); + cfg.llamacpp.picker.exclude = vec!["stale".into()]; + let w = catalog_warnings(&cfg, &served()).join("\n"); + assert!(w.contains("chat model 'gone' is not served"), "{w}"); + assert!(w.contains("vision model 'gone' is not served"), "{w}"); + assert!( + w.contains("768-dim vectors but embedding.dim is 1024"), + "{w}" + ); + assert!(w.contains("tts model 'embed' isn't kind: tts"), "{w}"); + assert!(w.contains("picker lists 'stale'"), "{w}"); + } + + #[test] + fn text_only_vision_model_and_unlabeled_models_are_reported() { + let mut cfg = llamacpp_cfg(); + cfg.llamacpp.vision = Some("chat".into()); + let mut models = served(); + models.push(DiscoveredModel::from_json(&json!({"id": "bare"}))); + let w = catalog_warnings(&cfg, &models).join("\n"); + assert!(w.contains("vision model 'chat' has no vision"), "{w}"); + assert!( + w.contains("no metadata in llama-swap config.yaml for: bare"), + "{w}" + ); + assert!(!w.contains("chat:sub"), "aliases never need metadata: {w}"); + } + + #[test] + fn ollama_backend_only_checks_tts() { + let mut cfg = LlmConfig::default(); + cfg.llamacpp.chat = "gone".into(); + assert!(catalog_warnings(&cfg, &served()).is_empty()); + } + #[test] fn picker_applies_exclude_and_include() { let models = || { diff --git a/src/main.rs b/src/main.rs index b7bed1f..fae1696 100644 --- a/src/main.rs +++ b/src/main.rs @@ -75,6 +75,11 @@ fn main() -> std::io::Result<()> { // table; we use that list to drive the initial thumbnail sweep. let app_data = Data::new(AppState::default()); + // Cross-check llm.toml against what llama-swap serves (warn-only). + if let Some(lc) = app_data.llamacpp.clone() { + tokio::spawn(async move { lc.log_catalog_warnings(ai::config::llm_config()).await }); + } + // Kick thumbnail generation onto a background thread so the HTTP // server can accept traffic while large libraries are backfilling. // Existing thumbs are re-used (exists() check inside the walk),