ai: collapse llamacpp into LLM_BACKEND env switch

Reverts the per-request backend="llamacpp" value. Chat/vision/embedding
backend is now a deploy-time decision (LLM_BACKEND=ollama|llamacpp),
applied globally across chat, vision describe, and embeddings — so
embedding vectors stay in one space across the index.

- Per-request backend whitelist back to "local"|"hybrid". A request
  arriving with backend="llamacpp" is rejected.
- LLM_BACKEND=llamacpp swaps the entire local stack to llama-swap:
  chat hits the chat slot, describe hits the vision slot, embeddings
  hit the embed slot. Hybrid mode still routes chat to OpenRouter
  but uses LLM_BACKEND for the describe pass.
- Drops env vars HYBRID_VISION_BACKEND, LLAMA_SWAP_VISION_MODELS,
  EMBEDDING_BACKEND (the last never shipped). Drops the
  LlamaCppClient.vision_models allowlist — capability inference now
  reports has_vision only for the configured vision_model slot.
- Drops the /insights/llamacpp/models handler. /insights/models is
  the single endpoint; returns Ollama servers under LLM_BACKEND=ollama
  and llama-swap slots (from LLAMA_SWAP_ALLOWED_MODELS) under
  LLM_BACKEND=llamacpp. Same envelope shape either way.
- New ai::embed_one helper routes embeddings through llama-swap when
  LLM_BACKEND=llamacpp (else Ollama). Wires it into the four
  insight_generator embedding sites.
- Cross-replay matrix simplifies to pre-llamacpp shape (local↔local,
  hybrid↔hybrid, hybrid→local allowed; local→hybrid rejected).
This commit is contained in:
Cameron Cordes committed 2026-05-21 11:36:58 -04:00
1 parent d14df63f19
commit be51421b38
9 files changed
+338 -301

No files matched your search

+9 -24
View File
@@ -11,10 +11,10 @@
// `model` field, which is how llama-swap selects which backend process to
// run.
// - `/v1/models` returns only the configured slot ids — capabilities aren't
// reported by the API, so `vision_models` is a config-time allowlist (env
// `LLAMA_SWAP_VISION_MODELS`) used to set `has_vision` on responses.
// `has_tool_calling` is assumed true for every slot, since llama-swap entries
// default to launching llama-server with `--jinja`.
// reported by the API, so we infer `has_vision` from a single config field
// (`vision_model`, defaulting to `"vision"`) and assume `has_tool_calling`
// is true for every slot, since llama-swap entries default to launching
// llama-server with `--jinja`.
//
// First consumer lands alongside the three-way backend dispatch in
// insight_generator / insight_chat.
@@ -50,16 +50,10 @@ pub struct LlamaCppClient {
/// Embedding model slot id (e.g. `"embed"`). Used for
/// `generate_embeddings`.
pub embedding_model: String,
/// Vision model slot id (e.g. `"vision"`). Used for `describe_image` and
/// included in `vision_models` automatically so capability lookups for
/// the default vision slot report `has_vision = true` even when the env
/// allowlist is empty.
/// Vision model slot id (e.g. `"vision"`). Used for `describe_image`,
/// and the only slot that reports `has_vision = true` in capability
/// lookups (llama-swap's `/v1/models` doesn't surface modality).
pub vision_model: String,
/// Operator-curated set of slot ids known to be multimodal. Drives the
/// `has_vision` field in `list_models` / `model_capabilities`, since
/// llama-swap's `/v1/models` doesn't report modality. Empty allowlist
/// still marks `vision_model` as vision-capable.
pub vision_models: Vec<String>,
num_ctx: Option<i32>,
temperature: Option<f32>,
top_p: Option<f32>,
@@ -83,7 +77,6 @@ impl LlamaCppClient {
primary_model: primary_model.unwrap_or_else(|| DEFAULT_PRIMARY_MODEL.to_string()),
embedding_model: DEFAULT_EMBEDDING_MODEL.to_string(),
vision_model: DEFAULT_VISION_MODEL.to_string(),
vision_models: Vec::new(),
num_ctx: None,
temperature: None,
top_p: None,
@@ -100,10 +93,6 @@ impl LlamaCppClient {
self.vision_model = model;
}
pub fn set_vision_models(&mut self, models: Vec<String>) {
self.vision_models = models;
}
pub fn set_num_ctx(&mut self, num_ctx: Option<i32>) {
self.num_ctx = num_ctx;
}
@@ -692,7 +681,7 @@ impl LlamaCppClient {
.and_then(|v| v.as_str())
.unwrap_or_default()
.to_string();
let has_vision = name == self.vision_model || self.vision_models.iter().any(|v| v == &name);
let has_vision = name == self.vision_model;
// Tool calling is the default for llama-swap entries we configure
// (--jinja flag); no negative-list mechanism yet, so report true.
ModelCapabilities {
@@ -954,25 +943,21 @@ mod tests {
}
#[test]
fn capability_inference_uses_vision_model_and_allowlist() {
fn capability_inference_marks_only_vision_slot() {
let mut c = LlamaCppClient::new(None, Some("chat".into()));
c.set_vision_model("vision".into());
c.set_vision_models(vec!["qwen-vl".into()]);
let m_chat = json!({ "id": "chat" });
let m_vision = json!({ "id": "vision" });
let m_qwen = json!({ "id": "qwen-vl" });
let m_other = json!({ "id": "embed" });
let chat = c.parse_model_capabilities(&m_chat);
let vision = c.parse_model_capabilities(&m_vision);
let qwen = c.parse_model_capabilities(&m_qwen);
let other = c.parse_model_capabilities(&m_other);
assert!(!chat.has_vision);
assert!(chat.has_tool_calling);
assert!(vision.has_vision);
assert!(qwen.has_vision);
assert!(!other.has_vision);
}
}