ai: collapse llamacpp into LLM_BACKEND env switch
Reverts the per-request backend="llamacpp" value. Chat/vision/embedding backend is now a deploy-time decision (LLM_BACKEND=ollama|llamacpp), applied globally across chat, vision describe, and embeddings — so embedding vectors stay in one space across the index. - Per-request backend whitelist back to "local"|"hybrid". A request arriving with backend="llamacpp" is rejected. - LLM_BACKEND=llamacpp swaps the entire local stack to llama-swap: chat hits the chat slot, describe hits the vision slot, embeddings hit the embed slot. Hybrid mode still routes chat to OpenRouter but uses LLM_BACKEND for the describe pass. - Drops env vars HYBRID_VISION_BACKEND, LLAMA_SWAP_VISION_MODELS, EMBEDDING_BACKEND (the last never shipped). Drops the LlamaCppClient.vision_models allowlist — capability inference now reports has_vision only for the configured vision_model slot. - Drops the /insights/llamacpp/models handler. /insights/models is the single endpoint; returns Ollama servers under LLM_BACKEND=ollama and llama-swap slots (from LLAMA_SWAP_ALLOWED_MODELS) under LLM_BACKEND=llamacpp. Same envelope shape either way. - New ai::embed_one helper routes embeddings through llama-swap when LLM_BACKEND=llamacpp (else Ollama). Wires it into the four insight_generator embedding sites. - Cross-replay matrix simplifies to pre-llamacpp shape (local↔local, hybrid↔hybrid, hybrid→local allowed; local→hybrid rejected).
This commit is contained in:
1 parent
d14df63f19
commit
be51421b38
9 files changed
+338
-301
No files matched your search
+9
-24
@@ -11,10 +11,10 @@
|
||||
// `model` field, which is how llama-swap selects which backend process to
|
||||
// run.
|
||||
// - `/v1/models` returns only the configured slot ids — capabilities aren't
|
||||
// reported by the API, so `vision_models` is a config-time allowlist (env
|
||||
// `LLAMA_SWAP_VISION_MODELS`) used to set `has_vision` on responses.
|
||||
// `has_tool_calling` is assumed true for every slot, since llama-swap entries
|
||||
// default to launching llama-server with `--jinja`.
|
||||
// reported by the API, so we infer `has_vision` from a single config field
|
||||
// (`vision_model`, defaulting to `"vision"`) and assume `has_tool_calling`
|
||||
// is true for every slot, since llama-swap entries default to launching
|
||||
// llama-server with `--jinja`.
|
||||
//
|
||||
// First consumer lands alongside the three-way backend dispatch in
|
||||
// insight_generator / insight_chat.
|
||||
@@ -50,16 +50,10 @@ pub struct LlamaCppClient {
|
||||
/// Embedding model slot id (e.g. `"embed"`). Used for
|
||||
/// `generate_embeddings`.
|
||||
pub embedding_model: String,
|
||||
/// Vision model slot id (e.g. `"vision"`). Used for `describe_image` and
|
||||
/// included in `vision_models` automatically so capability lookups for
|
||||
/// the default vision slot report `has_vision = true` even when the env
|
||||
/// allowlist is empty.
|
||||
/// Vision model slot id (e.g. `"vision"`). Used for `describe_image`,
|
||||
/// and the only slot that reports `has_vision = true` in capability
|
||||
/// lookups (llama-swap's `/v1/models` doesn't surface modality).
|
||||
pub vision_model: String,
|
||||
/// Operator-curated set of slot ids known to be multimodal. Drives the
|
||||
/// `has_vision` field in `list_models` / `model_capabilities`, since
|
||||
/// llama-swap's `/v1/models` doesn't report modality. Empty allowlist
|
||||
/// still marks `vision_model` as vision-capable.
|
||||
pub vision_models: Vec<String>,
|
||||
num_ctx: Option<i32>,
|
||||
temperature: Option<f32>,
|
||||
top_p: Option<f32>,
|
||||
@@ -83,7 +77,6 @@ impl LlamaCppClient {
|
||||
primary_model: primary_model.unwrap_or_else(|| DEFAULT_PRIMARY_MODEL.to_string()),
|
||||
embedding_model: DEFAULT_EMBEDDING_MODEL.to_string(),
|
||||
vision_model: DEFAULT_VISION_MODEL.to_string(),
|
||||
vision_models: Vec::new(),
|
||||
num_ctx: None,
|
||||
temperature: None,
|
||||
top_p: None,
|
||||
@@ -100,10 +93,6 @@ impl LlamaCppClient {
|
||||
self.vision_model = model;
|
||||
}
|
||||
|
||||
pub fn set_vision_models(&mut self, models: Vec<String>) {
|
||||
self.vision_models = models;
|
||||
}
|
||||
|
||||
pub fn set_num_ctx(&mut self, num_ctx: Option<i32>) {
|
||||
self.num_ctx = num_ctx;
|
||||
}
|
||||
@@ -692,7 +681,7 @@ impl LlamaCppClient {
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
let has_vision = name == self.vision_model || self.vision_models.iter().any(|v| v == &name);
|
||||
let has_vision = name == self.vision_model;
|
||||
// Tool calling is the default for llama-swap entries we configure
|
||||
// (--jinja flag); no negative-list mechanism yet, so report true.
|
||||
ModelCapabilities {
|
||||
@@ -954,25 +943,21 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn capability_inference_uses_vision_model_and_allowlist() {
|
||||
fn capability_inference_marks_only_vision_slot() {
|
||||
let mut c = LlamaCppClient::new(None, Some("chat".into()));
|
||||
c.set_vision_model("vision".into());
|
||||
c.set_vision_models(vec!["qwen-vl".into()]);
|
||||
|
||||
let m_chat = json!({ "id": "chat" });
|
||||
let m_vision = json!({ "id": "vision" });
|
||||
let m_qwen = json!({ "id": "qwen-vl" });
|
||||
let m_other = json!({ "id": "embed" });
|
||||
|
||||
let chat = c.parse_model_capabilities(&m_chat);
|
||||
let vision = c.parse_model_capabilities(&m_vision);
|
||||
let qwen = c.parse_model_capabilities(&m_qwen);
|
||||
let other = c.parse_model_capabilities(&m_other);
|
||||
|
||||
assert!(!chat.has_vision);
|
||||
assert!(chat.has_tool_calling);
|
||||
assert!(vision.has_vision);
|
||||
assert!(qwen.has_vision);
|
||||
assert!(!other.has_vision);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user