diff --git a/.env.example b/.env.example index f730de5..3646346 100644 --- a/.env.example +++ b/.env.example @@ -40,78 +40,16 @@ PLAYLIST_CLEANUP_INTERVAL_SECONDS=86400 # RUST_LOG also gates export: keep it at info or debug. # OTLP_OTLS_ENDPOINT=http://localhost:4317 -# ── AI Insights — Ollama (local LLM) ──────────────────────────────────── -OLLAMA_PRIMARY_URL=http://localhost:11434 -OLLAMA_PRIMARY_MODEL=nemotron-3-nano:30b -# Optional fallback server tried on connection failure. -# OLLAMA_FALLBACK_URL=http://server:11434 -# OLLAMA_FALLBACK_MODEL=llama3.2:3b -OLLAMA_REQUEST_TIMEOUT_SECONDS=120 -# Cap on tool-calling iterations per chat turn / agentic insight. -AGENTIC_MAX_ITERATIONS=6 -AGENTIC_CHAT_MAX_ITERATIONS=6 - -# ── AI Insights — OpenRouter (hybrid backend, optional) ───────────────── -# Set OPENROUTER_API_KEY to enable the hybrid backend (vision stays -# local on Ollama, chat routes to OpenRouter). +# ── AI Insights — LLMs ────────────────────────────────────────────────── +# Models, backends, the embedding contract, TTS and agentic caps live in +# llm.toml — copy llm.example.toml. Only the secret stays here. +# LLM_CONFIG_PATH=./llm.toml +# Enables the hybrid backend (chat via OpenRouter; photos stay local). # OPENROUTER_API_KEY=sk-or-... -# OPENROUTER_DEFAULT_MODEL=anthropic/claude-sonnet-4 -# OPENROUTER_ALLOWED_MODELS=openai/gpt-4o-mini,anthropic/claude-haiku-4-5,google/gemini-2.5-flash -# OPENROUTER_BASE_URL=https://openrouter.ai/api/v1 -# OPENROUTER_EMBEDDING_MODEL=openai/text-embedding-3-small -# OPENROUTER_HTTP_REFERER=https://your-site.example -# OPENROUTER_APP_TITLE=ImageApi -# Cap on generated tokens per call (sent as max_tokens). OpenRouter reserves -# credit against the requested max, so leaving it uncapped makes pricey -# models fail with credit-limit errors. Default 16384; 0 = no cap. -# OPENROUTER_MAX_TOKENS=16384 - -# ── AI Insights — local backend switch ────────────────────────────────── -# Picks which local LLM stack the server uses for chat, vision describe, -# and embeddings. `ollama` (default) uses the OLLAMA_* settings above; -# `llamacpp` uses the LLAMA_SWAP_* settings below. The switch is global -# and applies to both `backend=local` and `backend=hybrid` (hybrid keeps -# chat on OpenRouter but still uses this stack for the describe pass). -# Don't flip mid-deploy without re-embedding existing index rows — -# mixed vector spaces break similarity search. -# LLM_BACKEND=ollama - -# ── AI Insights — llama.cpp / llama-swap (optional) ───────────────────── -# Set LLAMA_SWAP_URL plus LLM_BACKEND=llamacpp to swap the local stack -# off Ollama. Talks OpenAI-compatible /v1 to a llama-swap proxy fronting -# per-slot llama-server instances. Chat models receive images directly -# via content-parts (vision-capable models assumed); a separate vision -# slot is used only by the describe_photo tool and describe-image utility. -# LLAMA_SWAP_URL=http://localhost:9292/v1 -# LLAMA_SWAP_PRIMARY_MODEL=chat -# Optional dedicated vision slot for describe_image. Defaults to -# PRIMARY_MODEL so describe_photo works without extra config. -# LLAMA_SWAP_VISION_MODEL=vision -# LLAMA_SWAP_EMBEDDING_MODEL=embed -# Comma-separated allowlist surfaced by /insights/models when -# LLM_BACKEND=llamacpp. All report has_vision=true. -# LLAMA_SWAP_ALLOWED_MODELS=chat,vision,embed -# LLAMA_SWAP_REQUEST_TIMEOUT_SECONDS=180 - -# ── Unified search translation model (optional) ───────────────────────── -# /photos/search/unified runs one small LLM call to translate a natural- -# language query into structured filters + a semantic term, then CLIP-ranks. -# That step needs an LLM AND CLIP available at once. On a tight VRAM budget a -# large chat model can't co-reside with CLIP, so pin a small, fast model here -# (it can stay loaded alongside CLIP and the chat model). Precedence: -# UNIFIED_SEARCH_MODEL > the client's selected model > the configured default. -# Use the configured backend (LLM_BACKEND); local only — no hybrid. -# UNIFIED_SEARCH_MODEL=qwen3-0.6b - -# ── Text-to-speech (optional, requires LLAMA_SWAP_URL) ─────────────────── -# TTS routes through the same llama-swap proxy (a Chatterbox model id), so it -# only needs LLAMA_SWAP_URL — it does NOT require LLM_BACKEND=llamacpp. -# Powers POST /tts/speech and the /tts/voices* endpoints (read-aloud insights -# + voice cloning in the mobile app). -# LLAMA_SWAP_TTS_MODEL=chatterbox # TTS model id in config.yaml -# LLAMA_SWAP_TTS_VOICE=m # default voice when a request omits one -# LLAMA_SWAP_TTS_REF_SECONDS=30 # max voice-clone reference clip length (s) -# LLAMA_SWAP_TTS_REQUEST_TIMEOUT_SECONDS=600 # synth timeout (long chunked text) +# +# The old OLLAMA_* / LLAMA_SWAP_* / OPENROUTER_* / LLM_BACKEND / EMBED* / +# AGENTIC_* / UNIFIED_SEARCH_MODEL vars still override llm.toml for now, with a +# startup deprecation warning. Move them into llm.toml. # ── AI Insights — sibling services (optional) ─────────────────────────── # Apollo (places, face inference, CLIP encoders). Single-Apollo deploys diff --git a/CLAUDE.md b/CLAUDE.md index b5d8423..bf7d59a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -561,11 +561,11 @@ GET /memories?path=...&recursive=true POST /insights/generate (non-agentic single-shot) POST /insights/generate/agentic (tool-calling loop; body: { file_path, backend?, model?, ... }) GET /insights?path=...&library=... -GET /insights/models (local-backend models + capabilities; Ollama OR llama-swap based on LLM_BACKEND) +GET /insights/models (local-backend models + capabilities; Ollama OR llama-swap per llm.toml `backend`) GET /insights/openrouter/models (curated OpenRouter allowlist) POST /insights/rate (thumbs up/down for training data) -// Text-to-Speech (Chatterbox via llama-swap; needs LLAMA_SWAP_URL) +// Text-to-Speech (Chatterbox via llama-swap; needs llamacpp.url) POST /tts/speech (read-aloud: { text, voice?, ... } -> { audio_base64, format }) GET /tts/voices (Chatterbox voice library) POST /tts/voices/upload (clone a voice from an uploaded clip; multipart) @@ -745,12 +745,9 @@ DATE_BACKFILL_MAX_PER_TICK=500 # Cap on canonical-date drain per watcher HLS_COPY_MAX_BITRATE=20000000 # Max source bits/s to stream-copy into HLS; above it (or >1080p) transcodes OTLP_OTLS_ENDPOINT=http://... # OTLP collector for span export (any build; unset = no export) -# AI Insights Configuration -OLLAMA_PRIMARY_URL=http://desktop:11434 # Primary Ollama server (e.g., desktop) -OLLAMA_FALLBACK_URL=http://server:11434 # Fallback Ollama server (optional, always-on) -OLLAMA_PRIMARY_MODEL=nemotron-3-nano:30b # Model for primary server (default: nemotron-3-nano:30b) -OLLAMA_FALLBACK_MODEL=llama3.2:3b # Model for fallback server (optional, uses primary if not set) -OLLAMA_REQUEST_TIMEOUT_SECONDS=120 # Per-request generation timeout (default 120). Increase for slow CPU-offloaded models. +# AI Insights: LLM settings live in llm.toml (see "LLM configuration" below) +LLM_CONFIG_PATH=./llm.toml # Default ./llm.toml; missing default file = env + defaults +OPENROUTER_API_KEY=sk-or-... # Secret, env-only. Enables the hybrid backend SMS_API_URL=http://localhost:8000 # SMS message API endpoint (default: localhost:8000) SMS_API_TOKEN=your-api-token # SMS API authentication token (optional) @@ -769,79 +766,49 @@ FACE_AUTOBIND_MIN_COS=0.4 # Phase 3: cosine-sim floor for FACE_DETECT_CONCURRENCY=8 # Phase 3: per-scan-tick parallel detect calls FACE_DETECT_TIMEOUT_SEC=60 # reqwest client timeout (CPU inference can be slow) -# OpenRouter (Hybrid Backend) - keeps embeddings + vision local, routes chat to OpenRouter -OPENROUTER_API_KEY=sk-or-... # Required to enable hybrid backend -OPENROUTER_DEFAULT_MODEL=anthropic/claude-sonnet-4 # Used when client doesn't pick a model -OPENROUTER_ALLOWED_MODELS=openai/gpt-4o-mini,anthropic/claude-haiku-4-5,google/gemini-2.5-flash - # Curated allowlist exposed to clients via - # GET /insights/openrouter/models. Empty = no picker. -OPENROUTER_BASE_URL=https://openrouter.ai/api/v1 # Override base URL (optional) -OPENROUTER_EMBEDDING_MODEL=openai/text-embedding-3-small # Optional, embeddings stay local today -OPENROUTER_HTTP_REFERER=https://your-site.example # Optional attribution header -OPENROUTER_APP_TITLE=ImageApi # Optional attribution header -OPENROUTER_MAX_TOKENS=16384 # Per-call max_tokens cap (default 16384; 0 = uncapped). - # OpenRouter reserves credit against this, so an uncapped - # request to a pricey model can fail on credit limits. - -# Local LLM backend switch. `ollama` (default) keeps the OLLAMA_* settings -# above; `llamacpp` swaps the entire local stack (chat + vision describe + -# embeddings) over to llama-swap. The switch is global and applies to -# `backend=local` requests and to `backend=hybrid`'s describe pass (hybrid -# chat still goes to OpenRouter). Don't flip mid-deploy without -# re-embedding — mixed vector spaces break similarity search. -LLM_BACKEND=ollama - -# Embedding model contract. Corpus and queries must be embedded by the same -# model with matching prefixes — after changing the embed model or any of -# these, run `cargo run --bin reembed_embeddings` (all tables) or search is -# garbage. Prefix values may contain a literal \n (expanded to a newline). -EMBEDDING_DIM=768 # 768 = nomic-embed-text v1.5; 1024 = Qwen3-Embedding-0.6B -EMBED_QUERY_PREFIX= # nomic: "search_query: " | Qwen3: "Instruct: \nQuery: " -EMBED_DOCUMENT_PREFIX= # nomic: "search_document: " | Qwen3: leave empty - -# llama.cpp / llama-swap (used when LLM_BACKEND=llamacpp). OpenAI-compatible -# proxy hosting one or more llama-server processes. Chat models receive -# images directly via content-parts (all models assumed vision-capable). -LLAMA_SWAP_URL=http://localhost:9292/v1 # Required when LLM_BACKEND=llamacpp -LLAMA_SWAP_PRIMARY_MODEL=chat # Chat slot id (matches config.yaml) -LLAMA_SWAP_VISION_MODEL= # Dedicated vision slot for describe_image / describe_photo - # tool. Defaults to PRIMARY_MODEL when unset. -LLAMA_SWAP_EMBEDDING_MODEL=embed # Embedding slot id -LLAMA_SWAP_ALLOWED_MODELS=chat,coder # Curated allowlist surfaced by GET /insights/models - # when LLM_BACKEND=llamacpp. All report has_vision=true. - # Empty = picker shows only the configured primary model. -LLAMA_SWAP_REQUEST_TIMEOUT_SECONDS=180 # Per-request timeout; bump for slow CPU offload - -# Text-to-speech (Chatterbox served behind llama-swap). Only needs -# LLAMA_SWAP_URL — independent of LLM_BACKEND. Powers /tts/speech (read-aloud) -# and /tts/voices* (voice cloning). Reference audio is ffmpeg-normalized to WAV -# server-side, so any source format works. -LLAMA_SWAP_TTS_MODEL=chatterbox # TTS model id in config.yaml (default: chatterbox) -LLAMA_SWAP_TTS_VOICE=m # Default voice when /tts/speech omits one (optional) -LLAMA_SWAP_TTS_REF_SECONDS=30 # Max voice-clone reference clip length, seconds - # (Chatterbox is zero-shot; ~10-20s clean ref is ideal) -LLAMA_SWAP_TTS_REQUEST_TIMEOUT_SECONDS=600 # Per-request synth timeout (long chunked insights take - # minutes); overrides the shared client timeout for /tts/speech TTS_PRONUNCIATIONS_PATH=tts_pronunciations.json # JSON map of pronunciation overrides applied before synth # (see tts_pronunciations.example.json); hot-reloaded on change -# Insight Chat Continuation -AGENTIC_CHAT_MAX_ITERATIONS=6 # Cap on tool-calling iterations per chat turn (default 6) -AGENTIC_CHAT_DEFAULT_NUM_CTX=32768 # Assumed context window for the history-truncation budget - # when a chat request omits num_ctx AND the backend can't - # report its served window (llama-swap reports it via /props, - # so this mostly matters for Ollama/OpenRouter). Too small - # silently guts replayed history every turn (and destroys - # llama.cpp KV-cache prefix reuse). ``` +**LLM configuration (`llm.toml`, `src/ai/config.rs`):** + +Every LLM setting except the OpenRouter key lives in `llm.toml`; `llm.example.toml` +is the annotated schema (a test parses it). `ai::config::llm_config()` is a +process-wide `OnceLock` resolved as **defaults → `llm.toml` → legacy env vars**. +The legacy vars (`OLLAMA_*`, `LLAMA_SWAP_*`, `OPENROUTER_*` except the key, +`LLM_BACKEND`, `EMBEDDING_DIM`, `EMBED_*_PREFIX`, `AGENTIC_*`, +`UNIFIED_SEARCH_MODEL`, `INSIGHT_CHAT_TURN_TIMEOUT_SECS`) override the file for one +release, with a single startup deprecation warning naming the ones in use; +`LLAMA_SWAP_ALLOWED_MODELS` maps to `llamacpp.picker.include`. `main` loads it right +after tracing init, so a bad file (unknown key, zero timeout, `backend = +"llamacpp"` without `llamacpp.url`) fails startup. Readers call `llm_config()` +instead of `std::env::var`; the merge itself is a pure `resolve(file, env_fn)`, so +tests never touch the process environment. ADR-0003 records the split. + +Sections: `backend`, `unified_search_model`, `[llamacpp]` (`url`, `chat`, `vision`, +`timeout_secs`, `[llamacpp.picker]` `include`/`exclude`), `[ollama]`, +`[openrouter]`, `[embedding]` (`model`, `dim`, `query_prefix`, `document_prefix` — +the corpus contract: change any and run `reembed_embeddings`), `[tts]` +(`model`, `voice`, `ref_seconds`, `timeout_secs`), `[agentic]` (`max_iterations`, +`chat_max_iterations`, `chat_default_num_ctx`, `chat_turn_timeout_secs`). + +**llama-swap capabilities come from its `config.yaml`, not from here.** Each model +there carries `metadata: { kind: chat|embed|tts, vision, ctx, dim }`, which +`/v1/models` passes through (the deployed build merges it into +`meta.llamaswap`; `DiscoveredModel::from_json` also reads top-level `meta`). A model +without metadata is a text-only chat model. At startup a background task logs +`catalog_warnings`: role models that aren't served or lack the needed capability, +an embed `dim` that differs from `embedding.dim`, picker ids that aren't served, and +models with no metadata. + **AI Insights Fallback Behavior:** - Primary server is tried first with its configured model (5-second connection timeout) - On connection failure, automatically falls back to secondary server with its model (if configured) -- If `OLLAMA_FALLBACK_MODEL` not set, uses same model as primary server on fallback -- Total request timeout is 120 seconds to accommodate slow LLM inference +- If `ollama.fallback_model` is unset, uses same model as primary server on fallback +- Total request timeout is `ollama.timeout_secs` (default 120) to accommodate slow LLM inference - Logs indicate which server and model was used (info level) and failover attempts (warn level) -- Backwards compatible: `OLLAMA_URL` and `OLLAMA_MODEL` still supported as fallbacks +- Backwards compatible: legacy `OLLAMA_URL` / `OLLAMA_MODEL` env vars still map to the primary URL / model **Model Discovery:** The `OllamaClient` provides methods to query available models: @@ -850,25 +817,27 @@ The `OllamaClient` provides methods to query available models: This allows runtime verification of model availability before generating insights. -**Local backend switch (`LLM_BACKEND`):** +**Local backend switch (`backend` in llm.toml):** -One env var decides which "local" stack the server runs against — `ollama` +One setting decides which "local" stack the server runs against — `ollama` (default) or `llamacpp`. It's global on purpose: chat, vision, and embeddings all route through the same backend, so the embedding-vector column in SQLite stays in one vector space. Don't flip mid-deploy without re-embedding the affected rows — similarity search will collapse. -- `LLM_BACKEND=ollama`: chat, vision, and embeddings use Ollama. Vision +- `backend = "ollama"`: chat, vision, and embeddings use Ollama. Vision capability is probed per-model via `/api/show`. -- `LLM_BACKEND=llamacpp`: chat models receive images directly via OpenAI - content-parts (all models assumed vision-capable). Embeddings hit the - `embed` slot. A dedicated `LLAMA_SWAP_VISION_MODEL` slot (defaults to - the chat model) handles `describe_image` for the `describe_photo` tool. - Requires `LLAMA_SWAP_URL`. +- `backend = "llamacpp"`: chat models with `vision: true` metadata receive + images directly via OpenAI content-parts; text-only ones get the local + vision model's description inlined (the hybrid path), so a `--no-mmproj` + model never gets an image it can't read. If llama-swap can't be asked, + images are sent. Embeddings hit `embedding.model`. `llamacpp.vision` + (defaults to the chat model) handles `describe_image` for the + `describe_photo` tool. Requires `llamacpp.url`. The per-request `backend=hybrid` override is orthogonal: it always sends chat to OpenRouter (text-only, images are pre-described and inlined), but -the describe + embed passes still route through whichever `LLM_BACKEND` +the describe + embed passes still route through whichever `backend` is configured. **Backend dispatch (`ResolvedBackend`):** @@ -884,18 +853,20 @@ min_p per-request overrides. All downstream code (`execute_tool`, individual client references. `GET /insights/models` returns the local-backend models with capabilities -in the same envelope shape regardless of `LLM_BACKEND`: Ollama servers -when `ollama`, llama-swap slots (from `LLAMA_SWAP_ALLOWED_MODELS`) when -`llamacpp`. No `/insights/llamacpp/models` — the picker reads a single +in the same envelope shape regardless of `backend`: Ollama servers +when `ollama`; when `llamacpp`, llama-swap's models discovered live from +`/v1/models` (`picker_models`: every `kind: chat` model, `:sub`/`:fast` +aliases hidden, filtered by `[llamacpp.picker]`), with vision / context +window from metadata. No `/insights/llamacpp/models` — the picker reads a single endpoint. **Hybrid Backend (OpenRouter):** - Per-request opt-in via `backend=hybrid` on `POST /insights/generate/agentic`. - Vision describe happens before the agentic loop; the description is inlined into the chat prompt and the agentic loop runs on OpenRouter. Vision - routes through whichever `LLM_BACKEND` is configured. -- `request.model` (if provided) overrides `OPENROUTER_DEFAULT_MODEL` for that - call. The mobile picker reads from `OPENROUTER_ALLOWED_MODELS`. + routes through whichever `backend` is configured. +- `request.model` (if provided) overrides `openrouter.default_model` for that + call. The mobile picker reads from `openrouter.allowed`. - No live capability precheck — the operator-curated allowlist is trusted. A bad model id surfaces as a chat-call error. - `GET /insights/openrouter/models` returns `{ models, default_model, configured }` @@ -970,7 +941,7 @@ heuristic). The served window comes from `LlmClient::context_window()`: llama-swap models fix it at launch (`-c`) and ignore the request's `num_ctx`, so `LlamaCppClient` reads it from `/upstream//props` (cached 10 min). When neither is known the budget uses -`AGENTIC_CHAT_DEFAULT_NUM_CTX` (32768). Over budget, `apply_context_budget` +`agentic.chat_default_num_ctx` (32768). Over budget, `apply_context_budget` drops, oldest first: tool-call scaffolding in completed exchanges, then whole earlier exchanges (keeps user/assistant alternation), then the current turn's own tool results. The system prompt, the first user message @@ -981,12 +952,13 @@ server still rejects the prompt as too long, the budget is shrunk from its reported token counts and the call retried once. The `truncated` event / flag is surfaced to the client when a drop occurred. -Configurable env: -- `AGENTIC_CHAT_MAX_ITERATIONS` — cap on tool-calling iterations per turn +Configurable in `llm.toml` `[agentic]`: +- `chat_max_iterations` — cap on tool-calling iterations per turn (default 6). Per-request `max_iterations` is clamped to this cap. -- `AGENTIC_CHAT_DEFAULT_NUM_CTX` — assumed context window for the truncation +- `chat_default_num_ctx` — assumed context window for the truncation budget when the request omits `num_ctx` and the backend can't report its - served window (default 32768). + served window (default 32768). Too small silently guts replayed history + every turn (and destroys llama.cpp KV-cache prefix reuse). **Apollo Places integration (optional):** diff --git a/CONTEXT.md b/CONTEXT.md index 51c5040..e41f8ec 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -74,13 +74,14 @@ _Avoid_: user prefs, user settings, preferences (ambiguous with device-local cli **Deployment config**: Values the server needs before it can start serving — bind address, database path, media -roots, the URLs used to build clients — plus every secret. Lives in `.env` only; changing -it is a redeploy, by design. +roots, the URLs used to build clients — plus every secret. Lives in `.env`, except LLM +settings, which live in `llm.toml` (ADR-0003); secrets stay in `.env` only. Changing it +is a restart, by design. _Avoid_: environment config, system config **Hard default**: -The env/code fallback a request lands on when neither an explicit param nor a stored +The config/code fallback (`.env`, `llm.toml` or a code constant) a request lands on when neither an explicit param nor a stored owner default supplies the field (e.g. backend `local`, persona `default`, reel span -day). It is the floor under the owner-default layer: env changes keep working whenever a +day). It is the floor under the owner-default layer: config changes keep working whenever a default column is NULL. _Avoid_: fallback value, env default diff --git a/README.md b/README.md index 650d667..4384ff2 100644 --- a/README.md +++ b/README.md @@ -83,44 +83,41 @@ They should be defined where the binary is located or above it in an `.env` file ### AI Insights Configuration (Optional) -The following environment variables configure AI-powered photo insights and daily conversation summaries: +AI-powered photo insights, chat, read-aloud and daily conversation summaries are +configured in **`llm.toml`**, not `.env`. -#### Ollama Configuration -- `OLLAMA_PRIMARY_URL` - Primary Ollama server URL [default: `http://localhost:11434`] - - Example: `http://desktop:11434` (your main/powerful server) -- `OLLAMA_FALLBACK_URL` - Fallback Ollama server URL (optional) - - Example: `http://server:11434` (always-on backup server) -- `OLLAMA_PRIMARY_MODEL` - Model to use on primary server [default: `nemotron-3-nano:30b`] - - Example: `nemotron-3-nano:30b`, `llama3.2:3b`, etc. -- `OLLAMA_FALLBACK_MODEL` - Model to use on fallback server (optional) - - If not set, uses `OLLAMA_PRIMARY_MODEL` on fallback server +#### LLM configuration (`llm.toml`) +Copy `llm.example.toml` to `llm.toml` next to `.env` (or point `LLM_CONFIG_PATH` +at it). Every key is optional and the example documents each one: -**Legacy Variables** (still supported): -- `OLLAMA_URL` - Used if `OLLAMA_PRIMARY_URL` not set -- `OLLAMA_MODEL` - Used if `OLLAMA_PRIMARY_MODEL` not set +- `backend` — the local stack for chat, vision describe and embeddings: + `ollama` (default) or `llamacpp` (llama-swap). Global: don't flip it without + re-embedding. +- `[llamacpp]` — the llama-swap URL, the default `chat` model, the `vision` model + for `describe_photo`, and `[llamacpp.picker]` `exclude` / `include` filters. +- `[ollama]` — primary and fallback server URLs and models, request timeout. +- `[openrouter]` — hybrid backend: `default_model`, the curated `allowed` picker + list, `max_tokens`, attribution. The API key stays in env: + `OPENROUTER_API_KEY` enables the hybrid backend. +- `[embedding]` — embed model, `dim` and query/document prefixes. All of them + must match what the stored corpus was embedded with. +- `[tts]`, `[agentic]`, `unified_search_model` — see below and the example. -#### OpenRouter Configuration (Hybrid Backend) -The hybrid agentic backend keeps embeddings + vision local (Ollama) while routing -chat + tool-calling to OpenRouter. Enabled per-request when the client sends -`backend=hybrid`. +**The llama-swap model list is discovered, not configured.** `GET /insights/models` +lists every model llama-swap serves whose `metadata:` in llama-swap's +`config.yaml` says `kind: chat` (param aliases like `:sub` are hidden), with +`vision` and `ctx` from the same metadata. At startup the server logs a warning +for every role model, picker id or embed `dim` that doesn't match what +llama-swap serves. -- `OPENROUTER_API_KEY` - OpenRouter API key. Required to enable the hybrid backend. -- `OPENROUTER_DEFAULT_MODEL` - Model id used when the client doesn't specify one - [default: `anthropic/claude-sonnet-4`] - - Example: `openai/gpt-4o-mini`, `google/gemini-2.5-flash` -- `OPENROUTER_ALLOWED_MODELS` - Comma-separated curated allowlist exposed to - clients via `GET /insights/openrouter/models`. The mobile picker shows only - these. Empty/unset = no picker, server default is used. - - Example: `openai/gpt-4o-mini,anthropic/claude-haiku-4-5,google/gemini-2.5-flash` -- `OPENROUTER_BASE_URL` - Override base URL [default: `https://openrouter.ai/api/v1`] -- `OPENROUTER_EMBEDDING_MODEL` - Embedding model for OpenRouter - [default: `openai/text-embedding-3-small`]. Only used if/when embeddings are - routed through OpenRouter (currently embeddings stay local). -- `OPENROUTER_HTTP_REFERER` - Optional `HTTP-Referer` for OpenRouter attribution -- `OPENROUTER_APP_TITLE` - Optional `X-Title` for OpenRouter attribution +**Legacy env vars still work for now.** The old `OLLAMA_*`, `LLAMA_SWAP_*`, +`OPENROUTER_*` (except the key), `LLM_BACKEND`, `EMBEDDING_DIM`, `EMBED_*_PREFIX`, +`AGENTIC_*`, `UNIFIED_SEARCH_MODEL` and `INSIGHT_CHAT_TURN_TIMEOUT_SECS` override +`llm.toml`, with a startup deprecation warning listing the ones in use. +`LLAMA_SWAP_ALLOWED_MODELS` maps to `picker.include`. -Capability checks are skipped for the curated allowlist — bad model ids surface -as a 4xx from the chat call. Pick tool-capable models. +Capability checks are skipped for the curated OpenRouter list — bad model ids +surface as a 4xx from the chat call. Pick tool-capable models. #### SMS API Configuration - `SMS_API_URL` - URL to SMS message API [default: `http://localhost:8000`] @@ -128,7 +125,7 @@ as a 4xx from the chat call. Pick tool-capable models. - `SMS_API_TOKEN` - Authentication token for SMS API (optional) #### Agentic Insight Generation -- `AGENTIC_MAX_ITERATIONS` - Maximum tool-call iterations per agentic insight request [default: `10`] +- `agentic.max_iterations` - Maximum tool-call iterations per agentic insight request [default: `12`] - Controls how many times the model can invoke tools before being forced to produce a final answer - Increase for more thorough context gathering; decrease to limit response time @@ -147,13 +144,13 @@ Amend mode (`amend: true` in the chat request body) regenerates the insight's title and inserts a new row instead of appending to the existing transcript, so you can rewrite the saved summary from within chat. -- `AGENTIC_CHAT_MAX_ITERATIONS` - Cap on tool-calling iterations per chat turn [default: `6`] +- `agentic.chat_max_iterations` - Cap on tool-calling iterations per chat turn [default: `6`] - Per-request `max_iterations` (when sent by the client) is clamped to this cap #### Text-to-Speech (Optional) Reads insights aloud and manages cloned voices via a Chatterbox model served -behind the same llama-swap proxy. Only requires `LLAMA_SWAP_URL` (the TTS client -is built whenever that's set — independent of `LLM_BACKEND`). Endpoints: +behind the same llama-swap proxy. Only requires `llamacpp.url` (the TTS client +is built whenever that's set — independent of `backend`). Endpoints: - `POST /tts/speech` — body `{ text, voice?, format?, exaggeration?, cfg_weight?, temperature? }`; returns `{ audio_base64, format }`. Input is cleaned server-side (markdown + emoji stripped, then pronunciation overrides applied — @@ -189,21 +186,22 @@ matching rules are documented in `src/ai/pronunciation.rs`. Env: - `TTS_PRONUNCIATIONS_PATH` - pronunciation-override JSON file [default: `tts_pronunciations.json` in the working directory] -- `LLAMA_SWAP_TTS_MODEL` - TTS model id in llama-swap's `config.yaml` [default: `chatterbox`] -- `LLAMA_SWAP_TTS_VOICE` - default voice used when a `/tts/speech` request omits `voice` (optional) -- `LLAMA_SWAP_TTS_REF_SECONDS` - max voice-clone reference clip length in seconds + +`llm.toml` `[tts]`: +- `model` - TTS model id in llama-swap's `config.yaml` [default: `chatterbox`] +- `voice` - default voice used when a `/tts/speech` request omits `voice` (optional) +- `ref_seconds` - max voice-clone reference clip length in seconds [default: `30`]. Reference audio is ffmpeg-normalized to mono 24 kHz WAV (so any source format works); Chatterbox is zero-shot, so a clean ~10–20s sample is the sweet spot — more rarely helps. -- `LLAMA_SWAP_TTS_REQUEST_TIMEOUT_SECONDS` - per-request synthesis timeout in - seconds [default: `600`]. Long insights are chunked + synthesized server-side - and can take minutes; this is separate from (and overrides, for `/tts/speech`) - the shared `LLAMA_SWAP_REQUEST_TIMEOUT_SECONDS`. +- `timeout_secs` - per-request synthesis timeout in seconds [default: `600`]. Long + insights are chunked + synthesized server-side and can take minutes; this is + separate from (and overrides, for `/tts/speech`) `llamacpp.timeout_secs`. #### Fallback Behavior - Primary server is tried first with 5-second connection timeout - On failure, automatically falls back to secondary server (if configured) -- Total request timeout is 120 seconds to accommodate LLM inference +- Total request timeout is `ollama.timeout_secs` (default 120) to accommodate LLM inference - Logs indicate which server/model was used and any failover attempts #### Daily Summary Generation diff --git a/docs/adr/0003-llm-config-file.md b/docs/adr/0003-llm-config-file.md new file mode 100644 index 0000000..02137bf --- /dev/null +++ b/docs/adr/0003-llm-config-file.md @@ -0,0 +1,56 @@ +# ADR-0003: LLM deployment config moves to llm.toml; llama-swap owns model capabilities + +**Status**: accepted (2026-10-26). Amends ADR-0002's *deployment config* home for LLM settings. + +## Context + +ADR-0002 put LLM deployment config in `.env`: ~40 vars across `OLLAMA_*`, `LLAMA_SWAP_*`, +`OPENROUTER_*`, `LLM_BACKEND`, `EMBEDDING_DIM`, `EMBED_*_PREFIX`, `AGENTIC_*`. Flat env +vars can't express nested or list-shaped settings, and they duplicated facts that +llama-swap's `config.yaml` already owns. The deployed `LLAMA_SWAP_ALLOWED_MODELS` had +drifted from llama-swap: it listed three model ids that no longer existed (one a typo), +was missing three new ones, and every entry reported `has_vision: true`, including a +`--no-mmproj` text-only model. + +## Decision + +- **`llm.toml`** (TOML, `LLM_CONFIG_PATH`, default `./llm.toml`) holds every LLM + setting that ImageApi decides: the local `backend`, which model fills each role + (chat, vision, embedding, TTS, unified search), the embedding contract, the Ollama / + OpenRouter / TTS settings, agentic caps and the picker filters. Unknown keys fail startup. +- **Secrets stay in `.env`** (`OPENROUTER_API_KEY`), per ADR-0002. +- **llama-swap's `config.yaml` owns model capabilities.** Each model carries + `metadata: { kind, vision, ctx, dim }`, which `/v1/models` passes through. ImageApi + discovers the picker from it (every `kind: chat` model, param aliases hidden, minus + `picker.exclude`, optionally restricted to `picker.include`) instead of keeping a + hand-written allowlist. Text-only models get the vision model's description instead + of the image. +- **Startup cross-checks, warn-only.** Role models that aren't served or lack the needed + capability, and an embed `dim` that differs from `embedding.dim`, are logged. They + don't fail startup, because llama-swap may still be starting. +- **One release of env compatibility.** The legacy vars override the file, with one + deprecation warning listing the ones in use. Then they're removed. + +The resolution chain from ADR-0002 is unchanged: request param → owner default +(`user_ai_prefs`) → hard default. `llm.toml` replaces `.env` as the hard-default layer +for LLM fields. + +## Considered options + +- **A standalone ImageApi model list** (roles plus per-model vision/ctx in `llm.toml`): + easier to read than env, but still a second copy of llama-swap's facts, so it drifts + the same way. Rejected. +- **Reading llama-swap's `config.yaml` directly**: couples ImageApi to a file on another + host and to llama-swap's schema. `/v1/models` is the published interface. Rejected. +- **Owner defaults in the DB** (`user_ai_prefs`): right for per-request tuning, which + stays there, but URLs and the embedding contract are needed before serving. + Rejected for these fields, as in ADR-0002. + +## Non-obvious rules worth remembering + +- The deployed llama-swap merges `metadata` into `meta.llamaswap`, while upstream + documents it at `meta`. `DiscoveredModel::from_json` reads both. +- A model with no metadata is treated as a text-only chat model, and the startup check + lists it. +- `embedding.query_prefix` / `document_prefix` default to empty because the existing + corpus was embedded without them. Set them only together with `reembed_embeddings`. diff --git a/src/ai/handlers.rs b/src/ai/handlers.rs index d587be6..eb3d4aa 100644 --- a/src/ai/handlers.rs +++ b/src/ai/handlers.rs @@ -1089,7 +1089,7 @@ pub struct OpenRouterModelsResponse { /// GET /insights/openrouter/models - Curated OpenRouter model ids exposed /// to clients for the hybrid backend. Returned verbatim from -/// `OPENROUTER_ALLOWED_MODELS`; no live call to OpenRouter. +/// `openrouter.allowed`; no live call to OpenRouter. #[get("/insights/openrouter/models")] pub async fn get_openrouter_models_handler( _claims: Claims, diff --git a/src/ai/insight_chat.rs b/src/ai/insight_chat.rs index 7e30f76..2f52fab 100644 --- a/src/ai/insight_chat.rs +++ b/src/ai/insight_chat.rs @@ -2164,7 +2164,7 @@ fn resolve_date_taken_for_context( /// symmetry and future rules. /// /// Whether "local" routes through Ollama or llama-swap is decided at -/// startup by `LLM_BACKEND`; both share the same transcript shape from +/// startup by llm.toml's `backend`; both share the same transcript shape from /// the chat-replay perspective. fn validate_cross_replay(_stored: &str, effective: &str) -> Result<()> { if !matches!(effective, "local" | "hybrid") { @@ -3829,7 +3829,7 @@ mod tests { #[test] fn bootstrap_backend_rejects_unknown_label() { // `llamacpp` is no longer a per-request backend value — it's chosen - // at deploy time via `LLM_BACKEND`. + // at deploy time via llm.toml's `backend`. for label in &["openrouter", "llamacpp", "ollama"] { let err = resolve_bootstrap_backend(Some(label)).unwrap_err(); let msg = format!("{}", err); diff --git a/src/ai/insight_generator.rs b/src/ai/insight_generator.rs index eb95370..0643b2c 100644 --- a/src/ai/insight_generator.rs +++ b/src/ai/insight_generator.rs @@ -234,7 +234,7 @@ pub struct InsightGenerator { /// `None` when `OPENROUTER_API_KEY` is not configured. openrouter: Option>, /// Optional llama-swap client, used when `backend=llamacpp` is requested. - /// `None` when `LLAMA_SWAP_URL` is not configured. + /// `None` when `llamacpp.url` is not configured. llamacpp: Option>, sms_client: SmsApiClient, /// Optional integration with Apollo's user-defined Places. When the @@ -735,7 +735,7 @@ impl InsightGenerator { log::info!("========================================"); // Generate embedding for the query via the configured local backend - // (`LLM_BACKEND` switch). Must match the backend that populated the + // (llm.toml `backend` switch). Must match the backend that populated the // daily-summary embeddings or similarity search will be garbage. let query_embedding = crate::ai::embed_query(&self.ollama, self.llamacpp.as_deref(), &query).await?; @@ -3331,7 +3331,7 @@ Return ONLY the summary, nothing else."#, } /// Tool: store_entity — upsert an entity into the knowledge memory. - /// Embeddings go through the configured local backend (`LLM_BACKEND`), + /// Embeddings go through the configured local backend (llm.toml `backend`), /// independent of the per-request chat backend in the caller. async fn tool_store_entity(&self, args: &serde_json::Value) -> String { use crate::database::models::InsertEntity; diff --git a/src/ai/llamacpp.rs b/src/ai/llamacpp.rs index 0e7de7a..f3efe79 100644 --- a/src/ai/llamacpp.rs +++ b/src/ai/llamacpp.rs @@ -69,11 +69,11 @@ pub struct LlamaCppClient { pub embedding_model: String, /// Vision model slot id. Used for `describe_image` routing. Defaults /// to `primary_model` so describe_image works out of the box; override - /// via `LLAMA_SWAP_VISION_MODEL` for a dedicated vision slot. + /// via `llamacpp.vision` for a dedicated vision slot. pub vision_model: String, /// TTS model slot id (e.g. `"chatterbox"`). Routes `text_to_speech` and /// is the `/upstream//voices` path segment for the voice library. - /// Override via `LLAMA_SWAP_TTS_MODEL`. + /// Set via `tts.model`. pub tts_model: String, num_ctx: Option, temperature: Option, @@ -1778,7 +1778,7 @@ mod tests { // In hybrid mode, overrides.model is an OpenRouter model id // (e.g. "google/gemini-3-flash-preview"). The local llamacpp // client must NOT adopt that as its vision_model — it should - // keep the configured LLAMA_SWAP_VISION_MODEL so describe_image + // keep the configured `llamacpp.vision` so describe_image // hits the correct local slot instead of sending an unknown // model name to llama-swap. let mut base = LlamaCppClient::new(None, Some("chat".into())); diff --git a/src/ai/local_llm.rs b/src/ai/local_llm.rs index ad5c96e..f2ff2a4 100644 --- a/src/ai/local_llm.rs +++ b/src/ai/local_llm.rs @@ -1,10 +1,10 @@ //! Bundle of the local LLM pair (Ollama + optional llama-swap) with the -//! `LLM_BACKEND` dispatch baked in. +//! `backend` (llm.toml) dispatch baked in. //! //! Exists because passing the pair around as loose values invited the same //! bug three times: import/backfill tooling embedded corpora via //! `OllamaClient` directly while the query side dispatched through -//! `embed_one`, so flipping `LLM_BACKEND=llamacpp` silently split queries +//! `embed_one`, so flipping `backend = "llamacpp"` silently split queries //! and corpus into different vector spaces. Anything that writes or reads //! embeddings should go through this type (or `embed_one`/`embed_many`), //! never a concrete client. diff --git a/src/ai/ollama.rs b/src/ai/ollama.rs index de70864..233e569 100644 --- a/src/ai/ollama.rs +++ b/src/ai/ollama.rs @@ -1163,7 +1163,7 @@ Analyze the image and use specific details from both the visual content and the } }; - // Validate embedding dimensions (EMBEDDING_DIM; 768 for nomic-embed-text:v1.5) + // Validate embedding dimensions (`embedding.dim`; 768 for nomic-embed-text:v1.5) for (i, embedding) in embeddings.iter().enumerate() { if embedding.len() != crate::ai::embedding_dim() { log::warn!( diff --git a/src/ai/openrouter.rs b/src/ai/openrouter.rs index ae0c544..74814d5 100644 --- a/src/ai/openrouter.rs +++ b/src/ai/openrouter.rs @@ -26,7 +26,7 @@ const CACHE_DURATION_SECS: u64 = 15 * 60; /// Default completion cap sent as `max_tokens`. OpenRouter pre-authorizes /// credit against the requested max (defaulting to the model's full output /// window when omitted), so expensive models fail with credit-limit errors -/// even when the actual reply is short. Override via `OPENROUTER_MAX_TOKENS`. +/// even when the actual reply is short. Override via `openrouter.max_tokens`. pub const DEFAULT_MAX_TOKENS: u32 = 16384; #[derive(Clone)] diff --git a/src/ai/tts.rs b/src/ai/tts.rs index 53183b8..7cc3c5b 100644 --- a/src/ai/tts.rs +++ b/src/ai/tts.rs @@ -391,7 +391,7 @@ pub async fn tts_speech_handler( let Some(client) = app_state.llamacpp.as_ref() else { span.set_status(Status::error("tts backend not configured")); return HttpResponse::ServiceUnavailable() - .json(json!({ "error": "TTS backend not configured (set LLAMA_SWAP_URL)" })); + .json(json!({ "error": "TTS backend not configured (set llamacpp.url in llm.toml)" })); }; let format = req @@ -521,7 +521,7 @@ pub async fn create_speech_job_handler( if app_state.llamacpp.is_none() { span.set_status(Status::error("tts backend not configured")); return HttpResponse::ServiceUnavailable() - .json(json!({ "error": "TTS backend not configured (set LLAMA_SWAP_URL)" })); + .json(json!({ "error": "TTS backend not configured (set llamacpp.url in llm.toml)" })); } let format = req @@ -958,7 +958,7 @@ pub struct CreateVoiceFromLibraryRequest { /// lets the client pick the clean-speech section of a long recording. #[serde(default)] pub start_seconds: Option, - /// Reference window length; clamped to LLAMA_SWAP_TTS_REF_SECONDS. + /// Reference window length; clamped to `tts.ref_seconds`. #[serde(default)] pub duration_seconds: Option, } @@ -966,7 +966,7 @@ pub struct CreateVoiceFromLibraryRequest { /// POST /tts/voices/from-library — register a cloned voice from a file already /// in a library. Audio and video alike are ffmpeg-normalized to a mono 24 kHz /// WAV reference clip (window selected by start/duration_seconds, length -/// capped by LLAMA_SWAP_TTS_REF_SECONDS). +/// capped by `tts.ref_seconds`). #[post("/tts/voices/from-library")] #[tracing::instrument(name = "http.tts.voices.from_library", skip_all)] pub async fn create_voice_from_library_handler( @@ -1153,7 +1153,7 @@ mod tests { #[test] fn resolve_ref_window_defaults_to_start_of_clip_at_cap_length() { - // Reads the live cap rather than mutating LLAMA_SWAP_TTS_REF_SECONDS: + // Reads the live cap rather than mutating `tts.ref_seconds`: // env mutation flakes under the parallel suite (see env_dispatch). let cap = f64::from(tts_ref_seconds()); assert_eq!(resolve_ref_window(None, None), Ok((0.0, cap))); diff --git a/src/bin/import_calendar.rs b/src/bin/import_calendar.rs index 2d140b3..b8fded8 100644 --- a/src/bin/import_calendar.rs +++ b/src/bin/import_calendar.rs @@ -42,7 +42,7 @@ async fn main() -> Result<()> { info!("Found {} calendar events", events.len()); - // LocalLlm dispatches per LLM_BACKEND, so embeddings written here land + // LocalLlm dispatches per llm.toml's backend, so embeddings written here land // in the same vector space the query side searches. let llm = if args.generate_embeddings { Some(LocalLlm::from_config()) diff --git a/src/bin/import_search_history.rs b/src/bin/import_search_history.rs index 566c59d..020bf8c 100644 --- a/src/bin/import_search_history.rs +++ b/src/bin/import_search_history.rs @@ -38,7 +38,7 @@ async fn main() -> Result<()> { info!("Found {} search records", searches.len()); - // LocalLlm dispatches per LLM_BACKEND, so embeddings written here land + // LocalLlm dispatches per llm.toml's backend, so embeddings written here land // in the same vector space the query side searches. let llm = LocalLlm::from_config(); diff --git a/src/bin/populate_knowledge.rs b/src/bin/populate_knowledge.rs index 9acd91d..f74771b 100644 --- a/src/bin/populate_knowledge.rs +++ b/src/bin/populate_knowledge.rs @@ -35,7 +35,7 @@ struct Args { #[arg(long)] path: Option, - /// Ollama model override. Defaults to OLLAMA_PRIMARY_MODEL from .env + /// Ollama model override. Defaults to ollama.primary_model from llm.toml #[arg(long)] model: Option, diff --git a/src/bin/reembed_embeddings.rs b/src/bin/reembed_embeddings.rs index 67e72f2..936ba7c 100644 --- a/src/bin/reembed_embeddings.rs +++ b/src/bin/reembed_embeddings.rs @@ -1,7 +1,7 @@ //! Re-embed stored corpora through `LocalLlm`, i.e. the same -//! `LLM_BACKEND` dispatch the query side uses. The original import / +//! `backend` (llm.toml) dispatch the query side uses. The original import / //! backfill tools always embedded via Ollama, so a deploy running -//! `LLM_BACKEND=llamacpp` queries vector spaces the corpora may not live +//! `backend = "llamacpp"` queries vector spaces the corpora may not live //! in. Three tables share the problem and are all covered here: //! //! - `daily_conversation_summaries` — re-embeds @@ -30,7 +30,7 @@ use image_api::bin_progress; use std::env; #[derive(Parser, Debug)] -#[command(author, version, about = "Re-embed stored corpora via the configured LLM_BACKEND", long_about = None)] +#[command(author, version, about = "Re-embed stored corpora via the configured local backend", long_about = None)] struct Args { /// Comma-separated tables to process: summaries, calendar, search, entities #[arg(long, default_value = "summaries,calendar,search,entities")] @@ -195,12 +195,12 @@ async fn reembed_table( }; // The whole pipeline (DAO checks, stored corpora) assumes - // EMBEDDING_DIM dims. A mismatch means the active embed slot is not + // embedding.dim dims. A mismatch means the active embed slot is not // serving the configured model — stop rather than corrupt the table. anyhow::ensure!( new_emb.len() == image_api::ai::embedding_dim(), "backend returned {}-dim embedding (expected {}) — '{}' does not \ - match the configured EMBEDDING_DIM", + match the configured embedding.dim", new_emb.len(), image_api::ai::embedding_dim(), llm.embedding_model_version() diff --git a/src/reels/mod.rs b/src/reels/mod.rs index 6a5a68f..b4ddc6d 100644 --- a/src/reels/mod.rs +++ b/src/reels/mod.rs @@ -157,7 +157,7 @@ pub enum ScripterMode { /// Fast path: single LLM call via the direct client. Fast, /// Agentic path: resolves the backend through the InsightGenerator - /// (honouring LLM_BACKEND, model overrides, etc.). Falls back to + /// (honouring llm.toml's backend, model overrides, etc.). Falls back to /// Fast on error so a scripting failure never sinks a reel. Agentic, } @@ -450,7 +450,7 @@ pub async fn create_reel_handler( ) -> impl Responder { if app_state.llamacpp.is_none() { return HttpResponse::ServiceUnavailable().json(json!({ - "error": "Reel narration needs the LLM/TTS backend (set LLAMA_SWAP_URL)" + "error": "Reel narration needs the LLM/TTS backend (set llamacpp.url in llm.toml)" })); } diff --git a/src/reels/script.rs b/src/reels/script.rs index 8fff21a..ef1c813 100644 --- a/src/reels/script.rs +++ b/src/reels/script.rs @@ -289,7 +289,7 @@ pub async fn generate_script( } /// Agentic version of script generation: resolves the backend via the -/// InsightGenerator (honouring LLM_BACKEND, model overrides, etc.), builds +/// InsightGenerator (honouring llm.toml's backend, model overrides, etc.), builds /// a read-only tool set, runs the tool loop, then parses the JSON response. /// Returns the same ReelScript shape. On failure the caller may fall back to /// `generate_script`. diff --git a/src/state.rs b/src/state.rs index a5da3f1..3bb1518 100644 --- a/src/state.rs +++ b/src/state.rs @@ -70,9 +70,9 @@ pub struct AppState { #[allow(dead_code)] pub openrouter: Option>, /// Curated list of OpenRouter model ids exposed to clients. Sourced from - /// `OPENROUTER_ALLOWED_MODELS` (comma-separated). Empty when unset. + /// `openrouter.allowed`. Empty when unset. pub openrouter_allowed_models: Vec, - /// `None` when `LLAMA_SWAP_URL` is not configured. Consulted only when a + /// `None` when `llamacpp.url` is not configured. Consulted only when a /// request explicitly opts into `backend=llamacpp`. Same shape as the /// `openrouter` slot — present here so handlers can route to it without /// threading through the generator. diff --git a/src/unified_search.rs b/src/unified_search.rs index 23a85ba..a9346d3 100644 --- a/src/unified_search.rs +++ b/src/unified_search.rs @@ -635,7 +635,7 @@ async fn translate( } }; - // Respect env/config for the LLM backend (LLM_BACKEND → ollama or + // Respect env/config for the LLM backend (llm.toml `backend` → ollama or // llama-swap); local only, no hybrid, per the feature's design. // // Translation-model precedence: