docs: document llm.toml and llama-swap metadata discovery

README, CLAUDE.md and .env.example now point at llm.toml for LLM settings
(OPENROUTER_API_KEY stays in env) and describe the legacy env override
window, the discovered picker and the startup cross-check. ADR-0003 records
the move and amends ADR-0002's deployment-config home; CONTEXT.md's
Deployment config / Hard default entries follow. Source comments and the
"set LLAMA_SWAP_URL" error strings name the llm.toml keys.

Also fixes README's AGENTIC_MAX_ITERATIONS default (10 -> 12, what the
code uses).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Cameron CordesandClaude Opus 5.5 committed 2026-10-26 17:40:42 -04:00
1 parent d68d9b09f3
commit 09b9e87396
21 files changed
+209 -244

No files matched your search

+9 -71
View File
@@ -40,78 +40,16 @@ PLAYLIST_CLEANUP_INTERVAL_SECONDS=86400
# RUST_LOG also gates export: keep it at info or debug. # RUST_LOG also gates export: keep it at info or debug.
# OTLP_OTLS_ENDPOINT=http://localhost:4317 # OTLP_OTLS_ENDPOINT=http://localhost:4317
# ── AI Insights — Ollama (local LLM) ──────────────────────────────────── # ── AI Insights — LLMs ──────────────────────────────────────────────────
OLLAMA_PRIMARY_URL=http://localhost:11434 # Models, backends, the embedding contract, TTS and agentic caps live in
OLLAMA_PRIMARY_MODEL=nemotron-3-nano:30b # llm.toml — copy llm.example.toml. Only the secret stays here.
# Optional fallback server tried on connection failure. # LLM_CONFIG_PATH=./llm.toml
# OLLAMA_FALLBACK_URL=http://server:11434 # Enables the hybrid backend (chat via OpenRouter; photos stay local).
# OLLAMA_FALLBACK_MODEL=llama3.2:3b
OLLAMA_REQUEST_TIMEOUT_SECONDS=120
# Cap on tool-calling iterations per chat turn / agentic insight.
AGENTIC_MAX_ITERATIONS=6
AGENTIC_CHAT_MAX_ITERATIONS=6
# ── AI Insights — OpenRouter (hybrid backend, optional) ─────────────────
# Set OPENROUTER_API_KEY to enable the hybrid backend (vision stays
# local on Ollama, chat routes to OpenRouter).
# OPENROUTER_API_KEY=sk-or-... # OPENROUTER_API_KEY=sk-or-...
# OPENROUTER_DEFAULT_MODEL=anthropic/claude-sonnet-4 #
# OPENROUTER_ALLOWED_MODELS=openai/gpt-4o-mini,anthropic/claude-haiku-4-5,google/gemini-2.5-flash # The old OLLAMA_* / LLAMA_SWAP_* / OPENROUTER_* / LLM_BACKEND / EMBED* /
# OPENROUTER_BASE_URL=https://openrouter.ai/api/v1 # AGENTIC_* / UNIFIED_SEARCH_MODEL vars still override llm.toml for now, with a
# OPENROUTER_EMBEDDING_MODEL=openai/text-embedding-3-small # startup deprecation warning. Move them into llm.toml.
# OPENROUTER_HTTP_REFERER=https://your-site.example
# OPENROUTER_APP_TITLE=ImageApi
# Cap on generated tokens per call (sent as max_tokens). OpenRouter reserves
# credit against the requested max, so leaving it uncapped makes pricey
# models fail with credit-limit errors. Default 16384; 0 = no cap.
# OPENROUTER_MAX_TOKENS=16384
# ── AI Insights — local backend switch ──────────────────────────────────
# Picks which local LLM stack the server uses for chat, vision describe,
# and embeddings. `ollama` (default) uses the OLLAMA_* settings above;
# `llamacpp` uses the LLAMA_SWAP_* settings below. The switch is global
# and applies to both `backend=local` and `backend=hybrid` (hybrid keeps
# chat on OpenRouter but still uses this stack for the describe pass).
# Don't flip mid-deploy without re-embedding existing index rows —
# mixed vector spaces break similarity search.
# LLM_BACKEND=ollama
# ── AI Insights — llama.cpp / llama-swap (optional) ─────────────────────
# Set LLAMA_SWAP_URL plus LLM_BACKEND=llamacpp to swap the local stack
# off Ollama. Talks OpenAI-compatible /v1 to a llama-swap proxy fronting
# per-slot llama-server instances. Chat models receive images directly
# via content-parts (vision-capable models assumed); a separate vision
# slot is used only by the describe_photo tool and describe-image utility.
# LLAMA_SWAP_URL=http://localhost:9292/v1
# LLAMA_SWAP_PRIMARY_MODEL=chat
# Optional dedicated vision slot for describe_image. Defaults to
# PRIMARY_MODEL so describe_photo works without extra config.
# LLAMA_SWAP_VISION_MODEL=vision
# LLAMA_SWAP_EMBEDDING_MODEL=embed
# Comma-separated allowlist surfaced by /insights/models when
# LLM_BACKEND=llamacpp. All report has_vision=true.
# LLAMA_SWAP_ALLOWED_MODELS=chat,vision,embed
# LLAMA_SWAP_REQUEST_TIMEOUT_SECONDS=180
# ── Unified search translation model (optional) ─────────────────────────
# /photos/search/unified runs one small LLM call to translate a natural-
# language query into structured filters + a semantic term, then CLIP-ranks.
# That step needs an LLM AND CLIP available at once. On a tight VRAM budget a
# large chat model can't co-reside with CLIP, so pin a small, fast model here
# (it can stay loaded alongside CLIP and the chat model). Precedence:
# UNIFIED_SEARCH_MODEL > the client's selected model > the configured default.
# Use the configured backend (LLM_BACKEND); local only — no hybrid.
# UNIFIED_SEARCH_MODEL=qwen3-0.6b
# ── Text-to-speech (optional, requires LLAMA_SWAP_URL) ───────────────────
# TTS routes through the same llama-swap proxy (a Chatterbox model id), so it
# only needs LLAMA_SWAP_URL — it does NOT require LLM_BACKEND=llamacpp.
# Powers POST /tts/speech and the /tts/voices* endpoints (read-aloud insights
# + voice cloning in the mobile app).
# LLAMA_SWAP_TTS_MODEL=chatterbox # TTS model id in config.yaml
# LLAMA_SWAP_TTS_VOICE=m # default voice when a request omits one
# LLAMA_SWAP_TTS_REF_SECONDS=30 # max voice-clone reference clip length (s)
# LLAMA_SWAP_TTS_REQUEST_TIMEOUT_SECONDS=600 # synth timeout (long chunked text)
# ── AI Insights — sibling services (optional) ─────────────────────────── # ── AI Insights — sibling services (optional) ───────────────────────────
# Apollo (places, face inference, CLIP encoders). Single-Apollo deploys # Apollo (places, face inference, CLIP encoders). Single-Apollo deploys
+64 -92
View File
@@ -561,11 +561,11 @@ GET /memories?path=...&recursive=true
POST /insights/generate (non-agentic single-shot) POST /insights/generate (non-agentic single-shot)
POST /insights/generate/agentic (tool-calling loop; body: { file_path, backend?, model?, ... }) POST /insights/generate/agentic (tool-calling loop; body: { file_path, backend?, model?, ... })
GET /insights?path=...&library=... GET /insights?path=...&library=...
GET /insights/models (local-backend models + capabilities; Ollama OR llama-swap based on LLM_BACKEND) GET /insights/models (local-backend models + capabilities; Ollama OR llama-swap per llm.toml `backend`)
GET /insights/openrouter/models (curated OpenRouter allowlist) GET /insights/openrouter/models (curated OpenRouter allowlist)
POST /insights/rate (thumbs up/down for training data) POST /insights/rate (thumbs up/down for training data)
// Text-to-Speech (Chatterbox via llama-swap; needs LLAMA_SWAP_URL) // Text-to-Speech (Chatterbox via llama-swap; needs llamacpp.url)
POST /tts/speech (read-aloud: { text, voice?, ... } -> { audio_base64, format }) POST /tts/speech (read-aloud: { text, voice?, ... } -> { audio_base64, format })
GET /tts/voices (Chatterbox voice library) GET /tts/voices (Chatterbox voice library)
POST /tts/voices/upload (clone a voice from an uploaded clip; multipart) POST /tts/voices/upload (clone a voice from an uploaded clip; multipart)
@@ -745,12 +745,9 @@ DATE_BACKFILL_MAX_PER_TICK=500 # Cap on canonical-date drain per watcher
HLS_COPY_MAX_BITRATE=20000000 # Max source bits/s to stream-copy into HLS; above it (or >1080p) transcodes HLS_COPY_MAX_BITRATE=20000000 # Max source bits/s to stream-copy into HLS; above it (or >1080p) transcodes
OTLP_OTLS_ENDPOINT=http://... # OTLP collector for span export (any build; unset = no export) OTLP_OTLS_ENDPOINT=http://... # OTLP collector for span export (any build; unset = no export)
# AI Insights Configuration # AI Insights: LLM settings live in llm.toml (see "LLM configuration" below)
OLLAMA_PRIMARY_URL=http://desktop:11434 # Primary Ollama server (e.g., desktop) LLM_CONFIG_PATH=./llm.toml # Default ./llm.toml; missing default file = env + defaults
OLLAMA_FALLBACK_URL=http://server:11434 # Fallback Ollama server (optional, always-on) OPENROUTER_API_KEY=sk-or-... # Secret, env-only. Enables the hybrid backend
OLLAMA_PRIMARY_MODEL=nemotron-3-nano:30b # Model for primary server (default: nemotron-3-nano:30b)
OLLAMA_FALLBACK_MODEL=llama3.2:3b # Model for fallback server (optional, uses primary if not set)
OLLAMA_REQUEST_TIMEOUT_SECONDS=120 # Per-request generation timeout (default 120). Increase for slow CPU-offloaded models.
SMS_API_URL=http://localhost:8000 # SMS message API endpoint (default: localhost:8000) SMS_API_URL=http://localhost:8000 # SMS message API endpoint (default: localhost:8000)
SMS_API_TOKEN=your-api-token # SMS API authentication token (optional) SMS_API_TOKEN=your-api-token # SMS API authentication token (optional)
@@ -769,79 +766,49 @@ FACE_AUTOBIND_MIN_COS=0.4 # Phase 3: cosine-sim floor for
FACE_DETECT_CONCURRENCY=8 # Phase 3: per-scan-tick parallel detect calls FACE_DETECT_CONCURRENCY=8 # Phase 3: per-scan-tick parallel detect calls
FACE_DETECT_TIMEOUT_SEC=60 # reqwest client timeout (CPU inference can be slow) FACE_DETECT_TIMEOUT_SEC=60 # reqwest client timeout (CPU inference can be slow)
# OpenRouter (Hybrid Backend) - keeps embeddings + vision local, routes chat to OpenRouter
OPENROUTER_API_KEY=sk-or-... # Required to enable hybrid backend
OPENROUTER_DEFAULT_MODEL=anthropic/claude-sonnet-4 # Used when client doesn't pick a model
OPENROUTER_ALLOWED_MODELS=openai/gpt-4o-mini,anthropic/claude-haiku-4-5,google/gemini-2.5-flash
# Curated allowlist exposed to clients via
# GET /insights/openrouter/models. Empty = no picker.
OPENROUTER_BASE_URL=https://openrouter.ai/api/v1 # Override base URL (optional)
OPENROUTER_EMBEDDING_MODEL=openai/text-embedding-3-small # Optional, embeddings stay local today
OPENROUTER_HTTP_REFERER=https://your-site.example # Optional attribution header
OPENROUTER_APP_TITLE=ImageApi # Optional attribution header
OPENROUTER_MAX_TOKENS=16384 # Per-call max_tokens cap (default 16384; 0 = uncapped).
# OpenRouter reserves credit against this, so an uncapped
# request to a pricey model can fail on credit limits.
# Local LLM backend switch. `ollama` (default) keeps the OLLAMA_* settings
# above; `llamacpp` swaps the entire local stack (chat + vision describe +
# embeddings) over to llama-swap. The switch is global and applies to
# `backend=local` requests and to `backend=hybrid`'s describe pass (hybrid
# chat still goes to OpenRouter). Don't flip mid-deploy without
# re-embedding — mixed vector spaces break similarity search.
LLM_BACKEND=ollama
# Embedding model contract. Corpus and queries must be embedded by the same
# model with matching prefixes — after changing the embed model or any of
# these, run `cargo run --bin reembed_embeddings` (all tables) or search is
# garbage. Prefix values may contain a literal \n (expanded to a newline).
EMBEDDING_DIM=768 # 768 = nomic-embed-text v1.5; 1024 = Qwen3-Embedding-0.6B
EMBED_QUERY_PREFIX= # nomic: "search_query: " | Qwen3: "Instruct: <task>\nQuery: "
EMBED_DOCUMENT_PREFIX= # nomic: "search_document: " | Qwen3: leave empty
# llama.cpp / llama-swap (used when LLM_BACKEND=llamacpp). OpenAI-compatible
# proxy hosting one or more llama-server processes. Chat models receive
# images directly via content-parts (all models assumed vision-capable).
LLAMA_SWAP_URL=http://localhost:9292/v1 # Required when LLM_BACKEND=llamacpp
LLAMA_SWAP_PRIMARY_MODEL=chat # Chat slot id (matches config.yaml)
LLAMA_SWAP_VISION_MODEL= # Dedicated vision slot for describe_image / describe_photo
# tool. Defaults to PRIMARY_MODEL when unset.
LLAMA_SWAP_EMBEDDING_MODEL=embed # Embedding slot id
LLAMA_SWAP_ALLOWED_MODELS=chat,coder # Curated allowlist surfaced by GET /insights/models
# when LLM_BACKEND=llamacpp. All report has_vision=true.
# Empty = picker shows only the configured primary model.
LLAMA_SWAP_REQUEST_TIMEOUT_SECONDS=180 # Per-request timeout; bump for slow CPU offload
# Text-to-speech (Chatterbox served behind llama-swap). Only needs
# LLAMA_SWAP_URL — independent of LLM_BACKEND. Powers /tts/speech (read-aloud)
# and /tts/voices* (voice cloning). Reference audio is ffmpeg-normalized to WAV
# server-side, so any source format works.
LLAMA_SWAP_TTS_MODEL=chatterbox # TTS model id in config.yaml (default: chatterbox)
LLAMA_SWAP_TTS_VOICE=m # Default voice when /tts/speech omits one (optional)
LLAMA_SWAP_TTS_REF_SECONDS=30 # Max voice-clone reference clip length, seconds
# (Chatterbox is zero-shot; ~10-20s clean ref is ideal)
LLAMA_SWAP_TTS_REQUEST_TIMEOUT_SECONDS=600 # Per-request synth timeout (long chunked insights take
# minutes); overrides the shared client timeout for /tts/speech
TTS_PRONUNCIATIONS_PATH=tts_pronunciations.json # JSON map of pronunciation overrides applied before synth TTS_PRONUNCIATIONS_PATH=tts_pronunciations.json # JSON map of pronunciation overrides applied before synth
# (see tts_pronunciations.example.json); hot-reloaded on change # (see tts_pronunciations.example.json); hot-reloaded on change
# Insight Chat Continuation
AGENTIC_CHAT_MAX_ITERATIONS=6 # Cap on tool-calling iterations per chat turn (default 6)
AGENTIC_CHAT_DEFAULT_NUM_CTX=32768 # Assumed context window for the history-truncation budget
# when a chat request omits num_ctx AND the backend can't
# report its served window (llama-swap reports it via /props,
# so this mostly matters for Ollama/OpenRouter). Too small
# silently guts replayed history every turn (and destroys
# llama.cpp KV-cache prefix reuse).
``` ```
**LLM configuration (`llm.toml`, `src/ai/config.rs`):**
Every LLM setting except the OpenRouter key lives in `llm.toml`; `llm.example.toml`
is the annotated schema (a test parses it). `ai::config::llm_config()` is a
process-wide `OnceLock` resolved as **defaults → `llm.toml` → legacy env vars**.
The legacy vars (`OLLAMA_*`, `LLAMA_SWAP_*`, `OPENROUTER_*` except the key,
`LLM_BACKEND`, `EMBEDDING_DIM`, `EMBED_*_PREFIX`, `AGENTIC_*`,
`UNIFIED_SEARCH_MODEL`, `INSIGHT_CHAT_TURN_TIMEOUT_SECS`) override the file for one
release, with a single startup deprecation warning naming the ones in use;
`LLAMA_SWAP_ALLOWED_MODELS` maps to `llamacpp.picker.include`. `main` loads it right
after tracing init, so a bad file (unknown key, zero timeout, `backend =
"llamacpp"` without `llamacpp.url`) fails startup. Readers call `llm_config()`
instead of `std::env::var`; the merge itself is a pure `resolve(file, env_fn)`, so
tests never touch the process environment. ADR-0003 records the split.
Sections: `backend`, `unified_search_model`, `[llamacpp]` (`url`, `chat`, `vision`,
`timeout_secs`, `[llamacpp.picker]` `include`/`exclude`), `[ollama]`,
`[openrouter]`, `[embedding]` (`model`, `dim`, `query_prefix`, `document_prefix` —
the corpus contract: change any and run `reembed_embeddings`), `[tts]`
(`model`, `voice`, `ref_seconds`, `timeout_secs`), `[agentic]` (`max_iterations`,
`chat_max_iterations`, `chat_default_num_ctx`, `chat_turn_timeout_secs`).
**llama-swap capabilities come from its `config.yaml`, not from here.** Each model
there carries `metadata: { kind: chat|embed|tts, vision, ctx, dim }`, which
`/v1/models` passes through (the deployed build merges it into
`meta.llamaswap`; `DiscoveredModel::from_json` also reads top-level `meta`). A model
without metadata is a text-only chat model. At startup a background task logs
`catalog_warnings`: role models that aren't served or lack the needed capability,
an embed `dim` that differs from `embedding.dim`, picker ids that aren't served, and
models with no metadata.
**AI Insights Fallback Behavior:** **AI Insights Fallback Behavior:**
- Primary server is tried first with its configured model (5-second connection timeout) - Primary server is tried first with its configured model (5-second connection timeout)
- On connection failure, automatically falls back to secondary server with its model (if configured) - On connection failure, automatically falls back to secondary server with its model (if configured)
- If `OLLAMA_FALLBACK_MODEL` not set, uses same model as primary server on fallback - If `ollama.fallback_model` is unset, uses same model as primary server on fallback
- Total request timeout is 120 seconds to accommodate slow LLM inference - Total request timeout is `ollama.timeout_secs` (default 120) to accommodate slow LLM inference
- Logs indicate which server and model was used (info level) and failover attempts (warn level) - Logs indicate which server and model was used (info level) and failover attempts (warn level)
- Backwards compatible: `OLLAMA_URL` and `OLLAMA_MODEL` still supported as fallbacks - Backwards compatible: legacy `OLLAMA_URL` / `OLLAMA_MODEL` env vars still map to the primary URL / model
**Model Discovery:** **Model Discovery:**
The `OllamaClient` provides methods to query available models: The `OllamaClient` provides methods to query available models:
@@ -850,25 +817,27 @@ The `OllamaClient` provides methods to query available models:
This allows runtime verification of model availability before generating insights. This allows runtime verification of model availability before generating insights.
**Local backend switch (`LLM_BACKEND`):** **Local backend switch (`backend` in llm.toml):**
One env var decides which "local" stack the server runs against — `ollama` One setting decides which "local" stack the server runs against — `ollama`
(default) or `llamacpp`. It's global on purpose: chat, vision, and (default) or `llamacpp`. It's global on purpose: chat, vision, and
embeddings all route through the same backend, so the embedding-vector embeddings all route through the same backend, so the embedding-vector
column in SQLite stays in one vector space. Don't flip mid-deploy without column in SQLite stays in one vector space. Don't flip mid-deploy without
re-embedding the affected rows — similarity search will collapse. re-embedding the affected rows — similarity search will collapse.
- `LLM_BACKEND=ollama`: chat, vision, and embeddings use Ollama. Vision - `backend = "ollama"`: chat, vision, and embeddings use Ollama. Vision
capability is probed per-model via `/api/show`. capability is probed per-model via `/api/show`.
- `LLM_BACKEND=llamacpp`: chat models receive images directly via OpenAI - `backend = "llamacpp"`: chat models with `vision: true` metadata receive
content-parts (all models assumed vision-capable). Embeddings hit the images directly via OpenAI content-parts; text-only ones get the local
`embed` slot. A dedicated `LLAMA_SWAP_VISION_MODEL` slot (defaults to vision model's description inlined (the hybrid path), so a `--no-mmproj`
the chat model) handles `describe_image` for the `describe_photo` tool. model never gets an image it can't read. If llama-swap can't be asked,
Requires `LLAMA_SWAP_URL`. images are sent. Embeddings hit `embedding.model`. `llamacpp.vision`
(defaults to the chat model) handles `describe_image` for the
`describe_photo` tool. Requires `llamacpp.url`.
The per-request `backend=hybrid` override is orthogonal: it always sends The per-request `backend=hybrid` override is orthogonal: it always sends
chat to OpenRouter (text-only, images are pre-described and inlined), but chat to OpenRouter (text-only, images are pre-described and inlined), but
the describe + embed passes still route through whichever `LLM_BACKEND` the describe + embed passes still route through whichever `backend`
is configured. is configured.
**Backend dispatch (`ResolvedBackend`):** **Backend dispatch (`ResolvedBackend`):**
@@ -884,18 +853,20 @@ min_p per-request overrides. All downstream code (`execute_tool`,
individual client references. individual client references.
`GET /insights/models` returns the local-backend models with capabilities `GET /insights/models` returns the local-backend models with capabilities
in the same envelope shape regardless of `LLM_BACKEND`: Ollama servers in the same envelope shape regardless of `backend`: Ollama servers
when `ollama`, llama-swap slots (from `LLAMA_SWAP_ALLOWED_MODELS`) when when `ollama`; when `llamacpp`, llama-swap's models discovered live from
`llamacpp`. No `/insights/llamacpp/models` — the picker reads a single `/v1/models` (`picker_models`: every `kind: chat` model, `:sub`/`:fast`
aliases hidden, filtered by `[llamacpp.picker]`), with vision / context
window from metadata. No `/insights/llamacpp/models` — the picker reads a single
endpoint. endpoint.
**Hybrid Backend (OpenRouter):** **Hybrid Backend (OpenRouter):**
- Per-request opt-in via `backend=hybrid` on `POST /insights/generate/agentic`. - Per-request opt-in via `backend=hybrid` on `POST /insights/generate/agentic`.
- Vision describe happens before the agentic loop; the description is inlined - Vision describe happens before the agentic loop; the description is inlined
into the chat prompt and the agentic loop runs on OpenRouter. Vision into the chat prompt and the agentic loop runs on OpenRouter. Vision
routes through whichever `LLM_BACKEND` is configured. routes through whichever `backend` is configured.
- `request.model` (if provided) overrides `OPENROUTER_DEFAULT_MODEL` for that - `request.model` (if provided) overrides `openrouter.default_model` for that
call. The mobile picker reads from `OPENROUTER_ALLOWED_MODELS`. call. The mobile picker reads from `openrouter.allowed`.
- No live capability precheck — the operator-curated allowlist is trusted. - No live capability precheck — the operator-curated allowlist is trusted.
A bad model id surfaces as a chat-call error. A bad model id surfaces as a chat-call error.
- `GET /insights/openrouter/models` returns `{ models, default_model, configured }` - `GET /insights/openrouter/models` returns `{ models, default_model, configured }`
@@ -970,7 +941,7 @@ heuristic). The served window comes from `LlmClient::context_window()`:
llama-swap models fix it at launch (`-c`) and ignore the request's llama-swap models fix it at launch (`-c`) and ignore the request's
`num_ctx`, so `LlamaCppClient` reads it from `/upstream/<model>/props` `num_ctx`, so `LlamaCppClient` reads it from `/upstream/<model>/props`
(cached 10 min). When neither is known the budget uses (cached 10 min). When neither is known the budget uses
`AGENTIC_CHAT_DEFAULT_NUM_CTX` (32768). Over budget, `apply_context_budget` `agentic.chat_default_num_ctx` (32768). Over budget, `apply_context_budget`
drops, oldest first: tool-call scaffolding in completed exchanges, then drops, oldest first: tool-call scaffolding in completed exchanges, then
whole earlier exchanges (keeps user/assistant alternation), then the whole earlier exchanges (keeps user/assistant alternation), then the
current turn's own tool results. The system prompt, the first user message current turn's own tool results. The system prompt, the first user message
@@ -981,12 +952,13 @@ server still rejects the prompt as too long, the budget is shrunk from its
reported token counts and the call retried once. reported token counts and the call retried once.
The `truncated` event / flag is surfaced to the client when a drop occurred. The `truncated` event / flag is surfaced to the client when a drop occurred.
Configurable env: Configurable in `llm.toml` `[agentic]`:
- `AGENTIC_CHAT_MAX_ITERATIONS` — cap on tool-calling iterations per turn - `chat_max_iterations` — cap on tool-calling iterations per turn
(default 6). Per-request `max_iterations` is clamped to this cap. (default 6). Per-request `max_iterations` is clamped to this cap.
- `AGENTIC_CHAT_DEFAULT_NUM_CTX` — assumed context window for the truncation - `chat_default_num_ctx` — assumed context window for the truncation
budget when the request omits `num_ctx` and the backend can't report its budget when the request omits `num_ctx` and the backend can't report its
served window (default 32768). served window (default 32768). Too small silently guts replayed history
every turn (and destroys llama.cpp KV-cache prefix reuse).
**Apollo Places integration (optional):** **Apollo Places integration (optional):**
+5 -4
View File
@@ -74,13 +74,14 @@ _Avoid_: user prefs, user settings, preferences (ambiguous with device-local cli
**Deployment config**: **Deployment config**:
Values the server needs before it can start serving — bind address, database path, media Values the server needs before it can start serving — bind address, database path, media
roots, the URLs used to build clients — plus every secret. Lives in `.env` only; changing roots, the URLs used to build clients — plus every secret. Lives in `.env`, except LLM
it is a redeploy, by design. settings, which live in `llm.toml` (ADR-0003); secrets stay in `.env` only. Changing it
is a restart, by design.
_Avoid_: environment config, system config _Avoid_: environment config, system config
**Hard default**: **Hard default**:
The env/code fallback a request lands on when neither an explicit param nor a stored The config/code fallback (`.env`, `llm.toml` or a code constant) a request lands on when neither an explicit param nor a stored
owner default supplies the field (e.g. backend `local`, persona `default`, reel span owner default supplies the field (e.g. backend `local`, persona `default`, reel span
day). It is the floor under the owner-default layer: env changes keep working whenever a day). It is the floor under the owner-default layer: config changes keep working whenever a
default column is NULL. default column is NULL.
_Avoid_: fallback value, env default _Avoid_: fallback value, env default
+43 -45
View File
@@ -83,44 +83,41 @@ They should be defined where the binary is located or above it in an `.env` file
### AI Insights Configuration (Optional) ### AI Insights Configuration (Optional)
The following environment variables configure AI-powered photo insights and daily conversation summaries: AI-powered photo insights, chat, read-aloud and daily conversation summaries are
configured in **`llm.toml`**, not `.env`.
#### Ollama Configuration #### LLM configuration (`llm.toml`)
- `OLLAMA_PRIMARY_URL` - Primary Ollama server URL [default: `http://localhost:11434`] Copy `llm.example.toml` to `llm.toml` next to `.env` (or point `LLM_CONFIG_PATH`
- Example: `http://desktop:11434` (your main/powerful server) at it). Every key is optional and the example documents each one:
- `OLLAMA_FALLBACK_URL` - Fallback Ollama server URL (optional)
- Example: `http://server:11434` (always-on backup server)
- `OLLAMA_PRIMARY_MODEL` - Model to use on primary server [default: `nemotron-3-nano:30b`]
- Example: `nemotron-3-nano:30b`, `llama3.2:3b`, etc.
- `OLLAMA_FALLBACK_MODEL` - Model to use on fallback server (optional)
- If not set, uses `OLLAMA_PRIMARY_MODEL` on fallback server
**Legacy Variables** (still supported): - `backend` — the local stack for chat, vision describe and embeddings:
- `OLLAMA_URL` - Used if `OLLAMA_PRIMARY_URL` not set `ollama` (default) or `llamacpp` (llama-swap). Global: don't flip it without
- `OLLAMA_MODEL` - Used if `OLLAMA_PRIMARY_MODEL` not set re-embedding.
- `[llamacpp]` — the llama-swap URL, the default `chat` model, the `vision` model
for `describe_photo`, and `[llamacpp.picker]` `exclude` / `include` filters.
- `[ollama]` — primary and fallback server URLs and models, request timeout.
- `[openrouter]` — hybrid backend: `default_model`, the curated `allowed` picker
list, `max_tokens`, attribution. The API key stays in env:
`OPENROUTER_API_KEY` enables the hybrid backend.
- `[embedding]` — embed model, `dim` and query/document prefixes. All of them
must match what the stored corpus was embedded with.
- `[tts]`, `[agentic]`, `unified_search_model` — see below and the example.
#### OpenRouter Configuration (Hybrid Backend) **The llama-swap model list is discovered, not configured.** `GET /insights/models`
The hybrid agentic backend keeps embeddings + vision local (Ollama) while routing lists every model llama-swap serves whose `metadata:` in llama-swap's
chat + tool-calling to OpenRouter. Enabled per-request when the client sends `config.yaml` says `kind: chat` (param aliases like `:sub` are hidden), with
`backend=hybrid`. `vision` and `ctx` from the same metadata. At startup the server logs a warning
for every role model, picker id or embed `dim` that doesn't match what
llama-swap serves.
- `OPENROUTER_API_KEY` - OpenRouter API key. Required to enable the hybrid backend. **Legacy env vars still work for now.** The old `OLLAMA_*`, `LLAMA_SWAP_*`,
- `OPENROUTER_DEFAULT_MODEL` - Model id used when the client doesn't specify one `OPENROUTER_*` (except the key), `LLM_BACKEND`, `EMBEDDING_DIM`, `EMBED_*_PREFIX`,
[default: `anthropic/claude-sonnet-4`] `AGENTIC_*`, `UNIFIED_SEARCH_MODEL` and `INSIGHT_CHAT_TURN_TIMEOUT_SECS` override
- Example: `openai/gpt-4o-mini`, `google/gemini-2.5-flash` `llm.toml`, with a startup deprecation warning listing the ones in use.
- `OPENROUTER_ALLOWED_MODELS` - Comma-separated curated allowlist exposed to `LLAMA_SWAP_ALLOWED_MODELS` maps to `picker.include`.
clients via `GET /insights/openrouter/models`. The mobile picker shows only
these. Empty/unset = no picker, server default is used.
- Example: `openai/gpt-4o-mini,anthropic/claude-haiku-4-5,google/gemini-2.5-flash`
- `OPENROUTER_BASE_URL` - Override base URL [default: `https://openrouter.ai/api/v1`]
- `OPENROUTER_EMBEDDING_MODEL` - Embedding model for OpenRouter
[default: `openai/text-embedding-3-small`]. Only used if/when embeddings are
routed through OpenRouter (currently embeddings stay local).
- `OPENROUTER_HTTP_REFERER` - Optional `HTTP-Referer` for OpenRouter attribution
- `OPENROUTER_APP_TITLE` - Optional `X-Title` for OpenRouter attribution
Capability checks are skipped for the curated allowlist — bad model ids surface Capability checks are skipped for the curated OpenRouter list — bad model ids
as a 4xx from the chat call. Pick tool-capable models. surface as a 4xx from the chat call. Pick tool-capable models.
#### SMS API Configuration #### SMS API Configuration
- `SMS_API_URL` - URL to SMS message API [default: `http://localhost:8000`] - `SMS_API_URL` - URL to SMS message API [default: `http://localhost:8000`]
@@ -128,7 +125,7 @@ as a 4xx from the chat call. Pick tool-capable models.
- `SMS_API_TOKEN` - Authentication token for SMS API (optional) - `SMS_API_TOKEN` - Authentication token for SMS API (optional)
#### Agentic Insight Generation #### Agentic Insight Generation
- `AGENTIC_MAX_ITERATIONS` - Maximum tool-call iterations per agentic insight request [default: `10`] - `agentic.max_iterations` - Maximum tool-call iterations per agentic insight request [default: `12`]
- Controls how many times the model can invoke tools before being forced to produce a final answer - Controls how many times the model can invoke tools before being forced to produce a final answer
- Increase for more thorough context gathering; decrease to limit response time - Increase for more thorough context gathering; decrease to limit response time
@@ -147,13 +144,13 @@ Amend mode (`amend: true` in the chat request body) regenerates the insight's
title and inserts a new row instead of appending to the existing transcript, title and inserts a new row instead of appending to the existing transcript,
so you can rewrite the saved summary from within chat. so you can rewrite the saved summary from within chat.
- `AGENTIC_CHAT_MAX_ITERATIONS` - Cap on tool-calling iterations per chat turn [default: `6`] - `agentic.chat_max_iterations` - Cap on tool-calling iterations per chat turn [default: `6`]
- Per-request `max_iterations` (when sent by the client) is clamped to this cap - Per-request `max_iterations` (when sent by the client) is clamped to this cap
#### Text-to-Speech (Optional) #### Text-to-Speech (Optional)
Reads insights aloud and manages cloned voices via a Chatterbox model served Reads insights aloud and manages cloned voices via a Chatterbox model served
behind the same llama-swap proxy. Only requires `LLAMA_SWAP_URL` (the TTS client behind the same llama-swap proxy. Only requires `llamacpp.url` (the TTS client
is built whenever that's set — independent of `LLM_BACKEND`). Endpoints: is built whenever that's set — independent of `backend`). Endpoints:
- `POST /tts/speech` — body `{ text, voice?, format?, exaggeration?, cfg_weight?, - `POST /tts/speech` — body `{ text, voice?, format?, exaggeration?, cfg_weight?,
temperature? }`; returns `{ audio_base64, format }`. Input is cleaned temperature? }`; returns `{ audio_base64, format }`. Input is cleaned
server-side (markdown + emoji stripped, then pronunciation overrides applied — server-side (markdown + emoji stripped, then pronunciation overrides applied —
@@ -189,21 +186,22 @@ matching rules are documented in `src/ai/pronunciation.rs`.
Env: Env:
- `TTS_PRONUNCIATIONS_PATH` - pronunciation-override JSON file - `TTS_PRONUNCIATIONS_PATH` - pronunciation-override JSON file
[default: `tts_pronunciations.json` in the working directory] [default: `tts_pronunciations.json` in the working directory]
- `LLAMA_SWAP_TTS_MODEL` - TTS model id in llama-swap's `config.yaml` [default: `chatterbox`]
- `LLAMA_SWAP_TTS_VOICE` - default voice used when a `/tts/speech` request omits `voice` (optional) `llm.toml` `[tts]`:
- `LLAMA_SWAP_TTS_REF_SECONDS` - max voice-clone reference clip length in seconds - `model` - TTS model id in llama-swap's `config.yaml` [default: `chatterbox`]
- `voice` - default voice used when a `/tts/speech` request omits `voice` (optional)
- `ref_seconds` - max voice-clone reference clip length in seconds
[default: `30`]. Reference audio is ffmpeg-normalized to mono 24 kHz WAV (so any [default: `30`]. Reference audio is ffmpeg-normalized to mono 24 kHz WAV (so any
source format works); Chatterbox is zero-shot, so a clean ~10–20s sample is the source format works); Chatterbox is zero-shot, so a clean ~10–20s sample is the
sweet spot — more rarely helps. sweet spot — more rarely helps.
- `LLAMA_SWAP_TTS_REQUEST_TIMEOUT_SECONDS` - per-request synthesis timeout in - `timeout_secs` - per-request synthesis timeout in seconds [default: `600`]. Long
seconds [default: `600`]. Long insights are chunked + synthesized server-side insights are chunked + synthesized server-side and can take minutes; this is
and can take minutes; this is separate from (and overrides, for `/tts/speech`) separate from (and overrides, for `/tts/speech`) `llamacpp.timeout_secs`.
the shared `LLAMA_SWAP_REQUEST_TIMEOUT_SECONDS`.
#### Fallback Behavior #### Fallback Behavior
- Primary server is tried first with 5-second connection timeout - Primary server is tried first with 5-second connection timeout
- On failure, automatically falls back to secondary server (if configured) - On failure, automatically falls back to secondary server (if configured)
- Total request timeout is 120 seconds to accommodate LLM inference - Total request timeout is `ollama.timeout_secs` (default 120) to accommodate LLM inference
- Logs indicate which server/model was used and any failover attempts - Logs indicate which server/model was used and any failover attempts
#### Daily Summary Generation #### Daily Summary Generation
+56
View File
@@ -0,0 +1,56 @@
# ADR-0003: LLM deployment config moves to llm.toml; llama-swap owns model capabilities
**Status**: accepted (2026-10-26). Amends ADR-0002's *deployment config* home for LLM settings.
## Context
ADR-0002 put LLM deployment config in `.env`: ~40 vars across `OLLAMA_*`, `LLAMA_SWAP_*`,
`OPENROUTER_*`, `LLM_BACKEND`, `EMBEDDING_DIM`, `EMBED_*_PREFIX`, `AGENTIC_*`. Flat env
vars can't express nested or list-shaped settings, and they duplicated facts that
llama-swap's `config.yaml` already owns. The deployed `LLAMA_SWAP_ALLOWED_MODELS` had
drifted from llama-swap: it listed three model ids that no longer existed (one a typo),
was missing three new ones, and every entry reported `has_vision: true`, including a
`--no-mmproj` text-only model.
## Decision
- **`llm.toml`** (TOML, `LLM_CONFIG_PATH`, default `./llm.toml`) holds every LLM
setting that ImageApi decides: the local `backend`, which model fills each role
(chat, vision, embedding, TTS, unified search), the embedding contract, the Ollama /
OpenRouter / TTS settings, agentic caps and the picker filters. Unknown keys fail startup.
- **Secrets stay in `.env`** (`OPENROUTER_API_KEY`), per ADR-0002.
- **llama-swap's `config.yaml` owns model capabilities.** Each model carries
`metadata: { kind, vision, ctx, dim }`, which `/v1/models` passes through. ImageApi
discovers the picker from it (every `kind: chat` model, param aliases hidden, minus
`picker.exclude`, optionally restricted to `picker.include`) instead of keeping a
hand-written allowlist. Text-only models get the vision model's description instead
of the image.
- **Startup cross-checks, warn-only.** Role models that aren't served or lack the needed
capability, and an embed `dim` that differs from `embedding.dim`, are logged. They
don't fail startup, because llama-swap may still be starting.
- **One release of env compatibility.** The legacy vars override the file, with one
deprecation warning listing the ones in use. Then they're removed.
The resolution chain from ADR-0002 is unchanged: request param → owner default
(`user_ai_prefs`) → hard default. `llm.toml` replaces `.env` as the hard-default layer
for LLM fields.
## Considered options
- **A standalone ImageApi model list** (roles plus per-model vision/ctx in `llm.toml`):
easier to read than env, but still a second copy of llama-swap's facts, so it drifts
the same way. Rejected.
- **Reading llama-swap's `config.yaml` directly**: couples ImageApi to a file on another
host and to llama-swap's schema. `/v1/models` is the published interface. Rejected.
- **Owner defaults in the DB** (`user_ai_prefs`): right for per-request tuning, which
stays there, but URLs and the embedding contract are needed before serving.
Rejected for these fields, as in ADR-0002.
## Non-obvious rules worth remembering
- The deployed llama-swap merges `metadata` into `meta.llamaswap`, while upstream
documents it at `meta`. `DiscoveredModel::from_json` reads both.
- A model with no metadata is treated as a text-only chat model, and the startup check
lists it.
- `embedding.query_prefix` / `document_prefix` default to empty because the existing
corpus was embedded without them. Set them only together with `reembed_embeddings`.
+1 -1
View File
@@ -1089,7 +1089,7 @@ pub struct OpenRouterModelsResponse {
/// GET /insights/openrouter/models - Curated OpenRouter model ids exposed /// GET /insights/openrouter/models - Curated OpenRouter model ids exposed
/// to clients for the hybrid backend. Returned verbatim from /// to clients for the hybrid backend. Returned verbatim from
/// `OPENROUTER_ALLOWED_MODELS`; no live call to OpenRouter. /// `openrouter.allowed`; no live call to OpenRouter.
#[get("/insights/openrouter/models")] #[get("/insights/openrouter/models")]
pub async fn get_openrouter_models_handler( pub async fn get_openrouter_models_handler(
_claims: Claims, _claims: Claims,
+2 -2
View File
@@ -2164,7 +2164,7 @@ fn resolve_date_taken_for_context(
/// symmetry and future rules. /// symmetry and future rules.
/// ///
/// Whether "local" routes through Ollama or llama-swap is decided at /// Whether "local" routes through Ollama or llama-swap is decided at
/// startup by `LLM_BACKEND`; both share the same transcript shape from /// startup by llm.toml's `backend`; both share the same transcript shape from
/// the chat-replay perspective. /// the chat-replay perspective.
fn validate_cross_replay(_stored: &str, effective: &str) -> Result<()> { fn validate_cross_replay(_stored: &str, effective: &str) -> Result<()> {
if !matches!(effective, "local" | "hybrid") { if !matches!(effective, "local" | "hybrid") {
@@ -3829,7 +3829,7 @@ mod tests {
#[test] #[test]
fn bootstrap_backend_rejects_unknown_label() { fn bootstrap_backend_rejects_unknown_label() {
// `llamacpp` is no longer a per-request backend value — it's chosen // `llamacpp` is no longer a per-request backend value — it's chosen
// at deploy time via `LLM_BACKEND`. // at deploy time via llm.toml's `backend`.
for label in &["openrouter", "llamacpp", "ollama"] { for label in &["openrouter", "llamacpp", "ollama"] {
let err = resolve_bootstrap_backend(Some(label)).unwrap_err(); let err = resolve_bootstrap_backend(Some(label)).unwrap_err();
let msg = format!("{}", err); let msg = format!("{}", err);
+3 -3
View File
@@ -234,7 +234,7 @@ pub struct InsightGenerator {
/// `None` when `OPENROUTER_API_KEY` is not configured. /// `None` when `OPENROUTER_API_KEY` is not configured.
openrouter: Option<Arc<OpenRouterClient>>, openrouter: Option<Arc<OpenRouterClient>>,
/// Optional llama-swap client, used when `backend=llamacpp` is requested. /// Optional llama-swap client, used when `backend=llamacpp` is requested.
/// `None` when `LLAMA_SWAP_URL` is not configured. /// `None` when `llamacpp.url` is not configured.
llamacpp: Option<Arc<LlamaCppClient>>, llamacpp: Option<Arc<LlamaCppClient>>,
sms_client: SmsApiClient, sms_client: SmsApiClient,
/// Optional integration with Apollo's user-defined Places. When the /// Optional integration with Apollo's user-defined Places. When the
@@ -735,7 +735,7 @@ impl InsightGenerator {
log::info!("========================================"); log::info!("========================================");
// Generate embedding for the query via the configured local backend // Generate embedding for the query via the configured local backend
// (`LLM_BACKEND` switch). Must match the backend that populated the // (llm.toml `backend` switch). Must match the backend that populated the
// daily-summary embeddings or similarity search will be garbage. // daily-summary embeddings or similarity search will be garbage.
let query_embedding = let query_embedding =
crate::ai::embed_query(&self.ollama, self.llamacpp.as_deref(), &query).await?; crate::ai::embed_query(&self.ollama, self.llamacpp.as_deref(), &query).await?;
@@ -3331,7 +3331,7 @@ Return ONLY the summary, nothing else."#,
} }
/// Tool: store_entity — upsert an entity into the knowledge memory. /// Tool: store_entity — upsert an entity into the knowledge memory.
/// Embeddings go through the configured local backend (`LLM_BACKEND`), /// Embeddings go through the configured local backend (llm.toml `backend`),
/// independent of the per-request chat backend in the caller. /// independent of the per-request chat backend in the caller.
async fn tool_store_entity(&self, args: &serde_json::Value) -> String { async fn tool_store_entity(&self, args: &serde_json::Value) -> String {
use crate::database::models::InsertEntity; use crate::database::models::InsertEntity;
+3 -3
View File
@@ -69,11 +69,11 @@ pub struct LlamaCppClient {
pub embedding_model: String, pub embedding_model: String,
/// Vision model slot id. Used for `describe_image` routing. Defaults /// Vision model slot id. Used for `describe_image` routing. Defaults
/// to `primary_model` so describe_image works out of the box; override /// to `primary_model` so describe_image works out of the box; override
/// via `LLAMA_SWAP_VISION_MODEL` for a dedicated vision slot. /// via `llamacpp.vision` for a dedicated vision slot.
pub vision_model: String, pub vision_model: String,
/// TTS model slot id (e.g. `"chatterbox"`). Routes `text_to_speech` and /// TTS model slot id (e.g. `"chatterbox"`). Routes `text_to_speech` and
/// is the `/upstream/<id>/voices` path segment for the voice library. /// is the `/upstream/<id>/voices` path segment for the voice library.
/// Override via `LLAMA_SWAP_TTS_MODEL`. /// Set via `tts.model`.
pub tts_model: String, pub tts_model: String,
num_ctx: Option<i32>, num_ctx: Option<i32>,
temperature: Option<f32>, temperature: Option<f32>,
@@ -1778,7 +1778,7 @@ mod tests {
// In hybrid mode, overrides.model is an OpenRouter model id // In hybrid mode, overrides.model is an OpenRouter model id
// (e.g. "google/gemini-3-flash-preview"). The local llamacpp // (e.g. "google/gemini-3-flash-preview"). The local llamacpp
// client must NOT adopt that as its vision_model — it should // client must NOT adopt that as its vision_model — it should
// keep the configured LLAMA_SWAP_VISION_MODEL so describe_image // keep the configured `llamacpp.vision` so describe_image
// hits the correct local slot instead of sending an unknown // hits the correct local slot instead of sending an unknown
// model name to llama-swap. // model name to llama-swap.
let mut base = LlamaCppClient::new(None, Some("chat".into())); let mut base = LlamaCppClient::new(None, Some("chat".into()));
+2 -2
View File
@@ -1,10 +1,10 @@
//! Bundle of the local LLM pair (Ollama + optional llama-swap) with the //! Bundle of the local LLM pair (Ollama + optional llama-swap) with the
//! `LLM_BACKEND` dispatch baked in. //! `backend` (llm.toml) dispatch baked in.
//! //!
//! Exists because passing the pair around as loose values invited the same //! Exists because passing the pair around as loose values invited the same
//! bug three times: import/backfill tooling embedded corpora via //! bug three times: import/backfill tooling embedded corpora via
//! `OllamaClient` directly while the query side dispatched through //! `OllamaClient` directly while the query side dispatched through
//! `embed_one`, so flipping `LLM_BACKEND=llamacpp` silently split queries //! `embed_one`, so flipping `backend = "llamacpp"` silently split queries
//! and corpus into different vector spaces. Anything that writes or reads //! and corpus into different vector spaces. Anything that writes or reads
//! embeddings should go through this type (or `embed_one`/`embed_many`), //! embeddings should go through this type (or `embed_one`/`embed_many`),
//! never a concrete client. //! never a concrete client.
+1 -1
View File
@@ -1163,7 +1163,7 @@ Analyze the image and use specific details from both the visual content and the
} }
}; };
// Validate embedding dimensions (EMBEDDING_DIM; 768 for nomic-embed-text:v1.5) // Validate embedding dimensions (`embedding.dim`; 768 for nomic-embed-text:v1.5)
for (i, embedding) in embeddings.iter().enumerate() { for (i, embedding) in embeddings.iter().enumerate() {
if embedding.len() != crate::ai::embedding_dim() { if embedding.len() != crate::ai::embedding_dim() {
log::warn!( log::warn!(
+1 -1
View File
@@ -26,7 +26,7 @@ const CACHE_DURATION_SECS: u64 = 15 * 60;
/// Default completion cap sent as `max_tokens`. OpenRouter pre-authorizes /// Default completion cap sent as `max_tokens`. OpenRouter pre-authorizes
/// credit against the requested max (defaulting to the model's full output /// credit against the requested max (defaulting to the model's full output
/// window when omitted), so expensive models fail with credit-limit errors /// window when omitted), so expensive models fail with credit-limit errors
/// even when the actual reply is short. Override via `OPENROUTER_MAX_TOKENS`. /// even when the actual reply is short. Override via `openrouter.max_tokens`.
pub const DEFAULT_MAX_TOKENS: u32 = 16384; pub const DEFAULT_MAX_TOKENS: u32 = 16384;
#[derive(Clone)] #[derive(Clone)]
+5 -5
View File
@@ -391,7 +391,7 @@ pub async fn tts_speech_handler(
let Some(client) = app_state.llamacpp.as_ref() else { let Some(client) = app_state.llamacpp.as_ref() else {
span.set_status(Status::error("tts backend not configured")); span.set_status(Status::error("tts backend not configured"));
return HttpResponse::ServiceUnavailable() return HttpResponse::ServiceUnavailable()
.json(json!({ "error": "TTS backend not configured (set LLAMA_SWAP_URL)" })); .json(json!({ "error": "TTS backend not configured (set llamacpp.url in llm.toml)" }));
}; };
let format = req let format = req
@@ -521,7 +521,7 @@ pub async fn create_speech_job_handler(
if app_state.llamacpp.is_none() { if app_state.llamacpp.is_none() {
span.set_status(Status::error("tts backend not configured")); span.set_status(Status::error("tts backend not configured"));
return HttpResponse::ServiceUnavailable() return HttpResponse::ServiceUnavailable()
.json(json!({ "error": "TTS backend not configured (set LLAMA_SWAP_URL)" })); .json(json!({ "error": "TTS backend not configured (set llamacpp.url in llm.toml)" }));
} }
let format = req let format = req
@@ -958,7 +958,7 @@ pub struct CreateVoiceFromLibraryRequest {
/// lets the client pick the clean-speech section of a long recording. /// lets the client pick the clean-speech section of a long recording.
#[serde(default)] #[serde(default)]
pub start_seconds: Option<f64>, pub start_seconds: Option<f64>,
/// Reference window length; clamped to LLAMA_SWAP_TTS_REF_SECONDS. /// Reference window length; clamped to `tts.ref_seconds`.
#[serde(default)] #[serde(default)]
pub duration_seconds: Option<f64>, pub duration_seconds: Option<f64>,
} }
@@ -966,7 +966,7 @@ pub struct CreateVoiceFromLibraryRequest {
/// POST /tts/voices/from-library — register a cloned voice from a file already /// POST /tts/voices/from-library — register a cloned voice from a file already
/// in a library. Audio and video alike are ffmpeg-normalized to a mono 24 kHz /// in a library. Audio and video alike are ffmpeg-normalized to a mono 24 kHz
/// WAV reference clip (window selected by start/duration_seconds, length /// WAV reference clip (window selected by start/duration_seconds, length
/// capped by LLAMA_SWAP_TTS_REF_SECONDS). /// capped by `tts.ref_seconds`).
#[post("/tts/voices/from-library")] #[post("/tts/voices/from-library")]
#[tracing::instrument(name = "http.tts.voices.from_library", skip_all)] #[tracing::instrument(name = "http.tts.voices.from_library", skip_all)]
pub async fn create_voice_from_library_handler( pub async fn create_voice_from_library_handler(
@@ -1153,7 +1153,7 @@ mod tests {
#[test] #[test]
fn resolve_ref_window_defaults_to_start_of_clip_at_cap_length() { fn resolve_ref_window_defaults_to_start_of_clip_at_cap_length() {
// Reads the live cap rather than mutating LLAMA_SWAP_TTS_REF_SECONDS: // Reads the live cap rather than mutating `tts.ref_seconds`:
// env mutation flakes under the parallel suite (see env_dispatch). // env mutation flakes under the parallel suite (see env_dispatch).
let cap = f64::from(tts_ref_seconds()); let cap = f64::from(tts_ref_seconds());
assert_eq!(resolve_ref_window(None, None), Ok((0.0, cap))); assert_eq!(resolve_ref_window(None, None), Ok((0.0, cap)));
+1 -1
View File
@@ -42,7 +42,7 @@ async fn main() -> Result<()> {
info!("Found {} calendar events", events.len()); info!("Found {} calendar events", events.len());
// LocalLlm dispatches per LLM_BACKEND, so embeddings written here land // LocalLlm dispatches per llm.toml's backend, so embeddings written here land
// in the same vector space the query side searches. // in the same vector space the query side searches.
let llm = if args.generate_embeddings { let llm = if args.generate_embeddings {
Some(LocalLlm::from_config()) Some(LocalLlm::from_config())
+1 -1
View File
@@ -38,7 +38,7 @@ async fn main() -> Result<()> {
info!("Found {} search records", searches.len()); info!("Found {} search records", searches.len());
// LocalLlm dispatches per LLM_BACKEND, so embeddings written here land // LocalLlm dispatches per llm.toml's backend, so embeddings written here land
// in the same vector space the query side searches. // in the same vector space the query side searches.
let llm = LocalLlm::from_config(); let llm = LocalLlm::from_config();
+1 -1
View File
@@ -35,7 +35,7 @@ struct Args {
#[arg(long)] #[arg(long)]
path: Option<String>, path: Option<String>,
/// Ollama model override. Defaults to OLLAMA_PRIMARY_MODEL from .env /// Ollama model override. Defaults to ollama.primary_model from llm.toml
#[arg(long)] #[arg(long)]
model: Option<String>, model: Option<String>,
+5 -5
View File
@@ -1,7 +1,7 @@
//! Re-embed stored corpora through `LocalLlm`, i.e. the same //! Re-embed stored corpora through `LocalLlm`, i.e. the same
//! `LLM_BACKEND` dispatch the query side uses. The original import / //! `backend` (llm.toml) dispatch the query side uses. The original import /
//! backfill tools always embedded via Ollama, so a deploy running //! backfill tools always embedded via Ollama, so a deploy running
//! `LLM_BACKEND=llamacpp` queries vector spaces the corpora may not live //! `backend = "llamacpp"` queries vector spaces the corpora may not live
//! in. Three tables share the problem and are all covered here: //! in. Three tables share the problem and are all covered here:
//! //!
//! - `daily_conversation_summaries` — re-embeds //! - `daily_conversation_summaries` — re-embeds
@@ -30,7 +30,7 @@ use image_api::bin_progress;
use std::env; use std::env;
#[derive(Parser, Debug)] #[derive(Parser, Debug)]
#[command(author, version, about = "Re-embed stored corpora via the configured LLM_BACKEND", long_about = None)] #[command(author, version, about = "Re-embed stored corpora via the configured local backend", long_about = None)]
struct Args { struct Args {
/// Comma-separated tables to process: summaries, calendar, search, entities /// Comma-separated tables to process: summaries, calendar, search, entities
#[arg(long, default_value = "summaries,calendar,search,entities")] #[arg(long, default_value = "summaries,calendar,search,entities")]
@@ -195,12 +195,12 @@ async fn reembed_table(
}; };
// The whole pipeline (DAO checks, stored corpora) assumes // The whole pipeline (DAO checks, stored corpora) assumes
// EMBEDDING_DIM dims. A mismatch means the active embed slot is not // embedding.dim dims. A mismatch means the active embed slot is not
// serving the configured model — stop rather than corrupt the table. // serving the configured model — stop rather than corrupt the table.
anyhow::ensure!( anyhow::ensure!(
new_emb.len() == image_api::ai::embedding_dim(), new_emb.len() == image_api::ai::embedding_dim(),
"backend returned {}-dim embedding (expected {}) — '{}' does not \ "backend returned {}-dim embedding (expected {}) — '{}' does not \
match the configured EMBEDDING_DIM", match the configured embedding.dim",
new_emb.len(), new_emb.len(),
image_api::ai::embedding_dim(), image_api::ai::embedding_dim(),
llm.embedding_model_version() llm.embedding_model_version()
+2 -2
View File
@@ -157,7 +157,7 @@ pub enum ScripterMode {
/// Fast path: single LLM call via the direct client. /// Fast path: single LLM call via the direct client.
Fast, Fast,
/// Agentic path: resolves the backend through the InsightGenerator /// Agentic path: resolves the backend through the InsightGenerator
/// (honouring LLM_BACKEND, model overrides, etc.). Falls back to /// (honouring llm.toml's backend, model overrides, etc.). Falls back to
/// Fast on error so a scripting failure never sinks a reel. /// Fast on error so a scripting failure never sinks a reel.
Agentic, Agentic,
} }
@@ -450,7 +450,7 @@ pub async fn create_reel_handler(
) -> impl Responder { ) -> impl Responder {
if app_state.llamacpp.is_none() { if app_state.llamacpp.is_none() {
return HttpResponse::ServiceUnavailable().json(json!({ return HttpResponse::ServiceUnavailable().json(json!({
"error": "Reel narration needs the LLM/TTS backend (set LLAMA_SWAP_URL)" "error": "Reel narration needs the LLM/TTS backend (set llamacpp.url in llm.toml)"
})); }));
} }
+1 -1
View File
@@ -289,7 +289,7 @@ pub async fn generate_script(
} }
/// Agentic version of script generation: resolves the backend via the /// Agentic version of script generation: resolves the backend via the
/// InsightGenerator (honouring LLM_BACKEND, model overrides, etc.), builds /// InsightGenerator (honouring llm.toml's backend, model overrides, etc.), builds
/// a read-only tool set, runs the tool loop, then parses the JSON response. /// a read-only tool set, runs the tool loop, then parses the JSON response.
/// Returns the same ReelScript shape. On failure the caller may fall back to /// Returns the same ReelScript shape. On failure the caller may fall back to
/// `generate_script`. /// `generate_script`.
+2 -2
View File
@@ -70,9 +70,9 @@ pub struct AppState {
#[allow(dead_code)] #[allow(dead_code)]
pub openrouter: Option<Arc<OpenRouterClient>>, pub openrouter: Option<Arc<OpenRouterClient>>,
/// Curated list of OpenRouter model ids exposed to clients. Sourced from /// Curated list of OpenRouter model ids exposed to clients. Sourced from
/// `OPENROUTER_ALLOWED_MODELS` (comma-separated). Empty when unset. /// `openrouter.allowed`. Empty when unset.
pub openrouter_allowed_models: Vec<String>, pub openrouter_allowed_models: Vec<String>,
/// `None` when `LLAMA_SWAP_URL` is not configured. Consulted only when a /// `None` when `llamacpp.url` is not configured. Consulted only when a
/// request explicitly opts into `backend=llamacpp`. Same shape as the /// request explicitly opts into `backend=llamacpp`. Same shape as the
/// `openrouter` slot — present here so handlers can route to it without /// `openrouter` slot — present here so handlers can route to it without
/// threading through the generator. /// threading through the generator.
+1 -1
View File
@@ -635,7 +635,7 @@ async fn translate<TagD: TagDao>(
} }
}; };
// Respect env/config for the LLM backend (LLM_BACKEND → ollama or // Respect env/config for the LLM backend (llm.toml `backend` → ollama or
// llama-swap); local only, no hybrid, per the feature's design. // llama-swap); local only, no hybrid, per the feature's design.
// //
// Translation-model precedence: // Translation-model precedence: