docs: document llm.toml and llama-swap metadata discovery

README, CLAUDE.md and .env.example now point at llm.toml for LLM settings
(OPENROUTER_API_KEY stays in env) and describe the legacy env override
window, the discovered picker and the startup cross-check. ADR-0003 records
the move and amends ADR-0002's deployment-config home; CONTEXT.md's
Deployment config / Hard default entries follow. Source comments and the
"set LLAMA_SWAP_URL" error strings name the llm.toml keys.

Also fixes README's AGENTIC_MAX_ITERATIONS default (10 -> 12, what the
code uses).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Cameron CordesandClaude Opus 5.5 committed 2026-10-26 17:40:42 -04:00
1 parent d68d9b09f3
commit 09b9e87396
21 files changed
+209 -244

No files matched your search

+9 -71
View File
@@ -40,78 +40,16 @@ PLAYLIST_CLEANUP_INTERVAL_SECONDS=86400
# RUST_LOG also gates export: keep it at info or debug.
# OTLP_OTLS_ENDPOINT=http://localhost:4317
# ── AI Insights — Ollama (local LLM) ────────────────────────────────────
OLLAMA_PRIMARY_URL=http://localhost:11434
OLLAMA_PRIMARY_MODEL=nemotron-3-nano:30b
# Optional fallback server tried on connection failure.
# OLLAMA_FALLBACK_URL=http://server:11434
# OLLAMA_FALLBACK_MODEL=llama3.2:3b
OLLAMA_REQUEST_TIMEOUT_SECONDS=120
# Cap on tool-calling iterations per chat turn / agentic insight.
AGENTIC_MAX_ITERATIONS=6
AGENTIC_CHAT_MAX_ITERATIONS=6
# ── AI Insights — OpenRouter (hybrid backend, optional) ─────────────────
# Set OPENROUTER_API_KEY to enable the hybrid backend (vision stays
# local on Ollama, chat routes to OpenRouter).
# ── AI Insights — LLMs ──────────────────────────────────────────────────
# Models, backends, the embedding contract, TTS and agentic caps live in
# llm.toml — copy llm.example.toml. Only the secret stays here.
# LLM_CONFIG_PATH=./llm.toml
# Enables the hybrid backend (chat via OpenRouter; photos stay local).
# OPENROUTER_API_KEY=sk-or-...
# OPENROUTER_DEFAULT_MODEL=anthropic/claude-sonnet-4
# OPENROUTER_ALLOWED_MODELS=openai/gpt-4o-mini,anthropic/claude-haiku-4-5,google/gemini-2.5-flash
# OPENROUTER_BASE_URL=https://openrouter.ai/api/v1
# OPENROUTER_EMBEDDING_MODEL=openai/text-embedding-3-small
# OPENROUTER_HTTP_REFERER=https://your-site.example
# OPENROUTER_APP_TITLE=ImageApi
# Cap on generated tokens per call (sent as max_tokens). OpenRouter reserves
# credit against the requested max, so leaving it uncapped makes pricey
# models fail with credit-limit errors. Default 16384; 0 = no cap.
# OPENROUTER_MAX_TOKENS=16384
# ── AI Insights — local backend switch ──────────────────────────────────
# Picks which local LLM stack the server uses for chat, vision describe,
# and embeddings. `ollama` (default) uses the OLLAMA_* settings above;
# `llamacpp` uses the LLAMA_SWAP_* settings below. The switch is global
# and applies to both `backend=local` and `backend=hybrid` (hybrid keeps
# chat on OpenRouter but still uses this stack for the describe pass).
# Don't flip mid-deploy without re-embedding existing index rows —
# mixed vector spaces break similarity search.
# LLM_BACKEND=ollama
# ── AI Insights — llama.cpp / llama-swap (optional) ─────────────────────
# Set LLAMA_SWAP_URL plus LLM_BACKEND=llamacpp to swap the local stack
# off Ollama. Talks OpenAI-compatible /v1 to a llama-swap proxy fronting
# per-slot llama-server instances. Chat models receive images directly
# via content-parts (vision-capable models assumed); a separate vision
# slot is used only by the describe_photo tool and describe-image utility.
# LLAMA_SWAP_URL=http://localhost:9292/v1
# LLAMA_SWAP_PRIMARY_MODEL=chat
# Optional dedicated vision slot for describe_image. Defaults to
# PRIMARY_MODEL so describe_photo works without extra config.
# LLAMA_SWAP_VISION_MODEL=vision
# LLAMA_SWAP_EMBEDDING_MODEL=embed
# Comma-separated allowlist surfaced by /insights/models when
# LLM_BACKEND=llamacpp. All report has_vision=true.
# LLAMA_SWAP_ALLOWED_MODELS=chat,vision,embed
# LLAMA_SWAP_REQUEST_TIMEOUT_SECONDS=180
# ── Unified search translation model (optional) ─────────────────────────
# /photos/search/unified runs one small LLM call to translate a natural-
# language query into structured filters + a semantic term, then CLIP-ranks.
# That step needs an LLM AND CLIP available at once. On a tight VRAM budget a
# large chat model can't co-reside with CLIP, so pin a small, fast model here
# (it can stay loaded alongside CLIP and the chat model). Precedence:
# UNIFIED_SEARCH_MODEL > the client's selected model > the configured default.
# Use the configured backend (LLM_BACKEND); local only — no hybrid.
# UNIFIED_SEARCH_MODEL=qwen3-0.6b
# ── Text-to-speech (optional, requires LLAMA_SWAP_URL) ───────────────────
# TTS routes through the same llama-swap proxy (a Chatterbox model id), so it
# only needs LLAMA_SWAP_URL — it does NOT require LLM_BACKEND=llamacpp.
# Powers POST /tts/speech and the /tts/voices* endpoints (read-aloud insights
# + voice cloning in the mobile app).
# LLAMA_SWAP_TTS_MODEL=chatterbox # TTS model id in config.yaml
# LLAMA_SWAP_TTS_VOICE=m # default voice when a request omits one
# LLAMA_SWAP_TTS_REF_SECONDS=30 # max voice-clone reference clip length (s)
# LLAMA_SWAP_TTS_REQUEST_TIMEOUT_SECONDS=600 # synth timeout (long chunked text)
#
# The old OLLAMA_* / LLAMA_SWAP_* / OPENROUTER_* / LLM_BACKEND / EMBED* /
# AGENTIC_* / UNIFIED_SEARCH_MODEL vars still override llm.toml for now, with a
# startup deprecation warning. Move them into llm.toml.
# ── AI Insights — sibling services (optional) ───────────────────────────
# Apollo (places, face inference, CLIP encoders). Single-Apollo deploys
+64 -92
View File
@@ -561,11 +561,11 @@ GET /memories?path=...&recursive=true
POST /insights/generate (non-agentic single-shot)
POST /insights/generate/agentic (tool-calling loop; body: { file_path, backend?, model?, ... })
GET /insights?path=...&library=...
GET /insights/models (local-backend models + capabilities; Ollama OR llama-swap based on LLM_BACKEND)
GET /insights/models (local-backend models + capabilities; Ollama OR llama-swap per llm.toml `backend`)
GET /insights/openrouter/models (curated OpenRouter allowlist)
POST /insights/rate (thumbs up/down for training data)
// Text-to-Speech (Chatterbox via llama-swap; needs LLAMA_SWAP_URL)
// Text-to-Speech (Chatterbox via llama-swap; needs llamacpp.url)
POST /tts/speech (read-aloud: { text, voice?, ... } -> { audio_base64, format })
GET /tts/voices (Chatterbox voice library)
POST /tts/voices/upload (clone a voice from an uploaded clip; multipart)
@@ -745,12 +745,9 @@ DATE_BACKFILL_MAX_PER_TICK=500 # Cap on canonical-date drain per watcher
HLS_COPY_MAX_BITRATE=20000000 # Max source bits/s to stream-copy into HLS; above it (or >1080p) transcodes
OTLP_OTLS_ENDPOINT=http://... # OTLP collector for span export (any build; unset = no export)
# AI Insights Configuration
OLLAMA_PRIMARY_URL=http://desktop:11434 # Primary Ollama server (e.g., desktop)
OLLAMA_FALLBACK_URL=http://server:11434 # Fallback Ollama server (optional, always-on)
OLLAMA_PRIMARY_MODEL=nemotron-3-nano:30b # Model for primary server (default: nemotron-3-nano:30b)
OLLAMA_FALLBACK_MODEL=llama3.2:3b # Model for fallback server (optional, uses primary if not set)
OLLAMA_REQUEST_TIMEOUT_SECONDS=120 # Per-request generation timeout (default 120). Increase for slow CPU-offloaded models.
# AI Insights: LLM settings live in llm.toml (see "LLM configuration" below)
LLM_CONFIG_PATH=./llm.toml # Default ./llm.toml; missing default file = env + defaults
OPENROUTER_API_KEY=sk-or-... # Secret, env-only. Enables the hybrid backend
SMS_API_URL=http://localhost:8000 # SMS message API endpoint (default: localhost:8000)
SMS_API_TOKEN=your-api-token # SMS API authentication token (optional)
@@ -769,79 +766,49 @@ FACE_AUTOBIND_MIN_COS=0.4 # Phase 3: cosine-sim floor for
FACE_DETECT_CONCURRENCY=8 # Phase 3: per-scan-tick parallel detect calls
FACE_DETECT_TIMEOUT_SEC=60 # reqwest client timeout (CPU inference can be slow)
# OpenRouter (Hybrid Backend) - keeps embeddings + vision local, routes chat to OpenRouter
OPENROUTER_API_KEY=sk-or-... # Required to enable hybrid backend
OPENROUTER_DEFAULT_MODEL=anthropic/claude-sonnet-4 # Used when client doesn't pick a model
OPENROUTER_ALLOWED_MODELS=openai/gpt-4o-mini,anthropic/claude-haiku-4-5,google/gemini-2.5-flash
# Curated allowlist exposed to clients via
# GET /insights/openrouter/models. Empty = no picker.
OPENROUTER_BASE_URL=https://openrouter.ai/api/v1 # Override base URL (optional)
OPENROUTER_EMBEDDING_MODEL=openai/text-embedding-3-small # Optional, embeddings stay local today
OPENROUTER_HTTP_REFERER=https://your-site.example # Optional attribution header
OPENROUTER_APP_TITLE=ImageApi # Optional attribution header
OPENROUTER_MAX_TOKENS=16384 # Per-call max_tokens cap (default 16384; 0 = uncapped).
# OpenRouter reserves credit against this, so an uncapped
# request to a pricey model can fail on credit limits.
# Local LLM backend switch. `ollama` (default) keeps the OLLAMA_* settings
# above; `llamacpp` swaps the entire local stack (chat + vision describe +
# embeddings) over to llama-swap. The switch is global and applies to
# `backend=local` requests and to `backend=hybrid`'s describe pass (hybrid
# chat still goes to OpenRouter). Don't flip mid-deploy without
# re-embedding — mixed vector spaces break similarity search.
LLM_BACKEND=ollama
# Embedding model contract. Corpus and queries must be embedded by the same
# model with matching prefixes — after changing the embed model or any of
# these, run `cargo run --bin reembed_embeddings` (all tables) or search is
# garbage. Prefix values may contain a literal \n (expanded to a newline).
EMBEDDING_DIM=768 # 768 = nomic-embed-text v1.5; 1024 = Qwen3-Embedding-0.6B
EMBED_QUERY_PREFIX= # nomic: "search_query: " | Qwen3: "Instruct: <task>\nQuery: "
EMBED_DOCUMENT_PREFIX= # nomic: "search_document: " | Qwen3: leave empty
# llama.cpp / llama-swap (used when LLM_BACKEND=llamacpp). OpenAI-compatible
# proxy hosting one or more llama-server processes. Chat models receive
# images directly via content-parts (all models assumed vision-capable).
LLAMA_SWAP_URL=http://localhost:9292/v1 # Required when LLM_BACKEND=llamacpp
LLAMA_SWAP_PRIMARY_MODEL=chat # Chat slot id (matches config.yaml)
LLAMA_SWAP_VISION_MODEL= # Dedicated vision slot for describe_image / describe_photo
# tool. Defaults to PRIMARY_MODEL when unset.
LLAMA_SWAP_EMBEDDING_MODEL=embed # Embedding slot id
LLAMA_SWAP_ALLOWED_MODELS=chat,coder # Curated allowlist surfaced by GET /insights/models
# when LLM_BACKEND=llamacpp. All report has_vision=true.
# Empty = picker shows only the configured primary model.
LLAMA_SWAP_REQUEST_TIMEOUT_SECONDS=180 # Per-request timeout; bump for slow CPU offload
# Text-to-speech (Chatterbox served behind llama-swap). Only needs
# LLAMA_SWAP_URL — independent of LLM_BACKEND. Powers /tts/speech (read-aloud)
# and /tts/voices* (voice cloning). Reference audio is ffmpeg-normalized to WAV
# server-side, so any source format works.
LLAMA_SWAP_TTS_MODEL=chatterbox # TTS model id in config.yaml (default: chatterbox)
LLAMA_SWAP_TTS_VOICE=m # Default voice when /tts/speech omits one (optional)
LLAMA_SWAP_TTS_REF_SECONDS=30 # Max voice-clone reference clip length, seconds
# (Chatterbox is zero-shot; ~10-20s clean ref is ideal)
LLAMA_SWAP_TTS_REQUEST_TIMEOUT_SECONDS=600 # Per-request synth timeout (long chunked insights take
# minutes); overrides the shared client timeout for /tts/speech
TTS_PRONUNCIATIONS_PATH=tts_pronunciations.json # JSON map of pronunciation overrides applied before synth
# (see tts_pronunciations.example.json); hot-reloaded on change
# Insight Chat Continuation
AGENTIC_CHAT_MAX_ITERATIONS=6 # Cap on tool-calling iterations per chat turn (default 6)
AGENTIC_CHAT_DEFAULT_NUM_CTX=32768 # Assumed context window for the history-truncation budget
# when a chat request omits num_ctx AND the backend can't
# report its served window (llama-swap reports it via /props,
# so this mostly matters for Ollama/OpenRouter). Too small
# silently guts replayed history every turn (and destroys
# llama.cpp KV-cache prefix reuse).
```
**LLM configuration (`llm.toml`, `src/ai/config.rs`):**
Every LLM setting except the OpenRouter key lives in `llm.toml`; `llm.example.toml`
is the annotated schema (a test parses it). `ai::config::llm_config()` is a
process-wide `OnceLock` resolved as **defaults → `llm.toml` → legacy env vars**.
The legacy vars (`OLLAMA_*`, `LLAMA_SWAP_*`, `OPENROUTER_*` except the key,
`LLM_BACKEND`, `EMBEDDING_DIM`, `EMBED_*_PREFIX`, `AGENTIC_*`,
`UNIFIED_SEARCH_MODEL`, `INSIGHT_CHAT_TURN_TIMEOUT_SECS`) override the file for one
release, with a single startup deprecation warning naming the ones in use;
`LLAMA_SWAP_ALLOWED_MODELS` maps to `llamacpp.picker.include`. `main` loads it right
after tracing init, so a bad file (unknown key, zero timeout, `backend =
"llamacpp"` without `llamacpp.url`) fails startup. Readers call `llm_config()`
instead of `std::env::var`; the merge itself is a pure `resolve(file, env_fn)`, so
tests never touch the process environment. ADR-0003 records the split.
Sections: `backend`, `unified_search_model`, `[llamacpp]` (`url`, `chat`, `vision`,
`timeout_secs`, `[llamacpp.picker]` `include`/`exclude`), `[ollama]`,
`[openrouter]`, `[embedding]` (`model`, `dim`, `query_prefix`, `document_prefix` —
the corpus contract: change any and run `reembed_embeddings`), `[tts]`
(`model`, `voice`, `ref_seconds`, `timeout_secs`), `[agentic]` (`max_iterations`,
`chat_max_iterations`, `chat_default_num_ctx`, `chat_turn_timeout_secs`).
**llama-swap capabilities come from its `config.yaml`, not from here.** Each model
there carries `metadata: { kind: chat|embed|tts, vision, ctx, dim }`, which
`/v1/models` passes through (the deployed build merges it into
`meta.llamaswap`; `DiscoveredModel::from_json` also reads top-level `meta`). A model
without metadata is a text-only chat model. At startup a background task logs
`catalog_warnings`: role models that aren't served or lack the needed capability,
an embed `dim` that differs from `embedding.dim`, picker ids that aren't served, and
models with no metadata.
**AI Insights Fallback Behavior:**
- Primary server is tried first with its configured model (5-second connection timeout)
- On connection failure, automatically falls back to secondary server with its model (if configured)
- If `OLLAMA_FALLBACK_MODEL` not set, uses same model as primary server on fallback
- Total request timeout is 120 seconds to accommodate slow LLM inference
- If `ollama.fallback_model` is unset, uses same model as primary server on fallback
- Total request timeout is `ollama.timeout_secs` (default 120) to accommodate slow LLM inference
- Logs indicate which server and model was used (info level) and failover attempts (warn level)
- Backwards compatible: `OLLAMA_URL` and `OLLAMA_MODEL` still supported as fallbacks
- Backwards compatible: legacy `OLLAMA_URL` / `OLLAMA_MODEL` env vars still map to the primary URL / model
**Model Discovery:**
The `OllamaClient` provides methods to query available models:
@@ -850,25 +817,27 @@ The `OllamaClient` provides methods to query available models:
This allows runtime verification of model availability before generating insights.
**Local backend switch (`LLM_BACKEND`):**
**Local backend switch (`backend` in llm.toml):**
One env var decides which "local" stack the server runs against — `ollama`
One setting decides which "local" stack the server runs against — `ollama`
(default) or `llamacpp`. It's global on purpose: chat, vision, and
embeddings all route through the same backend, so the embedding-vector
column in SQLite stays in one vector space. Don't flip mid-deploy without
re-embedding the affected rows — similarity search will collapse.
- `LLM_BACKEND=ollama`: chat, vision, and embeddings use Ollama. Vision
- `backend = "ollama"`: chat, vision, and embeddings use Ollama. Vision
capability is probed per-model via `/api/show`.
- `LLM_BACKEND=llamacpp`: chat models receive images directly via OpenAI
content-parts (all models assumed vision-capable). Embeddings hit the
`embed` slot. A dedicated `LLAMA_SWAP_VISION_MODEL` slot (defaults to
the chat model) handles `describe_image` for the `describe_photo` tool.
Requires `LLAMA_SWAP_URL`.
- `backend = "llamacpp"`: chat models with `vision: true` metadata receive
images directly via OpenAI content-parts; text-only ones get the local
vision model's description inlined (the hybrid path), so a `--no-mmproj`
model never gets an image it can't read. If llama-swap can't be asked,
images are sent. Embeddings hit `embedding.model`. `llamacpp.vision`
(defaults to the chat model) handles `describe_image` for the
`describe_photo` tool. Requires `llamacpp.url`.
The per-request `backend=hybrid` override is orthogonal: it always sends
chat to OpenRouter (text-only, images are pre-described and inlined), but
the describe + embed passes still route through whichever `LLM_BACKEND`
the describe + embed passes still route through whichever `backend`
is configured.
**Backend dispatch (`ResolvedBackend`):**
@@ -884,18 +853,20 @@ min_p per-request overrides. All downstream code (`execute_tool`,
individual client references.
`GET /insights/models` returns the local-backend models with capabilities
in the same envelope shape regardless of `LLM_BACKEND`: Ollama servers
when `ollama`, llama-swap slots (from `LLAMA_SWAP_ALLOWED_MODELS`) when
`llamacpp`. No `/insights/llamacpp/models` — the picker reads a single
in the same envelope shape regardless of `backend`: Ollama servers
when `ollama`; when `llamacpp`, llama-swap's models discovered live from
`/v1/models` (`picker_models`: every `kind: chat` model, `:sub`/`:fast`
aliases hidden, filtered by `[llamacpp.picker]`), with vision / context
window from metadata. No `/insights/llamacpp/models` — the picker reads a single
endpoint.
**Hybrid Backend (OpenRouter):**
- Per-request opt-in via `backend=hybrid` on `POST /insights/generate/agentic`.
- Vision describe happens before the agentic loop; the description is inlined
into the chat prompt and the agentic loop runs on OpenRouter. Vision
routes through whichever `LLM_BACKEND` is configured.
- `request.model` (if provided) overrides `OPENROUTER_DEFAULT_MODEL` for that
call. The mobile picker reads from `OPENROUTER_ALLOWED_MODELS`.
routes through whichever `backend` is configured.
- `request.model` (if provided) overrides `openrouter.default_model` for that
call. The mobile picker reads from `openrouter.allowed`.
- No live capability precheck — the operator-curated allowlist is trusted.
A bad model id surfaces as a chat-call error.
- `GET /insights/openrouter/models` returns `{ models, default_model, configured }`
@@ -970,7 +941,7 @@ heuristic). The served window comes from `LlmClient::context_window()`:
llama-swap models fix it at launch (`-c`) and ignore the request's
`num_ctx`, so `LlamaCppClient` reads it from `/upstream/<model>/props`
(cached 10 min). When neither is known the budget uses
`AGENTIC_CHAT_DEFAULT_NUM_CTX` (32768). Over budget, `apply_context_budget`
`agentic.chat_default_num_ctx` (32768). Over budget, `apply_context_budget`
drops, oldest first: tool-call scaffolding in completed exchanges, then
whole earlier exchanges (keeps user/assistant alternation), then the
current turn's own tool results. The system prompt, the first user message
@@ -981,12 +952,13 @@ server still rejects the prompt as too long, the budget is shrunk from its
reported token counts and the call retried once.
The `truncated` event / flag is surfaced to the client when a drop occurred.
Configurable env:
- `AGENTIC_CHAT_MAX_ITERATIONS` — cap on tool-calling iterations per turn
Configurable in `llm.toml` `[agentic]`:
- `chat_max_iterations` — cap on tool-calling iterations per turn
(default 6). Per-request `max_iterations` is clamped to this cap.
- `AGENTIC_CHAT_DEFAULT_NUM_CTX` — assumed context window for the truncation
- `chat_default_num_ctx` — assumed context window for the truncation
budget when the request omits `num_ctx` and the backend can't report its
served window (default 32768).
served window (default 32768). Too small silently guts replayed history
every turn (and destroys llama.cpp KV-cache prefix reuse).
**Apollo Places integration (optional):**
+5 -4
View File
@@ -74,13 +74,14 @@ _Avoid_: user prefs, user settings, preferences (ambiguous with device-local cli
**Deployment config**:
Values the server needs before it can start serving — bind address, database path, media
roots, the URLs used to build clients — plus every secret. Lives in `.env` only; changing
it is a redeploy, by design.
roots, the URLs used to build clients — plus every secret. Lives in `.env`, except LLM
settings, which live in `llm.toml` (ADR-0003); secrets stay in `.env` only. Changing it
is a restart, by design.
_Avoid_: environment config, system config
**Hard default**:
The env/code fallback a request lands on when neither an explicit param nor a stored
The config/code fallback (`.env`, `llm.toml` or a code constant) a request lands on when neither an explicit param nor a stored
owner default supplies the field (e.g. backend `local`, persona `default`, reel span
day). It is the floor under the owner-default layer: env changes keep working whenever a
day). It is the floor under the owner-default layer: config changes keep working whenever a
default column is NULL.
_Avoid_: fallback value, env default
+43 -45
View File
@@ -83,44 +83,41 @@ They should be defined where the binary is located or above it in an `.env` file
### AI Insights Configuration (Optional)
The following environment variables configure AI-powered photo insights and daily conversation summaries:
AI-powered photo insights, chat, read-aloud and daily conversation summaries are
configured in **`llm.toml`**, not `.env`.
#### Ollama Configuration
- `OLLAMA_PRIMARY_URL` - Primary Ollama server URL [default: `http://localhost:11434`]
- Example: `http://desktop:11434` (your main/powerful server)
- `OLLAMA_FALLBACK_URL` - Fallback Ollama server URL (optional)
- Example: `http://server:11434` (always-on backup server)
- `OLLAMA_PRIMARY_MODEL` - Model to use on primary server [default: `nemotron-3-nano:30b`]
- Example: `nemotron-3-nano:30b`, `llama3.2:3b`, etc.
- `OLLAMA_FALLBACK_MODEL` - Model to use on fallback server (optional)
- If not set, uses `OLLAMA_PRIMARY_MODEL` on fallback server
#### LLM configuration (`llm.toml`)
Copy `llm.example.toml` to `llm.toml` next to `.env` (or point `LLM_CONFIG_PATH`
at it). Every key is optional and the example documents each one:
**Legacy Variables** (still supported):
- `OLLAMA_URL` - Used if `OLLAMA_PRIMARY_URL` not set
- `OLLAMA_MODEL` - Used if `OLLAMA_PRIMARY_MODEL` not set
- `backend` — the local stack for chat, vision describe and embeddings:
`ollama` (default) or `llamacpp` (llama-swap). Global: don't flip it without
re-embedding.
- `[llamacpp]` — the llama-swap URL, the default `chat` model, the `vision` model
for `describe_photo`, and `[llamacpp.picker]` `exclude` / `include` filters.
- `[ollama]` — primary and fallback server URLs and models, request timeout.
- `[openrouter]` — hybrid backend: `default_model`, the curated `allowed` picker
list, `max_tokens`, attribution. The API key stays in env:
`OPENROUTER_API_KEY` enables the hybrid backend.
- `[embedding]` — embed model, `dim` and query/document prefixes. All of them
must match what the stored corpus was embedded with.
- `[tts]`, `[agentic]`, `unified_search_model` — see below and the example.
#### OpenRouter Configuration (Hybrid Backend)
The hybrid agentic backend keeps embeddings + vision local (Ollama) while routing
chat + tool-calling to OpenRouter. Enabled per-request when the client sends
`backend=hybrid`.
**The llama-swap model list is discovered, not configured.** `GET /insights/models`
lists every model llama-swap serves whose `metadata:` in llama-swap's
`config.yaml` says `kind: chat` (param aliases like `:sub` are hidden), with
`vision` and `ctx` from the same metadata. At startup the server logs a warning
for every role model, picker id or embed `dim` that doesn't match what
llama-swap serves.
- `OPENROUTER_API_KEY` - OpenRouter API key. Required to enable the hybrid backend.
- `OPENROUTER_DEFAULT_MODEL` - Model id used when the client doesn't specify one
[default: `anthropic/claude-sonnet-4`]
- Example: `openai/gpt-4o-mini`, `google/gemini-2.5-flash`
- `OPENROUTER_ALLOWED_MODELS` - Comma-separated curated allowlist exposed to
clients via `GET /insights/openrouter/models`. The mobile picker shows only
these. Empty/unset = no picker, server default is used.
- Example: `openai/gpt-4o-mini,anthropic/claude-haiku-4-5,google/gemini-2.5-flash`
- `OPENROUTER_BASE_URL` - Override base URL [default: `https://openrouter.ai/api/v1`]
- `OPENROUTER_EMBEDDING_MODEL` - Embedding model for OpenRouter
[default: `openai/text-embedding-3-small`]. Only used if/when embeddings are
routed through OpenRouter (currently embeddings stay local).
- `OPENROUTER_HTTP_REFERER` - Optional `HTTP-Referer` for OpenRouter attribution
- `OPENROUTER_APP_TITLE` - Optional `X-Title` for OpenRouter attribution
**Legacy env vars still work for now.** The old `OLLAMA_*`, `LLAMA_SWAP_*`,
`OPENROUTER_*` (except the key), `LLM_BACKEND`, `EMBEDDING_DIM`, `EMBED_*_PREFIX`,
`AGENTIC_*`, `UNIFIED_SEARCH_MODEL` and `INSIGHT_CHAT_TURN_TIMEOUT_SECS` override
`llm.toml`, with a startup deprecation warning listing the ones in use.
`LLAMA_SWAP_ALLOWED_MODELS` maps to `picker.include`.
Capability checks are skipped for the curated allowlist — bad model ids surface
as a 4xx from the chat call. Pick tool-capable models.
Capability checks are skipped for the curated OpenRouter list — bad model ids
surface as a 4xx from the chat call. Pick tool-capable models.
#### SMS API Configuration
- `SMS_API_URL` - URL to SMS message API [default: `http://localhost:8000`]
@@ -128,7 +125,7 @@ as a 4xx from the chat call. Pick tool-capable models.
- `SMS_API_TOKEN` - Authentication token for SMS API (optional)
#### Agentic Insight Generation
- `AGENTIC_MAX_ITERATIONS` - Maximum tool-call iterations per agentic insight request [default: `10`]
- `agentic.max_iterations` - Maximum tool-call iterations per agentic insight request [default: `12`]
- Controls how many times the model can invoke tools before being forced to produce a final answer
- Increase for more thorough context gathering; decrease to limit response time
@@ -147,13 +144,13 @@ Amend mode (`amend: true` in the chat request body) regenerates the insight's
title and inserts a new row instead of appending to the existing transcript,
so you can rewrite the saved summary from within chat.
- `AGENTIC_CHAT_MAX_ITERATIONS` - Cap on tool-calling iterations per chat turn [default: `6`]
- `agentic.chat_max_iterations` - Cap on tool-calling iterations per chat turn [default: `6`]
- Per-request `max_iterations` (when sent by the client) is clamped to this cap
#### Text-to-Speech (Optional)
Reads insights aloud and manages cloned voices via a Chatterbox model served
behind the same llama-swap proxy. Only requires `LLAMA_SWAP_URL` (the TTS client
is built whenever that's set — independent of `LLM_BACKEND`). Endpoints:
behind the same llama-swap proxy. Only requires `llamacpp.url` (the TTS client
is built whenever that's set — independent of `backend`). Endpoints:
- `POST /tts/speech` — body `{ text, voice?, format?, exaggeration?, cfg_weight?,
temperature? }`; returns `{ audio_base64, format }`. Input is cleaned
server-side (markdown + emoji stripped, then pronunciation overrides applied —
@@ -189,21 +186,22 @@ matching rules are documented in `src/ai/pronunciation.rs`.
Env:
- `TTS_PRONUNCIATIONS_PATH` - pronunciation-override JSON file
[default: `tts_pronunciations.json` in the working directory]
- `LLAMA_SWAP_TTS_MODEL` - TTS model id in llama-swap's `config.yaml` [default: `chatterbox`]
- `LLAMA_SWAP_TTS_VOICE` - default voice used when a `/tts/speech` request omits `voice` (optional)
- `LLAMA_SWAP_TTS_REF_SECONDS` - max voice-clone reference clip length in seconds
`llm.toml` `[tts]`:
- `model` - TTS model id in llama-swap's `config.yaml` [default: `chatterbox`]
- `voice` - default voice used when a `/tts/speech` request omits `voice` (optional)
- `ref_seconds` - max voice-clone reference clip length in seconds
[default: `30`]. Reference audio is ffmpeg-normalized to mono 24 kHz WAV (so any
source format works); Chatterbox is zero-shot, so a clean ~10–20s sample is the
sweet spot — more rarely helps.
- `LLAMA_SWAP_TTS_REQUEST_TIMEOUT_SECONDS` - per-request synthesis timeout in
seconds [default: `600`]. Long insights are chunked + synthesized server-side
and can take minutes; this is separate from (and overrides, for `/tts/speech`)
the shared `LLAMA_SWAP_REQUEST_TIMEOUT_SECONDS`.
- `timeout_secs` - per-request synthesis timeout in seconds [default: `600`]. Long
insights are chunked + synthesized server-side and can take minutes; this is
separate from (and overrides, for `/tts/speech`) `llamacpp.timeout_secs`.
#### Fallback Behavior
- Primary server is tried first with 5-second connection timeout
- On failure, automatically falls back to secondary server (if configured)
- Total request timeout is 120 seconds to accommodate LLM inference
- Total request timeout is `ollama.timeout_secs` (default 120) to accommodate LLM inference
- Logs indicate which server/model was used and any failover attempts
#### Daily Summary Generation
+56
View File
@@ -0,0 +1,56 @@
# ADR-0003: LLM deployment config moves to llm.toml; llama-swap owns model capabilities
**Status**: accepted (2026-10-26). Amends ADR-0002's *deployment config* home for LLM settings.
## Context
ADR-0002 put LLM deployment config in `.env`: ~40 vars across `OLLAMA_*`, `LLAMA_SWAP_*`,
`OPENROUTER_*`, `LLM_BACKEND`, `EMBEDDING_DIM`, `EMBED_*_PREFIX`, `AGENTIC_*`. Flat env
vars can't express nested or list-shaped settings, and they duplicated facts that
llama-swap's `config.yaml` already owns. The deployed `LLAMA_SWAP_ALLOWED_MODELS` had
drifted from llama-swap: it listed three model ids that no longer existed (one a typo),
was missing three new ones, and every entry reported `has_vision: true`, including a
`--no-mmproj` text-only model.
## Decision
- **`llm.toml`** (TOML, `LLM_CONFIG_PATH`, default `./llm.toml`) holds every LLM
setting that ImageApi decides: the local `backend`, which model fills each role
(chat, vision, embedding, TTS, unified search), the embedding contract, the Ollama /
OpenRouter / TTS settings, agentic caps and the picker filters. Unknown keys fail startup.
- **Secrets stay in `.env`** (`OPENROUTER_API_KEY`), per ADR-0002.
- **llama-swap's `config.yaml` owns model capabilities.** Each model carries
`metadata: { kind, vision, ctx, dim }`, which `/v1/models` passes through. ImageApi
discovers the picker from it (every `kind: chat` model, param aliases hidden, minus
`picker.exclude`, optionally restricted to `picker.include`) instead of keeping a
hand-written allowlist. Text-only models get the vision model's description instead
of the image.
- **Startup cross-checks, warn-only.** Role models that aren't served or lack the needed
capability, and an embed `dim` that differs from `embedding.dim`, are logged. They
don't fail startup, because llama-swap may still be starting.
- **One release of env compatibility.** The legacy vars override the file, with one
deprecation warning listing the ones in use. Then they're removed.
The resolution chain from ADR-0002 is unchanged: request param → owner default
(`user_ai_prefs`) → hard default. `llm.toml` replaces `.env` as the hard-default layer
for LLM fields.
## Considered options
- **A standalone ImageApi model list** (roles plus per-model vision/ctx in `llm.toml`):
easier to read than env, but still a second copy of llama-swap's facts, so it drifts
the same way. Rejected.
- **Reading llama-swap's `config.yaml` directly**: couples ImageApi to a file on another
host and to llama-swap's schema. `/v1/models` is the published interface. Rejected.
- **Owner defaults in the DB** (`user_ai_prefs`): right for per-request tuning, which
stays there, but URLs and the embedding contract are needed before serving.
Rejected for these fields, as in ADR-0002.
## Non-obvious rules worth remembering
- The deployed llama-swap merges `metadata` into `meta.llamaswap`, while upstream
documents it at `meta`. `DiscoveredModel::from_json` reads both.
- A model with no metadata is treated as a text-only chat model, and the startup check
lists it.
- `embedding.query_prefix` / `document_prefix` default to empty because the existing
corpus was embedded without them. Set them only together with `reembed_embeddings`.
+1 -1
View File
@@ -1089,7 +1089,7 @@ pub struct OpenRouterModelsResponse {
/// GET /insights/openrouter/models - Curated OpenRouter model ids exposed
/// to clients for the hybrid backend. Returned verbatim from
/// `OPENROUTER_ALLOWED_MODELS`; no live call to OpenRouter.
/// `openrouter.allowed`; no live call to OpenRouter.
#[get("/insights/openrouter/models")]
pub async fn get_openrouter_models_handler(
_claims: Claims,
+2 -2
View File
@@ -2164,7 +2164,7 @@ fn resolve_date_taken_for_context(
/// symmetry and future rules.
///
/// Whether "local" routes through Ollama or llama-swap is decided at
/// startup by `LLM_BACKEND`; both share the same transcript shape from
/// startup by llm.toml's `backend`; both share the same transcript shape from
/// the chat-replay perspective.
fn validate_cross_replay(_stored: &str, effective: &str) -> Result<()> {
if !matches!(effective, "local" | "hybrid") {
@@ -3829,7 +3829,7 @@ mod tests {
#[test]
fn bootstrap_backend_rejects_unknown_label() {
// `llamacpp` is no longer a per-request backend value — it's chosen
// at deploy time via `LLM_BACKEND`.
// at deploy time via llm.toml's `backend`.
for label in &["openrouter", "llamacpp", "ollama"] {
let err = resolve_bootstrap_backend(Some(label)).unwrap_err();
let msg = format!("{}", err);
+3 -3
View File
@@ -234,7 +234,7 @@ pub struct InsightGenerator {
/// `None` when `OPENROUTER_API_KEY` is not configured.
openrouter: Option<Arc<OpenRouterClient>>,
/// Optional llama-swap client, used when `backend=llamacpp` is requested.
/// `None` when `LLAMA_SWAP_URL` is not configured.
/// `None` when `llamacpp.url` is not configured.
llamacpp: Option<Arc<LlamaCppClient>>,
sms_client: SmsApiClient,
/// Optional integration with Apollo's user-defined Places. When the
@@ -735,7 +735,7 @@ impl InsightGenerator {
log::info!("========================================");
// Generate embedding for the query via the configured local backend
// (`LLM_BACKEND` switch). Must match the backend that populated the
// (llm.toml `backend` switch). Must match the backend that populated the
// daily-summary embeddings or similarity search will be garbage.
let query_embedding =
crate::ai::embed_query(&self.ollama, self.llamacpp.as_deref(), &query).await?;
@@ -3331,7 +3331,7 @@ Return ONLY the summary, nothing else."#,
}
/// Tool: store_entity — upsert an entity into the knowledge memory.
/// Embeddings go through the configured local backend (`LLM_BACKEND`),
/// Embeddings go through the configured local backend (llm.toml `backend`),
/// independent of the per-request chat backend in the caller.
async fn tool_store_entity(&self, args: &serde_json::Value) -> String {
use crate::database::models::InsertEntity;
+3 -3
View File
@@ -69,11 +69,11 @@ pub struct LlamaCppClient {
pub embedding_model: String,
/// Vision model slot id. Used for `describe_image` routing. Defaults
/// to `primary_model` so describe_image works out of the box; override
/// via `LLAMA_SWAP_VISION_MODEL` for a dedicated vision slot.
/// via `llamacpp.vision` for a dedicated vision slot.
pub vision_model: String,
/// TTS model slot id (e.g. `"chatterbox"`). Routes `text_to_speech` and
/// is the `/upstream/<id>/voices` path segment for the voice library.
/// Override via `LLAMA_SWAP_TTS_MODEL`.
/// Set via `tts.model`.
pub tts_model: String,
num_ctx: Option<i32>,
temperature: Option<f32>,
@@ -1778,7 +1778,7 @@ mod tests {
// In hybrid mode, overrides.model is an OpenRouter model id
// (e.g. "google/gemini-3-flash-preview"). The local llamacpp
// client must NOT adopt that as its vision_model — it should
// keep the configured LLAMA_SWAP_VISION_MODEL so describe_image
// keep the configured `llamacpp.vision` so describe_image
// hits the correct local slot instead of sending an unknown
// model name to llama-swap.
let mut base = LlamaCppClient::new(None, Some("chat".into()));
+2 -2
View File
@@ -1,10 +1,10 @@
//! Bundle of the local LLM pair (Ollama + optional llama-swap) with the
//! `LLM_BACKEND` dispatch baked in.
//! `backend` (llm.toml) dispatch baked in.
//!
//! Exists because passing the pair around as loose values invited the same
//! bug three times: import/backfill tooling embedded corpora via
//! `OllamaClient` directly while the query side dispatched through
//! `embed_one`, so flipping `LLM_BACKEND=llamacpp` silently split queries
//! `embed_one`, so flipping `backend = "llamacpp"` silently split queries
//! and corpus into different vector spaces. Anything that writes or reads
//! embeddings should go through this type (or `embed_one`/`embed_many`),
//! never a concrete client.
+1 -1
View File
@@ -1163,7 +1163,7 @@ Analyze the image and use specific details from both the visual content and the
}
};
// Validate embedding dimensions (EMBEDDING_DIM; 768 for nomic-embed-text:v1.5)
// Validate embedding dimensions (`embedding.dim`; 768 for nomic-embed-text:v1.5)
for (i, embedding) in embeddings.iter().enumerate() {
if embedding.len() != crate::ai::embedding_dim() {
log::warn!(
+1 -1
View File
@@ -26,7 +26,7 @@ const CACHE_DURATION_SECS: u64 = 15 * 60;
/// Default completion cap sent as `max_tokens`. OpenRouter pre-authorizes
/// credit against the requested max (defaulting to the model's full output
/// window when omitted), so expensive models fail with credit-limit errors
/// even when the actual reply is short. Override via `OPENROUTER_MAX_TOKENS`.
/// even when the actual reply is short. Override via `openrouter.max_tokens`.
pub const DEFAULT_MAX_TOKENS: u32 = 16384;
#[derive(Clone)]
+5 -5
View File
@@ -391,7 +391,7 @@ pub async fn tts_speech_handler(
let Some(client) = app_state.llamacpp.as_ref() else {
span.set_status(Status::error("tts backend not configured"));
return HttpResponse::ServiceUnavailable()
.json(json!({ "error": "TTS backend not configured (set LLAMA_SWAP_URL)" }));
.json(json!({ "error": "TTS backend not configured (set llamacpp.url in llm.toml)" }));
};
let format = req
@@ -521,7 +521,7 @@ pub async fn create_speech_job_handler(
if app_state.llamacpp.is_none() {
span.set_status(Status::error("tts backend not configured"));
return HttpResponse::ServiceUnavailable()
.json(json!({ "error": "TTS backend not configured (set LLAMA_SWAP_URL)" }));
.json(json!({ "error": "TTS backend not configured (set llamacpp.url in llm.toml)" }));
}
let format = req
@@ -958,7 +958,7 @@ pub struct CreateVoiceFromLibraryRequest {
/// lets the client pick the clean-speech section of a long recording.
#[serde(default)]
pub start_seconds: Option<f64>,
/// Reference window length; clamped to LLAMA_SWAP_TTS_REF_SECONDS.
/// Reference window length; clamped to `tts.ref_seconds`.
#[serde(default)]
pub duration_seconds: Option<f64>,
}
@@ -966,7 +966,7 @@ pub struct CreateVoiceFromLibraryRequest {
/// POST /tts/voices/from-library — register a cloned voice from a file already
/// in a library. Audio and video alike are ffmpeg-normalized to a mono 24 kHz
/// WAV reference clip (window selected by start/duration_seconds, length
/// capped by LLAMA_SWAP_TTS_REF_SECONDS).
/// capped by `tts.ref_seconds`).
#[post("/tts/voices/from-library")]
#[tracing::instrument(name = "http.tts.voices.from_library", skip_all)]
pub async fn create_voice_from_library_handler(
@@ -1153,7 +1153,7 @@ mod tests {
#[test]
fn resolve_ref_window_defaults_to_start_of_clip_at_cap_length() {
// Reads the live cap rather than mutating LLAMA_SWAP_TTS_REF_SECONDS:
// Reads the live cap rather than mutating `tts.ref_seconds`:
// env mutation flakes under the parallel suite (see env_dispatch).
let cap = f64::from(tts_ref_seconds());
assert_eq!(resolve_ref_window(None, None), Ok((0.0, cap)));
+1 -1
View File
@@ -42,7 +42,7 @@ async fn main() -> Result<()> {
info!("Found {} calendar events", events.len());
// LocalLlm dispatches per LLM_BACKEND, so embeddings written here land
// LocalLlm dispatches per llm.toml's backend, so embeddings written here land
// in the same vector space the query side searches.
let llm = if args.generate_embeddings {
Some(LocalLlm::from_config())
+1 -1
View File
@@ -38,7 +38,7 @@ async fn main() -> Result<()> {
info!("Found {} search records", searches.len());
// LocalLlm dispatches per LLM_BACKEND, so embeddings written here land
// LocalLlm dispatches per llm.toml's backend, so embeddings written here land
// in the same vector space the query side searches.
let llm = LocalLlm::from_config();
+1 -1
View File
@@ -35,7 +35,7 @@ struct Args {
#[arg(long)]
path: Option<String>,
/// Ollama model override. Defaults to OLLAMA_PRIMARY_MODEL from .env
/// Ollama model override. Defaults to ollama.primary_model from llm.toml
#[arg(long)]
model: Option<String>,
+5 -5
View File
@@ -1,7 +1,7 @@
//! Re-embed stored corpora through `LocalLlm`, i.e. the same
//! `LLM_BACKEND` dispatch the query side uses. The original import /
//! `backend` (llm.toml) dispatch the query side uses. The original import /
//! backfill tools always embedded via Ollama, so a deploy running
//! `LLM_BACKEND=llamacpp` queries vector spaces the corpora may not live
//! `backend = "llamacpp"` queries vector spaces the corpora may not live
//! in. Three tables share the problem and are all covered here:
//!
//! - `daily_conversation_summaries` — re-embeds
@@ -30,7 +30,7 @@ use image_api::bin_progress;
use std::env;
#[derive(Parser, Debug)]
#[command(author, version, about = "Re-embed stored corpora via the configured LLM_BACKEND", long_about = None)]
#[command(author, version, about = "Re-embed stored corpora via the configured local backend", long_about = None)]
struct Args {
/// Comma-separated tables to process: summaries, calendar, search, entities
#[arg(long, default_value = "summaries,calendar,search,entities")]
@@ -195,12 +195,12 @@ async fn reembed_table(
};
// The whole pipeline (DAO checks, stored corpora) assumes
// EMBEDDING_DIM dims. A mismatch means the active embed slot is not
// embedding.dim dims. A mismatch means the active embed slot is not
// serving the configured model — stop rather than corrupt the table.
anyhow::ensure!(
new_emb.len() == image_api::ai::embedding_dim(),
"backend returned {}-dim embedding (expected {}) — '{}' does not \
match the configured EMBEDDING_DIM",
match the configured embedding.dim",
new_emb.len(),
image_api::ai::embedding_dim(),
llm.embedding_model_version()
+2 -2
View File
@@ -157,7 +157,7 @@ pub enum ScripterMode {
/// Fast path: single LLM call via the direct client.
Fast,
/// Agentic path: resolves the backend through the InsightGenerator
/// (honouring LLM_BACKEND, model overrides, etc.). Falls back to
/// (honouring llm.toml's backend, model overrides, etc.). Falls back to
/// Fast on error so a scripting failure never sinks a reel.
Agentic,
}
@@ -450,7 +450,7 @@ pub async fn create_reel_handler(
) -> impl Responder {
if app_state.llamacpp.is_none() {
return HttpResponse::ServiceUnavailable().json(json!({
"error": "Reel narration needs the LLM/TTS backend (set LLAMA_SWAP_URL)"
"error": "Reel narration needs the LLM/TTS backend (set llamacpp.url in llm.toml)"
}));
}
+1 -1
View File
@@ -289,7 +289,7 @@ pub async fn generate_script(
}
/// Agentic version of script generation: resolves the backend via the
/// InsightGenerator (honouring LLM_BACKEND, model overrides, etc.), builds
/// InsightGenerator (honouring llm.toml's backend, model overrides, etc.), builds
/// a read-only tool set, runs the tool loop, then parses the JSON response.
/// Returns the same ReelScript shape. On failure the caller may fall back to
/// `generate_script`.
+2 -2
View File
@@ -70,9 +70,9 @@ pub struct AppState {
#[allow(dead_code)]
pub openrouter: Option<Arc<OpenRouterClient>>,
/// Curated list of OpenRouter model ids exposed to clients. Sourced from
/// `OPENROUTER_ALLOWED_MODELS` (comma-separated). Empty when unset.
/// `openrouter.allowed`. Empty when unset.
pub openrouter_allowed_models: Vec<String>,
/// `None` when `LLAMA_SWAP_URL` is not configured. Consulted only when a
/// `None` when `llamacpp.url` is not configured. Consulted only when a
/// request explicitly opts into `backend=llamacpp`. Same shape as the
/// `openrouter` slot — present here so handlers can route to it without
/// threading through the generator.
+1 -1
View File
@@ -635,7 +635,7 @@ async fn translate<TagD: TagDao>(
}
};
// Respect env/config for the LLM backend (LLM_BACKEND → ollama or
// Respect env/config for the LLM backend (llm.toml `backend` → ollama or
// llama-swap); local only, no hybrid, per the feature's design.
//
// Translation-model precedence: