Cameron CordesandClaude Opus 5.5
5b8a99a2b2
Adds ai::config: one serde struct for every LLM setting (backend, llama.cpp, Ollama, OpenRouter, embedding, TTS, agentic caps), resolved as defaults -> llm.toml (LLM_CONFIG_PATH, default ./llm.toml) -> the old env vars. Env overrides are kept for one release and listed in a single startup deprecation warning; LLAMA_SWAP_ALLOWED_MODELS maps to picker.include so current deploys keep their picker. Unknown keys, zero timeouts and backend=llamacpp without a URL fail startup. OPENROUTER_API_KEY stays env-only. llm.example.toml documents the schema and is parsed by a test. main loads the config right after tracing init; readers switch over next. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
74 lines
3.3 KiB
TOML
74 lines
3.3 KiB
TOML
# ImageApi LLM configuration. Copy to `llm.toml` next to `.env` (or point
|
|
# LLM_CONFIG_PATH at it). Every key is optional; omitted keys use the
|
|
# defaults shown in the comments.
|
|
#
|
|
# Secrets stay in env: OPENROUTER_API_KEY enables the hybrid backend.
|
|
#
|
|
# The local model catalog (which models exist, vision, context size) is NOT
|
|
# listed here. It is discovered from llama-swap's /v1/models, using each
|
|
# model's `metadata:` block in llama-swap/config.yaml (kind/vision/ctx/dim).
|
|
#
|
|
# The old env vars (LLAMA_SWAP_*, OLLAMA_*, OPENROUTER_* except the key,
|
|
# EMBED_*, EMBEDDING_DIM, AGENTIC_*, LLM_BACKEND, UNIFIED_SEARCH_MODEL) still
|
|
# work for now and OVERRIDE this file, with a deprecation warning at startup.
|
|
|
|
# Local stack for chat, vision describe and embeddings: "ollama" | "llamacpp".
|
|
# Global on purpose: don't flip it without re-embedding (mixed vector spaces
|
|
# break similarity search).
|
|
backend = "llamacpp"
|
|
|
|
# Small model pinned for /photos/search/unified query translation, so it can
|
|
# stay loaded alongside CLIP. Unset = the client's selected model.
|
|
unified_search_model = "gemma4-e4b"
|
|
|
|
[llamacpp]
|
|
url = "http://localhost:9292/v1" # unset = llama.cpp client + TTS disabled
|
|
chat = "qwen3.8-27b-abliterated" # default chat model (default "chat")
|
|
vision = "qwen3.8-27b-code-vision" # describe_photo model (default: chat)
|
|
timeout_secs = 180
|
|
|
|
# The picker shows every llama-swap model with `kind: chat` metadata (param
|
|
# aliases like `:sub` / `:fast` are always hidden), minus `exclude`.
|
|
# `include`, if set, is a strict allowlist instead.
|
|
[llamacpp.picker]
|
|
exclude = ["nemotron3.5-lightning-30b-A3B"]
|
|
# include = ["qwen3.8-27b-code", "gemma4-31"]
|
|
|
|
[embedding]
|
|
# Must all agree with what the stored corpus was embedded with. Changing any
|
|
# of them means `cargo run --bin reembed_embeddings`.
|
|
model = "embed" # llama-swap id (Ollama always uses nomic-embed-text:v1.5)
|
|
dim = 768 # 768 nomic v1.5 | 1024 Qwen3-Embedding-0.6B
|
|
query_prefix = "search_query: " # Qwen3: "Instruct: <task>\nQuery: "
|
|
document_prefix = "search_document: " # Qwen3: ""
|
|
|
|
[tts]
|
|
# Chatterbox behind llama-swap; needs llamacpp.url but not backend = "llamacpp".
|
|
model = "chatterbox"
|
|
voice = "m" # default voice when a request omits one (unset = server default)
|
|
ref_seconds = 30 # max voice-clone reference clip length
|
|
timeout_secs = 600 # long chunked text takes minutes
|
|
|
|
[ollama]
|
|
primary_url = "http://localhost:11434"
|
|
primary_model = "nemotron-3-nano:30b"
|
|
# Tried on connection failure. fallback_model unset = reuse primary_model.
|
|
# fallback_url = "http://server:11434"
|
|
# fallback_model = "llama3.2:3b"
|
|
timeout_secs = 120
|
|
|
|
[openrouter]
|
|
default_model = "anthropic/claude-sonnet-4"
|
|
allowed = ["openai/gpt-4o-mini", "anthropic/claude-haiku-4-5", "google/gemini-2.5-flash"]
|
|
max_tokens = 16384 # 0 = uncapped (OpenRouter reserves credit against it)
|
|
# base_url = "https://openrouter.ai/api/v1"
|
|
# embedding_model = "openai/text-embedding-3-small"
|
|
# http_referer = "https://your-site.example"
|
|
# app_title = "ImageApi"
|
|
|
|
[agentic]
|
|
max_iterations = 12 # tool-loop cap for agentic insight generation
|
|
chat_max_iterations = 6 # per chat turn; per-request values are clamped to it
|
|
chat_default_num_ctx = 32768 # assumed window when neither request nor backend reports one
|
|
chat_turn_timeout_secs = 300 # reconnectable chat turns are dropped after this
|