From aa3536afeebb7b1e73a8106862692dd018950c25 Mon Sep 17 00:00:00 2001 From: Cameron Cordes Date: Mon, 26 Oct 2026 17:30:24 -0400 Subject: [PATCH] refactor(ai): read LLM settings from LlmConfig instead of env Every LLM env reader now goes through ai::config::llm_config(): build_ollama / build_llamacpp / build_openrouter take config sections (OPENROUTER_API_KEY is still read from env), and the backend switch, embedding dim + prefixes, Ollama / llama.cpp / TTS timeouts, TTS voice and ref length, agentic caps, chat turn timeout and the unified-search model read their sections. LocalLlm::from_env -> from_config; the test_daily_summary and populate_knowledge bins take Ollama defaults from the config. env_max_iterations / env_default_num_ctx are renamed to max_chat_iterations / default_num_ctx. The picker still uses picker.include (= LLAMA_SWAP_ALLOWED_MODELS) until discovery lands. Legacy env vars keep working through the config's override layer; checked against the deployed .env, which resolves to the same values. The LLM_BACKEND parsing tests move to ai::config. llm.example.toml now leaves the embed prefixes empty: copying the nomic prefixes onto a corpus embedded without them breaks search. Co-Authored-By: Claude Opus 5.5 --- llm.example.toml | 6 +- src/ai/config.rs | 20 ++++- src/ai/handlers.rs | 5 +- src/ai/insight_chat.rs | 44 +++++------ src/ai/llamacpp.rs | 12 +-- src/ai/local_llm.rs | 19 ++--- src/ai/mod.rs | 112 +++++---------------------- src/ai/ollama.rs | 14 +--- src/ai/persona_chat.rs | 4 +- src/ai/tts.rs | 15 +--- src/bin/import_calendar.rs | 2 +- src/bin/import_search_history.rs | 2 +- src/bin/populate_knowledge.rs | 17 ++--- src/bin/reembed_embeddings.rs | 2 +- src/bin/test_daily_summary.rs | 29 +++---- src/state.rs | 125 ++++++++++--------------------- src/unified_search.rs | 8 +- 17 files changed, 144 insertions(+), 292 deletions(-) diff --git a/llm.example.toml b/llm.example.toml index f71df01..5a5682c 100644 --- a/llm.example.toml +++ b/llm.example.toml @@ -39,8 +39,10 @@ exclude = ["nemotron3.5-lightning-30b-A3B"] # of them means `cargo run --bin reembed_embeddings`. model = "embed" # llama-swap id (Ollama always uses nomic-embed-text:v1.5) dim = 768 # 768 nomic v1.5 | 1024 Qwen3-Embedding-0.6B -query_prefix = "search_query: " # Qwen3: "Instruct: \nQuery: " -document_prefix = "search_document: " # Qwen3: "" +# Prefixes default to empty. Set them only together with a re-embed; a +# corpus embedded without them won't match queries embedded with them. +query_prefix = "" # nomic v1.5: "search_query: " | Qwen3: "Instruct: \nQuery: " +document_prefix = "" # nomic v1.5: "search_document: " | Qwen3: "" [tts] # Chatterbox behind llama-swap; needs llamacpp.url but not backend = "llamacpp". diff --git a/src/ai/config.rs b/src/ai/config.rs index baecbc2..c441f2f 100644 --- a/src/ai/config.rs +++ b/src/ai/config.rs @@ -221,7 +221,7 @@ impl Default for AgenticConfig { Self { max_iterations: 12, chat_max_iterations: crate::ai::insight_chat::DEFAULT_MAX_ITERATIONS, - chat_default_num_ctx: 32768, + chat_default_num_ctx: crate::ai::insight_chat::DEFAULT_NUM_CTX, chat_turn_timeout_secs: 300, } } @@ -500,7 +500,7 @@ mod tests { let c = r.config; assert_eq!(c.backend, LocalBackend::Llamacpp); assert!(c.llamacpp.url.is_some()); - assert!(c.embedding.query_prefix.ends_with(' ')); + assert_eq!(c.embedding.dim, 768); } #[test] @@ -539,6 +539,22 @@ mod tests { ); } + #[test] + fn llm_backend_env_is_case_insensitive_and_unknown_means_ollama() { + let backend = |v: &str| { + resolve( + None, + env_of(&[("LLM_BACKEND", v), ("LLAMA_SWAP_URL", "http://x/v1")]), + ) + .unwrap() + .config + .backend + }; + assert_eq!(backend("LlamaCpp"), LocalBackend::Llamacpp); + assert_eq!(backend(" llamacpp "), LocalBackend::Llamacpp); + assert_eq!(backend("vllm"), LocalBackend::Ollama); + } + #[test] fn blank_env_values_count_as_unset() { let r = resolve( diff --git a/src/ai/handlers.rs b/src/ai/handlers.rs index b53d05e..b3a5b53 100644 --- a/src/ai/handlers.rs +++ b/src/ai/handlers.rs @@ -867,10 +867,7 @@ pub async fn generate_agentic_insight_handler( .persona_system_prompt(user_id, &persona_id) }); - let max_iterations: usize = std::env::var("AGENTIC_MAX_ITERATIONS") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(12); + let max_iterations = crate::ai::config::llm_config().agentic.max_iterations; // Spawn background task with timeout let generator = app_state.insight_generator.clone(); diff --git a/src/ai/insight_chat.rs b/src/ai/insight_chat.rs index 3d384dd..7e30f76 100644 --- a/src/ai/insight_chat.rs +++ b/src/ai/insight_chat.rs @@ -26,8 +26,8 @@ pub const DEFAULT_MAX_ITERATIONS: usize = 6; /// `effective_num_ctx`). Real conversations rarely pass ~16k tokens, so 32k /// keeps the truncation pass from gutting history the server could hold /// (which also destroys the server's KV-cache prefix reuse). Override -/// per-deploy with AGENTIC_CHAT_DEFAULT_NUM_CTX. -const DEFAULT_NUM_CTX: i32 = 32768; +/// per-deploy with `agentic.chat_default_num_ctx` in llm.toml. +pub(crate) const DEFAULT_NUM_CTX: i32 = 32768; /// Headroom reserved for the model's response, deducted from the context /// budget when deciding whether to truncate the replayed history. pub(crate) const RESPONSE_HEADROOM_TOKENS: usize = 2048; @@ -341,7 +341,7 @@ impl InsightChatService { let max_iterations = req .max_iterations .unwrap_or(DEFAULT_MAX_ITERATIONS) - .clamp(1, env_max_iterations()); + .clamp(1, max_chat_iterations()); span.set_attribute("max_iterations", max_iterations as i64); let stored_model = insight.model_version.clone(); @@ -1043,7 +1043,7 @@ impl InsightChatService { let max_iterations = req .max_iterations .unwrap_or(DEFAULT_MAX_ITERATIONS) - .clamp(1, env_max_iterations()); + .clamp(1, max_chat_iterations()); let stored_model = insight.model_version.clone(); let overrides = SamplingOverrides { @@ -1246,7 +1246,7 @@ impl InsightChatService { let max_iterations = req .max_iterations .unwrap_or(DEFAULT_MAX_ITERATIONS) - .clamp(1, env_max_iterations()); + .clamp(1, max_chat_iterations()); let overrides = SamplingOverrides { model: req.model.clone().filter(|m| !m.is_empty()), @@ -1526,7 +1526,7 @@ impl InsightChatService { let max_iterations = req .max_iterations .unwrap_or(DEFAULT_MAX_ITERATIONS) - .clamp(1, env_max_iterations()); + .clamp(1, max_chat_iterations()); let stored_model = insight.model_version.clone(); let overrides = SamplingOverrides { @@ -1721,7 +1721,7 @@ impl InsightChatService { let max_iterations = req .max_iterations .unwrap_or(DEFAULT_MAX_ITERATIONS) - .clamp(1, env_max_iterations()); + .clamp(1, max_chat_iterations()); let overrides = SamplingOverrides { model: req.model.clone().filter(|m| !m.is_empty()), @@ -2415,25 +2415,19 @@ pub(crate) fn find_raw_cut( None } -/// Read AGENTIC_CHAT_MAX_ITERATIONS once per call. Cheap; keeps the code -/// free of static globals and lets the operator change the cap by env without -/// a restart in test harnesses (the running server still caches via Default). -pub fn env_max_iterations() -> usize { - std::env::var("AGENTIC_CHAT_MAX_ITERATIONS") - .ok() - .and_then(|s| s.parse::().ok()) - .unwrap_or(DEFAULT_MAX_ITERATIONS) - .max(1) +/// Cap on tool-calling iterations per chat turn (`agentic.chat_max_iterations`, +/// validated >= 1). Per-request values are clamped to it. +pub fn max_chat_iterations() -> usize { + crate::ai::config::llm_config().agentic.chat_max_iterations } -/// Read AGENTIC_CHAT_DEFAULT_NUM_CTX once per call — the assumed context -/// window for the truncation budget when the request omits `num_ctx`. Same -/// no-static-global rationale as `env_max_iterations` above. -pub(crate) fn env_default_num_ctx() -> i32 { - std::env::var("AGENTIC_CHAT_DEFAULT_NUM_CTX") - .ok() - .and_then(|s| s.parse::().ok()) - .unwrap_or(DEFAULT_NUM_CTX) +/// Assumed context window for the truncation budget when the request omits +/// `num_ctx` and the backend can't report one (`agentic.chat_default_num_ctx`). +/// Floored so the budget always leaves room for the response headroom. +pub(crate) fn default_num_ctx() -> i32 { + crate::ai::config::llm_config() + .agentic + .chat_default_num_ctx .max(RESPONSE_HEADROOM_TOKENS as i32 + 1024) } @@ -2451,7 +2445,7 @@ pub(crate) async fn effective_num_ctx( (Some(r), Some(served)) => r.min(served), (None, Some(served)) => served, (Some(r), None) => r, - (None, None) => env_default_num_ctx() as usize, + (None, None) => default_num_ctx() as usize, } } diff --git a/src/ai/llamacpp.rs b/src/ai/llamacpp.rs index 9bcd24d..07d1afa 100644 --- a/src/ai/llamacpp.rs +++ b/src/ai/llamacpp.rs @@ -41,7 +41,6 @@ const DEFAULT_PRIMARY_MODEL: &str = "chat"; const DEFAULT_VISION_MODEL: &str = "vision"; const DEFAULT_EMBEDDING_MODEL: &str = "embed"; const DEFAULT_TTS_MODEL: &str = "chatterbox"; -const DEFAULT_REQUEST_TIMEOUT_SECS: u64 = 180; /// How long a served-context-window lookup stays cached per model. const CONTEXT_WINDOW_CACHE_TTL: Duration = Duration::from_secs(600); @@ -90,10 +89,7 @@ pub struct LlamaCppClient { impl LlamaCppClient { pub fn new(base_url: Option, primary_model: Option) -> Self { - let timeout_secs = std::env::var("LLAMA_SWAP_REQUEST_TIMEOUT_SECONDS") - .ok() - .and_then(|v| v.parse::().ok()) - .unwrap_or(DEFAULT_REQUEST_TIMEOUT_SECS); + let timeout_secs = crate::ai::config::llm_config().llamacpp.timeout_secs; let pm = primary_model.unwrap_or_else(|| DEFAULT_PRIMARY_MODEL.to_string()); Self { client: Client::builder() @@ -287,11 +283,7 @@ impl LlamaCppClient { // TTS gets its own (longer) timeout: synthesizing a long, internally // chunked insight can take minutes, well past the shared chat/embedding // client timeout. Per-request `.timeout()` overrides the client default. - let tts_timeout = std::env::var("LLAMA_SWAP_TTS_REQUEST_TIMEOUT_SECONDS") - .ok() - .and_then(|v| v.parse::().ok()) - .filter(|n| *n > 0) - .unwrap_or(600); + let tts_timeout = crate::ai::config::llm_config().tts.timeout_secs; let resp = self .client diff --git a/src/ai/local_llm.rs b/src/ai/local_llm.rs index 23c3daa..ad5c96e 100644 --- a/src/ai/local_llm.rs +++ b/src/ai/local_llm.rs @@ -32,27 +32,28 @@ impl LocalLlm { Self { ollama, llamacpp } } - /// Construct from the canonical env wiring shared with `AppState`. - pub fn from_env() -> Self { + /// Construct from `llm.toml`, the same wiring `AppState` uses. + pub fn from_config() -> Self { + let cfg = crate::ai::config::llm_config(); Self::new( - crate::state::build_ollama_from_env(), - crate::state::build_llamacpp_from_env(), + crate::state::build_ollama(&cfg.ollama), + crate::state::build_llamacpp(cfg), ) } - /// Embed a search query (applies `EMBED_QUERY_PREFIX`). Callers must + /// Embed a search query (applies `embedding.query_prefix`). Callers must /// pick query vs document — retrieval models treat the two sides /// differently and an unmarked embed invites prefix-mismatch bugs. pub async fn embed_query(&self, text: &str) -> Result> { super::embed_query(&self.ollama, self.llamacpp.as_deref(), text).await } - /// Embed corpus text (applies `EMBED_DOCUMENT_PREFIX`). + /// Embed corpus text (applies `embedding.document_prefix`). pub async fn embed_document(&self, text: &str) -> Result> { super::embed_document(&self.ollama, self.llamacpp.as_deref(), text).await } - /// Single-shot local text generation via the `LLM_BACKEND`-selected + /// Single-shot local text generation via the `backend`-selected /// client (offline tooling; chat turns belong to `ResolvedBackend`). pub async fn generate(&self, prompt: &str, system: Option<&str>) -> Result { if super::local_backend_is_llamacpp() { @@ -60,8 +61,8 @@ impl LocalLlm { return ::generate(lc, prompt, system, None).await; } anyhow::bail!( - "LLM_BACKEND=llamacpp but LlamaCppClient is unconfigured — \ - set LLAMA_SWAP_URL or switch to LLM_BACKEND=ollama" + "backend = \"llamacpp\" but LlamaCppClient is unconfigured — \ + set llamacpp.url in llm.toml or switch to backend = \"ollama\"" ); } self.ollama.generate(prompt, system).await diff --git a/src/ai/mod.rs b/src/ai/mod.rs index 5d4cf5d..a1c798c 100644 --- a/src/ai/mod.rs +++ b/src/ai/mod.rs @@ -68,51 +68,26 @@ pub fn user_display_name() -> String { std::env::var("USER_NAME").unwrap_or_else(|_| "Me".to_string()) } -/// One switch for the "local" LLM stack: when `LLM_BACKEND=llamacpp` is -/// set, chat / vision describe / embeddings all route through llama-swap -/// instead of Ollama. Any other value (including unset, the default) is -/// Ollama. This is intentionally global — embeddings must be drawn from -/// a single source or similarity search across the index breaks (mixed -/// vector spaces, possibly mixed dims). The `backend=hybrid` per-request -/// override remains orthogonal: it always sends chat to OpenRouter, and -/// uses `LLM_BACKEND` for the describe-then-inline vision pass. +/// One switch for the "local" LLM stack: with `backend = "llamacpp"` in +/// `llm.toml`, chat / vision describe / embeddings all route through +/// llama-swap instead of Ollama. Intentionally global — embeddings must be +/// drawn from a single source or similarity search across the index breaks +/// (mixed vector spaces, possibly mixed dims). The `backend=hybrid` +/// per-request override remains orthogonal: it always sends chat to +/// OpenRouter, and uses this switch for the describe-then-inline vision pass. pub fn local_backend_is_llamacpp() -> bool { - matches!( - std::env::var("LLM_BACKEND") - .ok() - .as_deref() - .map(|s| s.trim().to_lowercase()) - .as_deref(), - Some("llamacpp") - ) + config::llm_config().backend == config::LocalBackend::Llamacpp } -/// Expected embedding dimensionality, env-overridable via `EMBEDDING_DIM` -/// (default 768, nomic-embed-text). Every store/query dim check reads this — -/// swapping to a different-dim model (e.g. Qwen3-Embedding-0.6B at 1024) is -/// then a config flip plus a `reembed_embeddings` run, not a code change. -/// Cached for the process lifetime; a flip requires a restart anyway since -/// the corpus must be re-embedded with it. +/// Expected embedding dimensionality (`embedding.dim`, default 768 for +/// nomic-embed-text). Every store/query dim check reads this — swapping to a +/// different-dim model (e.g. Qwen3-Embedding-0.6B at 1024) is a config flip +/// plus a `reembed_embeddings` run, not a code change. pub fn embedding_dim() -> usize { - static DIM: std::sync::OnceLock = std::sync::OnceLock::new(); - *DIM.get_or_init(|| { - std::env::var("EMBEDDING_DIM") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(768) - }) + config::llm_config().embedding.dim } -/// Read an embedding prefix from the environment. `.env` values can't hold -/// real newlines, so a literal `\n` in the value is expanded — Qwen3-style -/// query instructions need one ("Instruct: ...\nQuery: "). -fn embed_prefix(key: &str) -> String { - std::env::var(key) - .map(|v| v.replace("\\n", "\n")) - .unwrap_or_default() -} - -/// Embed a search query. Applies `EMBED_QUERY_PREFIX` (default empty) — +/// Embed a search query. Applies `embedding.query_prefix` (default empty) — /// retrieval models distinguish query-side from document-side text: /// nomic v1.5 wants `search_query: `, Qwen3-Embedding wants /// `Instruct: \nQuery: `. Must pair with the document prefix the @@ -122,24 +97,24 @@ pub async fn embed_query( llamacpp: Option<&LlamaCppClient>, text: &str, ) -> anyhow::Result> { - let prefixed = format!("{}{}", embed_prefix("EMBED_QUERY_PREFIX"), text); + let prefixed = format!("{}{}", config::llm_config().embedding.query_prefix, text); embed_one(ollama, llamacpp, &prefixed).await } /// Embed corpus text (the stored side of retrieval). Applies -/// `EMBED_DOCUMENT_PREFIX` (default empty; nomic v1.5 wants +/// `embedding.document_prefix` (default empty; nomic v1.5 wants /// `search_document: `, Qwen3-Embedding wants none). pub async fn embed_document( ollama: &OllamaClient, llamacpp: Option<&LlamaCppClient>, text: &str, ) -> anyhow::Result> { - let prefixed = format!("{}{}", embed_prefix("EMBED_DOCUMENT_PREFIX"), text); + let prefixed = format!("{}{}", config::llm_config().embedding.document_prefix, text); embed_one(ollama, llamacpp, &prefixed).await } /// Embed a batch of strings via the configured local backend. Routes -/// through llama-swap when `LLM_BACKEND=llamacpp` (and a client is +/// through llama-swap when `backend = "llamacpp"` (and a client is /// configured), else Ollama. See [`local_backend_is_llamacpp`] for the /// rationale on consistency. pub async fn embed_many( @@ -152,8 +127,8 @@ pub async fn embed_many( return ::generate_embeddings(lc, texts).await; } anyhow::bail!( - "LLM_BACKEND=llamacpp but LlamaCppClient is unconfigured — \ - set LLAMA_SWAP_URL or switch to LLM_BACKEND=ollama" + "backend = \"llamacpp\" but LlamaCppClient is unconfigured — \ + set llamacpp.url in llm.toml or switch to backend = \"ollama\"" ); } ollama.generate_embeddings(texts).await @@ -170,50 +145,3 @@ pub async fn embed_one( vecs.pop() .ok_or_else(|| anyhow::anyhow!("embedding backend returned no embeddings")) } - -#[cfg(test)] -mod env_dispatch_tests { - use super::*; - - /// Env vars are process-global, and the test harness runs in parallel — - /// without this lock the `LLM_BACKEND` tests race each other and flake. - static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); - - fn with_env(key: &str, val: Option<&str>, f: F) { - let _guard = ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner()); - let prev = std::env::var(key).ok(); - match val { - Some(v) => unsafe { std::env::set_var(key, v) }, - None => unsafe { std::env::remove_var(key) }, - } - f(); - match prev { - Some(v) => unsafe { std::env::set_var(key, v) }, - None => unsafe { std::env::remove_var(key) }, - } - } - - #[test] - fn llm_backend_defaults_to_ollama() { - with_env("LLM_BACKEND", None, || { - assert!(!local_backend_is_llamacpp()); - }); - } - - #[test] - fn llm_backend_llamacpp_case_insensitive() { - with_env("LLM_BACKEND", Some("LlamaCpp"), || { - assert!(local_backend_is_llamacpp()); - }); - with_env("LLM_BACKEND", Some(" llamacpp "), || { - assert!(local_backend_is_llamacpp()); - }); - } - - #[test] - fn llm_backend_unknown_value_is_ollama() { - with_env("LLM_BACKEND", Some("vllm"), || { - assert!(!local_backend_is_llamacpp()); - }); - } -} diff --git a/src/ai/ollama.rs b/src/ai/ollama.rs index bf7c454..de70864 100644 --- a/src/ai/ollama.rs +++ b/src/ai/ollama.rs @@ -23,17 +23,11 @@ pub use crate::ai::llm_client::{ToolCall, ToolCallFunction, ToolFunction}; // Cache duration: 15 minutes const CACHE_DURATION_SECS: u64 = 15 * 60; -/// Default total request timeout for generation calls, in seconds. -/// Overridable via `OLLAMA_REQUEST_TIMEOUT_SECONDS` env var for slow -/// CPU-offloaded models where inference can take several minutes. -const DEFAULT_REQUEST_TIMEOUT_SECS: u64 = 120; - +/// Total request timeout for generation calls (`ollama.timeout_secs`, +/// default 120). Raise it for slow CPU-offloaded models where inference can +/// take several minutes. fn configured_request_timeout_secs() -> u64 { - std::env::var("OLLAMA_REQUEST_TIMEOUT_SECONDS") - .ok() - .and_then(|v| v.parse::().ok()) - .filter(|&s| s > 0) - .unwrap_or(DEFAULT_REQUEST_TIMEOUT_SECS) + crate::ai::config::llm_config().ollama.timeout_secs } /// Embedding model used across the app. Callers that persist a diff --git a/src/ai/persona_chat.rs b/src/ai/persona_chat.rs index 6b3bb8c..4c27fc3 100644 --- a/src/ai/persona_chat.rs +++ b/src/ai/persona_chat.rs @@ -37,7 +37,7 @@ use uuid::Uuid; use crate::ai::backend::{BackendKind, SamplingOverrides}; use crate::ai::handlers::ReplayQuery; use crate::ai::insight_chat::{ - ChatStreamEvent, ContextFitter, DEFAULT_MAX_ITERATIONS, ForkInfo, env_max_iterations, + ChatStreamEvent, ContextFitter, DEFAULT_MAX_ITERATIONS, ForkInfo, max_chat_iterations, render_tree_path, }; use crate::ai::insight_generator::InsightGenerator; @@ -954,7 +954,7 @@ impl PersonaChatSession { Some("hybrid") => BackendKind::Hybrid, _ => BackendKind::Local, }; - let max_iterations = DEFAULT_MAX_ITERATIONS.clamp(1, env_max_iterations()); + let max_iterations = DEFAULT_MAX_ITERATIONS.clamp(1, max_chat_iterations()); let overrides = SamplingOverrides { model: req.model.clone().filter(|m| !m.is_empty()), num_ctx: req.num_ctx, diff --git a/src/ai/tts.rs b/src/ai/tts.rs index cd102d7..53183b8 100644 --- a/src/ai/tts.rs +++ b/src/ai/tts.rs @@ -169,13 +169,9 @@ fn sanitize_voice_name(raw: &str) -> Option { /// Reference-clip cap in seconds for voice cloning. Chatterbox is zero-shot — /// a clean ~10–20s sample is the sweet spot and more rarely helps. Tune via -/// `LLAMA_SWAP_TTS_REF_SECONDS` (default 30). +/// `tts.ref_seconds` (default 30). fn tts_ref_seconds() -> u32 { - std::env::var("LLAMA_SWAP_TTS_REF_SECONDS") - .ok() - .and_then(|s| s.trim().parse::().ok()) - .filter(|n| *n > 0) - .unwrap_or(30) + crate::ai::config::llm_config().tts.ref_seconds } /// Tag a (sanitized) voice name with the reference window used to create it: @@ -228,12 +224,9 @@ fn resolve_ref_window( } /// Optional default voice for synthesis when the request doesn't name one. -/// Set `LLAMA_SWAP_TTS_VOICE=m` to read insights in a cloned voice by default. +/// Set `tts.voice = "m"` to read insights in a cloned voice by default. fn default_voice() -> Option { - std::env::var("LLAMA_SWAP_TTS_VOICE") - .ok() - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) + crate::ai::config::llm_config().tts.voice.clone() } // Markdown / formatting strippers, compiled once. Insight text is markdown, diff --git a/src/bin/import_calendar.rs b/src/bin/import_calendar.rs index 3645db5..2d140b3 100644 --- a/src/bin/import_calendar.rs +++ b/src/bin/import_calendar.rs @@ -45,7 +45,7 @@ async fn main() -> Result<()> { // LocalLlm dispatches per LLM_BACKEND, so embeddings written here land // in the same vector space the query side searches. let llm = if args.generate_embeddings { - Some(LocalLlm::from_env()) + Some(LocalLlm::from_config()) } else { None }; diff --git a/src/bin/import_search_history.rs b/src/bin/import_search_history.rs index eee0a80..566c59d 100644 --- a/src/bin/import_search_history.rs +++ b/src/bin/import_search_history.rs @@ -40,7 +40,7 @@ async fn main() -> Result<()> { // LocalLlm dispatches per LLM_BACKEND, so embeddings written here land // in the same vector space the query side searches. - let llm = LocalLlm::from_env(); + let llm = LocalLlm::from_config(); let mut inserted_count = 0usize; let mut skipped_count = 0usize; diff --git a/src/bin/populate_knowledge.rs b/src/bin/populate_knowledge.rs index da7b3d7..9acd91d 100644 --- a/src/bin/populate_knowledge.rs +++ b/src/bin/populate_knowledge.rs @@ -129,24 +129,19 @@ async fn main() -> anyhow::Result<()> { } }; - // Ollama config from env with CLI overrides. - let primary_url = std::env::var("OLLAMA_PRIMARY_URL") - .or_else(|_| std::env::var("OLLAMA_URL")) - .unwrap_or_else(|_| "http://localhost:11434".to_string()); - let fallback_url = std::env::var("OLLAMA_FALLBACK_URL").ok(); + // Ollama config from llm.toml with CLI overrides. + let ollama_cfg = &image_api::ai::config::llm_config().ollama; + let primary_url = ollama_cfg.primary_url.clone(); let primary_model = args .model .clone() - .or_else(|| std::env::var("OLLAMA_PRIMARY_MODEL").ok()) - .or_else(|| std::env::var("OLLAMA_MODEL").ok()) - .unwrap_or_else(|| "nemotron-3-nano:30b".to_string()); - let fallback_model = std::env::var("OLLAMA_FALLBACK_MODEL").ok(); + .unwrap_or_else(|| ollama_cfg.primary_model.clone()); let mut ollama = OllamaClient::new( primary_url.clone(), - fallback_url, + ollama_cfg.fallback_url.clone(), primary_model.clone(), - fallback_model, + ollama_cfg.fallback_model.clone(), ) .with_request_timeout(args.timeout_secs); diff --git a/src/bin/reembed_embeddings.rs b/src/bin/reembed_embeddings.rs index a2fdd4c..67e72f2 100644 --- a/src/bin/reembed_embeddings.rs +++ b/src/bin/reembed_embeddings.rs @@ -284,7 +284,7 @@ async fn main() -> Result<()> { let mut conn = SqliteConnection::establish(&database_url) .with_context(|| format!("connecting to {}", database_url))?; - let llm = LocalLlm::from_env(); + let llm = LocalLlm::from_config(); let model_version = llm.embedding_model_version(); println!("Embedding via '{}'", model_version); if args.dry_run { diff --git a/src/bin/test_daily_summary.rs b/src/bin/test_daily_summary.rs index 2337fa9..53d32b9 100644 --- a/src/bin/test_daily_summary.rs +++ b/src/bin/test_daily_summary.rs @@ -66,6 +66,7 @@ async fn main() -> Result<()> { env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); let args = Args::parse(); + let ollama_cfg = &image_api::ai::config::llm_config().ollama; // Parse dates let start_date = NaiveDate::parse_from_str(&args.start, "%Y-%m-%d") @@ -82,12 +83,7 @@ async fn main() -> Result<()> { if let Some(ref model) = args.model { println!("Model: {}", model); } else { - println!( - "Model: {} (from env)", - env::var("OLLAMA_PRIMARY_MODEL") - .or_else(|_| env::var("OLLAMA_MODEL")) - .unwrap_or_else(|_| "nemotron-3-nano:30b".to_string()) - ); + println!("Model: {} (from llm config)", ollama_cfg.primary_model); } if args.test_mode { println!("⚠ TEST MODE: Results will NOT be saved to database"); @@ -96,22 +92,15 @@ async fn main() -> Result<()> { println!(); // Initialize AI clients - let ollama_primary_url = env::var("OLLAMA_PRIMARY_URL") - .or_else(|_| env::var("OLLAMA_URL")) - .unwrap_or_else(|_| "http://localhost:11434".to_string()); - - let ollama_fallback_url = env::var("OLLAMA_FALLBACK_URL").ok(); - - // Use provided model or fallback to env - let model_to_use = args.model.clone().unwrap_or_else(|| { - env::var("OLLAMA_PRIMARY_MODEL") - .or_else(|_| env::var("OLLAMA_MODEL")) - .unwrap_or_else(|_| "nemotron-3-nano:30b".to_string()) - }); + // Use provided model or fall back to the configured one + let model_to_use = args + .model + .clone() + .unwrap_or_else(|| ollama_cfg.primary_model.clone()); let mut ollama = OllamaClient::new( - ollama_primary_url, - ollama_fallback_url.clone(), + ollama_cfg.primary_url.clone(), + ollama_cfg.fallback_url.clone(), model_to_use.clone(), Some(model_to_use), // Use same model for fallback ); diff --git a/src/state.rs b/src/state.rs index 30ac59c..da6b997 100644 --- a/src/state.rs +++ b/src/state.rs @@ -1,5 +1,6 @@ use crate::ai::apollo_client::ApolloClient; use crate::ai::clip_client::ClipClient; +use crate::ai::config::{LlmConfig, OllamaConfig, OpenRouterConfig, llm_config}; use crate::ai::face_client::FaceClient; use crate::ai::insight_chat::{ChatLockMap, InsightChatService}; use crate::ai::llamacpp::LlamaCppClient; @@ -224,13 +225,14 @@ impl AppState { impl Default for AppState { fn default() -> Self { // Initialize AI clients - let ollama = build_ollama_from_env(); + let llm = llm_config(); + let ollama = build_ollama(&llm.ollama); - let openrouter = build_openrouter_from_env(); - let openrouter_allowed_models = parse_openrouter_allowed_models(); + let openrouter = build_openrouter(&llm.openrouter); + let openrouter_allowed_models = llm.openrouter.allowed.clone(); - let llamacpp = build_llamacpp_from_env(); - let llamacpp_allowed_models = parse_llamacpp_allowed_models(); + let llamacpp = build_llamacpp(llm); + let llamacpp_allowed_models = llm.llamacpp.picker.include.clone().unwrap_or_default(); let sms_api_url = env::var("SMS_API_URL").unwrap_or_else(|_| "http://localhost:8000".to_string()); @@ -346,10 +348,7 @@ impl Default for AppState { // Turn registry for reconnectable chat turns. 5-minute timeout for // stale turns (background cleaner drops entries older than this). - let timeout_secs: u64 = env::var("INSIGHT_CHAT_TURN_TIMEOUT_SECS") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(300); + let timeout_secs = llm_config().agentic.chat_turn_timeout_secs; let turn_registry = Arc::new(TurnRegistry::new(timeout_secs)); // Ensure preview clips directory exists @@ -387,94 +386,46 @@ impl Default for AppState { } } -/// Build an `OpenRouterClient` from environment variables. Returns `None` -/// when `OPENROUTER_API_KEY` is unset (the hybrid backend is then -/// unavailable and requests for it return a clear error). -fn build_openrouter_from_env() -> Option> { +/// Build an `OpenRouterClient` from `[openrouter]`. Returns `None` when +/// `OPENROUTER_API_KEY` (env-only, it's a secret) is unset — the hybrid +/// backend is then unavailable and requests for it return a clear error. +fn build_openrouter(cfg: &OpenRouterConfig) -> Option> { let api_key = env::var("OPENROUTER_API_KEY").ok()?; - let base_url = env::var("OPENROUTER_BASE_URL").ok(); - let default_model = env::var("OPENROUTER_DEFAULT_MODEL") - .unwrap_or_else(|_| "anthropic/claude-sonnet-4".to_string()); - let mut client = OpenRouterClient::new(api_key, base_url, default_model); - client.set_attribution( - env::var("OPENROUTER_HTTP_REFERER").ok(), - env::var("OPENROUTER_APP_TITLE").ok(), - ); - if let Ok(model) = env::var("OPENROUTER_EMBEDDING_MODEL") { - client.set_embedding_model(model); - } - // OPENROUTER_MAX_TOKENS=0 omits the cap (provider default). - if let Some(n) = env::var("OPENROUTER_MAX_TOKENS") - .ok() - .and_then(|v| v.trim().parse::().ok()) - { - client.set_max_tokens((n > 0).then_some(n)); + let mut client = + OpenRouterClient::new(api_key, cfg.base_url.clone(), cfg.default_model.clone()); + client.set_attribution(cfg.http_referer.clone(), cfg.app_title.clone()); + if let Some(model) = &cfg.embedding_model { + client.set_embedding_model(model.clone()); } + client.set_max_tokens(cfg.max_tokens_cap()); Some(Arc::new(client)) } -/// Parse `OPENROUTER_ALLOWED_MODELS` (comma-separated) into a vec. Returns -/// empty when unset, in which case `/insights/openrouter/models` reports no -/// curated picks and the server falls back to `OPENROUTER_DEFAULT_MODEL`. -fn parse_openrouter_allowed_models() -> Vec { - env::var("OPENROUTER_ALLOWED_MODELS") - .unwrap_or_default() - .split(',') - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) - .collect() +/// Build the `OllamaClient` from `[ollama]` — shared by the server +/// (`AppState::default`) and the standalone binaries. +pub fn build_ollama(cfg: &OllamaConfig) -> OllamaClient { + OllamaClient::new( + cfg.primary_url.clone(), + cfg.fallback_url.clone(), + cfg.primary_model.clone(), + cfg.fallback_model.clone(), + ) } -/// Build the `OllamaClient` from environment variables — the canonical -/// `OLLAMA_*` wiring shared by the server (`AppState::default`) and the -/// standalone binaries (which predate this helper and used to copy it). -pub fn build_ollama_from_env() -> OllamaClient { - let primary_url = env::var("OLLAMA_PRIMARY_URL").unwrap_or_else(|_| { - env::var("OLLAMA_URL").unwrap_or_else(|_| "http://localhost:11434".to_string()) - }); - let fallback_url = env::var("OLLAMA_FALLBACK_URL").ok(); - let primary_model = env::var("OLLAMA_PRIMARY_MODEL") - .or_else(|_| env::var("OLLAMA_MODEL")) - .unwrap_or_else(|_| "nemotron-3-nano:30b".to_string()); - let fallback_model = env::var("OLLAMA_FALLBACK_MODEL").ok(); - - OllamaClient::new(primary_url, fallback_url, primary_model, fallback_model) -} - -/// Build a `LlamaCppClient` from environment variables. Returns `None` when -/// `LLAMA_SWAP_URL` is unset. The client is constructed unconditionally -/// when the URL is set (so it's available even under `LLM_BACKEND=ollama` -/// for ad-hoc tooling), but the agentic / chat paths only route through it -/// when `LLM_BACKEND=llamacpp`. Slot ids default to the names the bundled -/// `llama-swap/config.yaml` uses — `chat` / `vision` / `embed`. -pub fn build_llamacpp_from_env() -> Option> { - let base_url = env::var("LLAMA_SWAP_URL").ok()?; - let primary_model = env::var("LLAMA_SWAP_PRIMARY_MODEL").ok(); - let mut client = LlamaCppClient::new(Some(base_url), primary_model); - if let Ok(model) = env::var("LLAMA_SWAP_EMBEDDING_MODEL") { - client.set_embedding_model(model); - } - if let Ok(model) = env::var("LLAMA_SWAP_VISION_MODEL") { - client.set_vision_model(model); - } - if let Ok(model) = env::var("LLAMA_SWAP_TTS_MODEL") { - client.set_tts_model(model); - } +/// Build a `LlamaCppClient` from `[llamacpp]` (+ the embed and TTS model +/// ids). Returns `None` when `llamacpp.url` is unset. The client is built +/// whenever the URL is set (TTS and ad-hoc tooling use it even under +/// `backend = "ollama"`), but chat / embeddings only route through it when +/// `backend = "llamacpp"`. +pub fn build_llamacpp(cfg: &LlmConfig) -> Option> { + let base_url = cfg.llamacpp.url.clone()?; + let mut client = LlamaCppClient::new(Some(base_url), Some(cfg.llamacpp.chat.clone())); + client.set_embedding_model(cfg.embedding.model.clone()); + client.set_vision_model(cfg.llamacpp.vision_model().to_string()); + client.set_tts_model(cfg.tts.model.clone()); Some(Arc::new(client)) } -/// Parse `LLAMA_SWAP_ALLOWED_MODELS` (comma-separated) into a vec. Used to -/// populate the model picker when `LLM_BACKEND=llamacpp` — `/insights/models` -/// surfaces these slots with capabilities. Empty when unset. -fn parse_llamacpp_allowed_models() -> Vec { - env::var("LLAMA_SWAP_ALLOWED_MODELS") - .unwrap_or_default() - .split(',') - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) - .collect() -} - #[cfg(test)] impl AppState { /// Creates an AppState instance for testing with temporary directories diff --git a/src/unified_search.rs b/src/unified_search.rs index aab0936..23a85ba 100644 --- a/src/unified_search.rs +++ b/src/unified_search.rs @@ -639,15 +639,15 @@ async fn translate( // llama-swap); local only, no hybrid, per the feature's design. // // Translation-model precedence: - // 1. UNIFIED_SEARCH_MODEL env — pin a small, fast model that can stay + // 1. `unified_search_model` (llm.toml) — pin a small, fast model that can stay // co-resident with CLIP (and the chat model) so translation never // evicts them. This is the recommended setup on a tight VRAM budget. // 2. the client-selected model — routes translation to whatever the user // already has loaded (no swap) when no dedicated model is pinned. // 3. None → resolve_backend uses the configured default local model. - let translation_model = std::env::var("UNIFIED_SEARCH_MODEL") - .ok() - .filter(|m| !m.trim().is_empty()) + let translation_model = crate::ai::config::llm_config() + .unified_search_model + .clone() .or_else(|| query.model.clone()) .filter(|m| !m.trim().is_empty()); let overrides = SamplingOverrides {