Apps/ImageApi
1
0 Fork 0
Code Issues 8 Pull Requests Releases Wiki Activity

refactor(ai): read LLM settings from LlmConfig instead of env

Browse Source 
Every LLM env reader now goes through ai::config::llm_config():
build_ollama / build_llamacpp / build_openrouter take config sections
(OPENROUTER_API_KEY is still read from env), and the backend switch,
embedding dim + prefixes, Ollama / llama.cpp / TTS timeouts, TTS voice and
ref length, agentic caps, chat turn timeout and the unified-search model
read their sections. LocalLlm::from_env -> from_config; the
test_daily_summary and populate_knowledge bins take Ollama defaults from
the config. env_max_iterations / env_default_num_ctx are renamed to
max_chat_iterations / default_num_ctx.

The picker still uses picker.include (= LLAMA_SWAP_ALLOWED_MODELS) until
discovery lands. Legacy env vars keep working through the config's
override layer; checked against the deployed .env, which resolves to the
same values. The LLM_BACKEND parsing tests move to ai::config.

llm.example.toml now leaves the embed prefixes empty: copying the
nomic prefixes onto a corpus embedded without them breaks search.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Cameron CordesandClaude Opus 5.5 committed 2026-10-26 17:30:24 -04:00
1 parent 5b8a99a2b2
commit aa3536afee
17 files changed
+144 -292
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/ai/config.rs
+18 -2
View File
@@ -221,7 +221,7 @@ impl Default for AgenticConfig {
Self {
max_iterations: 12,
chat_max_iterations: crate::ai::insight_chat::DEFAULT_MAX_ITERATIONS,
chat_default_num_ctx: 32768,
chat_default_num_ctx: crate::ai::insight_chat::DEFAULT_NUM_CTX,
chat_turn_timeout_secs: 300,
}
}
@@ -500,7 +500,7 @@ mod tests {
let c = r.config;
assert_eq!(c.backend, LocalBackend::Llamacpp);
assert!(c.llamacpp.url.is_some());
assert!(c.embedding.query_prefix.ends_with(' '));
assert_eq!(c.embedding.dim, 768);
}
#[test]
@@ -539,6 +539,22 @@ mod tests {
);
}
#[test]
fn llm_backend_env_is_case_insensitive_and_unknown_means_ollama() {
let backend = |v: &str| {
resolve(
None,
env_of(&[("LLM_BACKEND", v), ("LLAMA_SWAP_URL", "http://x/v1")]),
)
.unwrap()
.config
.backend
};
assert_eq!(backend("LlamaCpp"), LocalBackend::Llamacpp);
assert_eq!(backend(" llamacpp "), LocalBackend::Llamacpp);
assert_eq!(backend("vllm"), LocalBackend::Ollama);
}
#[test]
fn blank_env_values_count_as_unset() {
let r = resolve(
src/ai/handlers.rs
+1 -4
View File
@@ -867,10 +867,7 @@ pub async fn generate_agentic_insight_handler(
.persona_system_prompt(user_id, &persona_id)
});
let max_iterations: usize = std::env::var("AGENTIC_MAX_ITERATIONS")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(12);
let max_iterations = crate::ai::config::llm_config().agentic.max_iterations;
// Spawn background task with timeout
let generator = app_state.insight_generator.clone();
src/ai/insight_chat.rs
+19 -25
View File
@@ -26,8 +26,8 @@ pub const DEFAULT_MAX_ITERATIONS: usize = 6;
/// `effective_num_ctx`). Real conversations rarely pass ~16k tokens, so 32k
/// keeps the truncation pass from gutting history the server could hold
/// (which also destroys the server's KV-cache prefix reuse). Override
/// per-deploy with AGENTIC_CHAT_DEFAULT_NUM_CTX.
const DEFAULT_NUM_CTX: i32 = 32768;
/// per-deploy with `agentic.chat_default_num_ctx` in llm.toml.
pub(crate) const DEFAULT_NUM_CTX: i32 = 32768;
/// Headroom reserved for the model's response, deducted from the context
/// budget when deciding whether to truncate the replayed history.
pub(crate) const RESPONSE_HEADROOM_TOKENS: usize = 2048;
@@ -341,7 +341,7 @@ impl InsightChatService {
let max_iterations = req
.max_iterations
.unwrap_or(DEFAULT_MAX_ITERATIONS)
.clamp(1, env_max_iterations());
.clamp(1, max_chat_iterations());
span.set_attribute("max_iterations", max_iterations as i64);
let stored_model = insight.model_version.clone();
@@ -1043,7 +1043,7 @@ impl InsightChatService {
let max_iterations = req
.max_iterations
.unwrap_or(DEFAULT_MAX_ITERATIONS)
.clamp(1, env_max_iterations());
.clamp(1, max_chat_iterations());
let stored_model = insight.model_version.clone();
let overrides = SamplingOverrides {
@@ -1246,7 +1246,7 @@ impl InsightChatService {
let max_iterations = req
.max_iterations
.unwrap_or(DEFAULT_MAX_ITERATIONS)
.clamp(1, env_max_iterations());
.clamp(1, max_chat_iterations());
let overrides = SamplingOverrides {
model: req.model.clone().filter(|m| !m.is_empty()),
@@ -1526,7 +1526,7 @@ impl InsightChatService {
let max_iterations = req
.max_iterations
.unwrap_or(DEFAULT_MAX_ITERATIONS)
.clamp(1, env_max_iterations());
.clamp(1, max_chat_iterations());
let stored_model = insight.model_version.clone();
let overrides = SamplingOverrides {
@@ -1721,7 +1721,7 @@ impl InsightChatService {
let max_iterations = req
.max_iterations
.unwrap_or(DEFAULT_MAX_ITERATIONS)
.clamp(1, env_max_iterations());
.clamp(1, max_chat_iterations());
let overrides = SamplingOverrides {
model: req.model.clone().filter(|m| !m.is_empty()),
@@ -2415,25 +2415,19 @@ pub(crate) fn find_raw_cut(
None
}
/// Read AGENTIC_CHAT_MAX_ITERATIONS once per call. Cheap; keeps the code
/// free of static globals and lets the operator change the cap by env without
/// a restart in test harnesses (the running server still caches via Default).
pub fn env_max_iterations() -> usize {
std::env::var("AGENTIC_CHAT_MAX_ITERATIONS")
.ok()
.and_then(|s| s.parse::<usize>().ok())
.unwrap_or(DEFAULT_MAX_ITERATIONS)
.max(1)
/// Cap on tool-calling iterations per chat turn (`agentic.chat_max_iterations`,
/// validated >= 1). Per-request values are clamped to it.
pub fn max_chat_iterations() -> usize {
crate::ai::config::llm_config().agentic.chat_max_iterations
}
/// Read AGENTIC_CHAT_DEFAULT_NUM_CTX once per call — the assumed context
/// window for the truncation budget when the request omits `num_ctx`. Same
/// no-static-global rationale as `env_max_iterations` above.
pub(crate) fn env_default_num_ctx() -> i32 {
std::env::var("AGENTIC_CHAT_DEFAULT_NUM_CTX")
.ok()
.and_then(|s| s.parse::<i32>().ok())
.unwrap_or(DEFAULT_NUM_CTX)
/// Assumed context window for the truncation budget when the request omits
/// `num_ctx` and the backend can't report one (`agentic.chat_default_num_ctx`).
/// Floored so the budget always leaves room for the response headroom.
pub(crate) fn default_num_ctx() -> i32 {
crate::ai::config::llm_config()
.agentic
.chat_default_num_ctx
.max(RESPONSE_HEADROOM_TOKENS as i32 + 1024)
}
@@ -2451,7 +2445,7 @@ pub(crate) async fn effective_num_ctx(
(Some(r), Some(served)) => r.min(served),
(None, Some(served)) => served,
(Some(r), None) => r,
(None, None) => env_default_num_ctx() as usize,
(None, None) => default_num_ctx() as usize,
}
}
src/ai/llamacpp.rs
+2 -10
View File
@@ -41,7 +41,6 @@ const DEFAULT_PRIMARY_MODEL: &str = "chat";
const DEFAULT_VISION_MODEL: &str = "vision";
const DEFAULT_EMBEDDING_MODEL: &str = "embed";
const DEFAULT_TTS_MODEL: &str = "chatterbox";
const DEFAULT_REQUEST_TIMEOUT_SECS: u64 = 180;
/// How long a served-context-window lookup stays cached per model.
const CONTEXT_WINDOW_CACHE_TTL: Duration = Duration::from_secs(600);
@@ -90,10 +89,7 @@ pub struct LlamaCppClient {
impl LlamaCppClient {
pub fn new(base_url: Option<String>, primary_model: Option<String>) -> Self {
let timeout_secs = std::env::var("LLAMA_SWAP_REQUEST_TIMEOUT_SECONDS")
.ok()
.and_then(|v| v.parse::<u64>().ok())
.unwrap_or(DEFAULT_REQUEST_TIMEOUT_SECS);
let timeout_secs = crate::ai::config::llm_config().llamacpp.timeout_secs;
let pm = primary_model.unwrap_or_else(|| DEFAULT_PRIMARY_MODEL.to_string());
Self {
client: Client::builder()
@@ -287,11 +283,7 @@ impl LlamaCppClient {
// TTS gets its own (longer) timeout: synthesizing a long, internally
// chunked insight can take minutes, well past the shared chat/embedding
// client timeout. Per-request `.timeout()` overrides the client default.
let tts_timeout = std::env::var("LLAMA_SWAP_TTS_REQUEST_TIMEOUT_SECONDS")
.ok()
.and_then(|v| v.parse::<u64>().ok())
.filter(|n| *n > 0)
.unwrap_or(600);
let tts_timeout = crate::ai::config::llm_config().tts.timeout_secs;
let resp = self
.client
src/ai/local_llm.rs
+10 -9
View File
@@ -32,27 +32,28 @@ impl LocalLlm {
Self { ollama, llamacpp }
}
/// Construct from the canonical env wiring shared with `AppState`.
pub fn from_env() -> Self {
/// Construct from `llm.toml`, the same wiring `AppState` uses.
pub fn from_config() -> Self {
let cfg = crate::ai::config::llm_config();
Self::new(
crate::state::build_ollama_from_env(),
crate::state::build_llamacpp_from_env(),
crate::state::build_ollama(&cfg.ollama),
crate::state::build_llamacpp(cfg),
)
}
/// Embed a search query (applies `EMBED_QUERY_PREFIX`). Callers must
/// Embed a search query (applies `embedding.query_prefix`). Callers must
/// pick query vs document — retrieval models treat the two sides
/// differently and an unmarked embed invites prefix-mismatch bugs.
pub async fn embed_query(&self, text: &str) -> Result<Vec<f32>> {
super::embed_query(&self.ollama, self.llamacpp.as_deref(), text).await
}
/// Embed corpus text (applies `EMBED_DOCUMENT_PREFIX`).
/// Embed corpus text (applies `embedding.document_prefix`).
pub async fn embed_document(&self, text: &str) -> Result<Vec<f32>> {
super::embed_document(&self.ollama, self.llamacpp.as_deref(), text).await
}
/// Single-shot local text generation via the `LLM_BACKEND`-selected
/// Single-shot local text generation via the `backend`-selected
/// client (offline tooling; chat turns belong to `ResolvedBackend`).
pub async fn generate(&self, prompt: &str, system: Option<&str>) -> Result<String> {
if super::local_backend_is_llamacpp() {
@@ -60,8 +61,8 @@ impl LocalLlm {
return <LlamaCppClient as LlmClient>::generate(lc, prompt, system, None).await;
}
anyhow::bail!(
"LLM_BACKEND=llamacpp but LlamaCppClient is unconfigured — \
set LLAMA_SWAP_URL or switch to LLM_BACKEND=ollama"
"backend = \"llamacpp\" but LlamaCppClient is unconfigured — \
set llamacpp.url in llm.toml or switch to backend = \"ollama\""
);
}
self.ollama.generate(prompt, system).await
src/ai/mod.rs
+20 -92
View File
@@ -68,51 +68,26 @@ pub fn user_display_name() -> String {
std::env::var("USER_NAME").unwrap_or_else(|_| "Me".to_string())
}
/// One switch for the "local" LLM stack: when `LLM_BACKEND=llamacpp` is
/// set, chat / vision describe / embeddings all route through llama-swap
/// instead of Ollama. Any other value (including unset, the default) is
/// Ollama. This is intentionally global — embeddings must be drawn from
/// a single source or similarity search across the index breaks (mixed
/// vector spaces, possibly mixed dims). The `backend=hybrid` per-request
/// override remains orthogonal: it always sends chat to OpenRouter, and
/// uses `LLM_BACKEND` for the describe-then-inline vision pass.
/// One switch for the "local" LLM stack: with `backend = "llamacpp"` in
/// `llm.toml`, chat / vision describe / embeddings all route through
/// llama-swap instead of Ollama. Intentionally global — embeddings must be
/// drawn from a single source or similarity search across the index breaks
/// (mixed vector spaces, possibly mixed dims). The `backend=hybrid`
/// per-request override remains orthogonal: it always sends chat to
/// OpenRouter, and uses this switch for the describe-then-inline vision pass.
pub fn local_backend_is_llamacpp() -> bool {
matches!(
std::env::var("LLM_BACKEND")
.ok()
.as_deref()
.map(|s| s.trim().to_lowercase())
.as_deref(),
Some("llamacpp")
)
config::llm_config().backend == config::LocalBackend::Llamacpp
}
/// Expected embedding dimensionality, env-overridable via `EMBEDDING_DIM`
/// (default 768, nomic-embed-text). Every store/query dim check reads this —
/// swapping to a different-dim model (e.g. Qwen3-Embedding-0.6B at 1024) is
/// then a config flip plus a `reembed_embeddings` run, not a code change.
/// Cached for the process lifetime; a flip requires a restart anyway since
/// the corpus must be re-embedded with it.
/// Expected embedding dimensionality (`embedding.dim`, default 768 for
/// nomic-embed-text). Every store/query dim check reads this — swapping to a
/// different-dim model (e.g. Qwen3-Embedding-0.6B at 1024) is a config flip
/// plus a `reembed_embeddings` run, not a code change.
pub fn embedding_dim() -> usize {
static DIM: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
*DIM.get_or_init(|| {
std::env::var("EMBEDDING_DIM")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(768)
})
config::llm_config().embedding.dim
}
/// Read an embedding prefix from the environment. `.env` values can't hold
/// real newlines, so a literal `\n` in the value is expanded — Qwen3-style
/// query instructions need one ("Instruct: ...\nQuery: ").
fn embed_prefix(key: &str) -> String {
std::env::var(key)
.map(|v| v.replace("\\n", "\n"))
.unwrap_or_default()
}
/// Embed a search query. Applies `EMBED_QUERY_PREFIX` (default empty) —
/// Embed a search query. Applies `embedding.query_prefix` (default empty) —
/// retrieval models distinguish query-side from document-side text:
/// nomic v1.5 wants `search_query: `, Qwen3-Embedding wants
/// `Instruct: <task>\nQuery: `. Must pair with the document prefix the
@@ -122,24 +97,24 @@ pub async fn embed_query(
llamacpp: Option<&LlamaCppClient>,
text: &str,
) -> anyhow::Result<Vec<f32>> {
let prefixed = format!("{}{}", embed_prefix("EMBED_QUERY_PREFIX"), text);
let prefixed = format!("{}{}", config::llm_config().embedding.query_prefix, text);
embed_one(ollama, llamacpp, &prefixed).await
}
/// Embed corpus text (the stored side of retrieval). Applies
/// `EMBED_DOCUMENT_PREFIX` (default empty; nomic v1.5 wants
/// `embedding.document_prefix` (default empty; nomic v1.5 wants
/// `search_document: `, Qwen3-Embedding wants none).
pub async fn embed_document(
ollama: &OllamaClient,
llamacpp: Option<&LlamaCppClient>,
text: &str,
) -> anyhow::Result<Vec<f32>> {
let prefixed = format!("{}{}", embed_prefix("EMBED_DOCUMENT_PREFIX"), text);
let prefixed = format!("{}{}", config::llm_config().embedding.document_prefix, text);
embed_one(ollama, llamacpp, &prefixed).await
}
/// Embed a batch of strings via the configured local backend. Routes
/// through llama-swap when `LLM_BACKEND=llamacpp` (and a client is
/// through llama-swap when `backend = "llamacpp"` (and a client is
/// configured), else Ollama. See [`local_backend_is_llamacpp`] for the
/// rationale on consistency.
pub async fn embed_many(
@@ -152,8 +127,8 @@ pub async fn embed_many(
return <LlamaCppClient as LlmClient>::generate_embeddings(lc, texts).await;
}
anyhow::bail!(
"LLM_BACKEND=llamacpp but LlamaCppClient is unconfigured — \
set LLAMA_SWAP_URL or switch to LLM_BACKEND=ollama"
"backend = \"llamacpp\" but LlamaCppClient is unconfigured — \
set llamacpp.url in llm.toml or switch to backend = \"ollama\""
);
}
ollama.generate_embeddings(texts).await
@@ -170,50 +145,3 @@ pub async fn embed_one(
vecs.pop()
.ok_or_else(|| anyhow::anyhow!("embedding backend returned no embeddings"))
}
#[cfg(test)]
mod env_dispatch_tests {
use super::*;
/// Env vars are process-global, and the test harness runs in parallel —
/// without this lock the `LLM_BACKEND` tests race each other and flake.
static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
fn with_env<F: FnOnce()>(key: &str, val: Option<&str>, f: F) {
let _guard = ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner());
let prev = std::env::var(key).ok();
match val {
Some(v) => unsafe { std::env::set_var(key, v) },
None => unsafe { std::env::remove_var(key) },
}
f();
match prev {
Some(v) => unsafe { std::env::set_var(key, v) },
None => unsafe { std::env::remove_var(key) },
}
}
#[test]
fn llm_backend_defaults_to_ollama() {
with_env("LLM_BACKEND", None, || {
assert!(!local_backend_is_llamacpp());
});
}
#[test]
fn llm_backend_llamacpp_case_insensitive() {
with_env("LLM_BACKEND", Some("LlamaCpp"), || {
assert!(local_backend_is_llamacpp());
});
with_env("LLM_BACKEND", Some(" llamacpp "), || {
assert!(local_backend_is_llamacpp());
});
}
#[test]
fn llm_backend_unknown_value_is_ollama() {
with_env("LLM_BACKEND", Some("vllm"), || {
assert!(!local_backend_is_llamacpp());
});
}
}
src/ai/ollama.rs
+4 -10
View File
@@ -23,17 +23,11 @@ pub use crate::ai::llm_client::{ToolCall, ToolCallFunction, ToolFunction};
// Cache duration: 15 minutes
const CACHE_DURATION_SECS: u64 = 15 * 60;
/// Default total request timeout for generation calls, in seconds.
/// Overridable via `OLLAMA_REQUEST_TIMEOUT_SECONDS` env var for slow
/// CPU-offloaded models where inference can take several minutes.
const DEFAULT_REQUEST_TIMEOUT_SECS: u64 = 120;
/// Total request timeout for generation calls (`ollama.timeout_secs`,
/// default 120). Raise it for slow CPU-offloaded models where inference can
/// take several minutes.
fn configured_request_timeout_secs() -> u64 {
std::env::var("OLLAMA_REQUEST_TIMEOUT_SECONDS")
.ok()
.and_then(|v| v.parse::<u64>().ok())
.filter(|&s| s > 0)
.unwrap_or(DEFAULT_REQUEST_TIMEOUT_SECS)
crate::ai::config::llm_config().ollama.timeout_secs
}
/// Embedding model used across the app. Callers that persist a
src/ai/persona_chat.rs
+2 -2
View File
@@ -37,7 +37,7 @@ use uuid::Uuid;
use crate::ai::backend::{BackendKind, SamplingOverrides};
use crate::ai::handlers::ReplayQuery;
use crate::ai::insight_chat::{
ChatStreamEvent, ContextFitter, DEFAULT_MAX_ITERATIONS, ForkInfo, env_max_iterations,
ChatStreamEvent, ContextFitter, DEFAULT_MAX_ITERATIONS, ForkInfo, max_chat_iterations,
render_tree_path,
};
use crate::ai::insight_generator::InsightGenerator;
@@ -954,7 +954,7 @@ impl PersonaChatSession {
Some("hybrid") => BackendKind::Hybrid,
_ => BackendKind::Local,
};
let max_iterations = DEFAULT_MAX_ITERATIONS.clamp(1, env_max_iterations());
let max_iterations = DEFAULT_MAX_ITERATIONS.clamp(1, max_chat_iterations());
let overrides = SamplingOverrides {
model: req.model.clone().filter(|m| !m.is_empty()),
num_ctx: req.num_ctx,
src/ai/tts.rs
+4 -11
View File
@@ -169,13 +169,9 @@ fn sanitize_voice_name(raw: &str) -> Option<String> {
/// Reference-clip cap in seconds for voice cloning. Chatterbox is zero-shot —
/// a clean ~10–20s sample is the sweet spot and more rarely helps. Tune via
/// `LLAMA_SWAP_TTS_REF_SECONDS` (default 30).
/// `tts.ref_seconds` (default 30).
fn tts_ref_seconds() -> u32 {
std::env::var("LLAMA_SWAP_TTS_REF_SECONDS")
.ok()
.and_then(|s| s.trim().parse::<u32>().ok())
.filter(|n| *n > 0)
.unwrap_or(30)
crate::ai::config::llm_config().tts.ref_seconds
}
/// Tag a (sanitized) voice name with the reference window used to create it:
@@ -228,12 +224,9 @@ fn resolve_ref_window(
}
/// Optional default voice for synthesis when the request doesn't name one.
/// Set `LLAMA_SWAP_TTS_VOICE=m` to read insights in a cloned voice by default.
/// Set `tts.voice = "m"` to read insights in a cloned voice by default.
fn default_voice() -> Option<String> {
std::env::var("LLAMA_SWAP_TTS_VOICE")
.ok()
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
crate::ai::config::llm_config().tts.voice.clone()
}
// Markdown / formatting strippers, compiled once. Insight text is markdown,
src/bin/import_calendar.rs
+1 -1
View File
@@ -45,7 +45,7 @@ async fn main() -> Result<()> {
// LocalLlm dispatches per LLM_BACKEND, so embeddings written here land
// in the same vector space the query side searches.
let llm = if args.generate_embeddings {
Some(LocalLlm::from_env())
Some(LocalLlm::from_config())
} else {
None
};
src/bin/import_search_history.rs
+1 -1
View File
@@ -40,7 +40,7 @@ async fn main() -> Result<()> {
// LocalLlm dispatches per LLM_BACKEND, so embeddings written here land
// in the same vector space the query side searches.
let llm = LocalLlm::from_env();
let llm = LocalLlm::from_config();
let mut inserted_count = 0usize;
let mut skipped_count = 0usize;
src/bin/populate_knowledge.rs
+6 -11
View File
@@ -129,24 +129,19 @@ async fn main() -> anyhow::Result<()> {
}
};
// Ollama config from env with CLI overrides.
let primary_url = std::env::var("OLLAMA_PRIMARY_URL")
.or_else(|_| std::env::var("OLLAMA_URL"))
.unwrap_or_else(|_| "http://localhost:11434".to_string());
let fallback_url = std::env::var("OLLAMA_FALLBACK_URL").ok();
// Ollama config from llm.toml with CLI overrides.
let ollama_cfg = &image_api::ai::config::llm_config().ollama;
let primary_url = ollama_cfg.primary_url.clone();
let primary_model = args
.model
.clone()
.or_else(|| std::env::var("OLLAMA_PRIMARY_MODEL").ok())
.or_else(|| std::env::var("OLLAMA_MODEL").ok())
.unwrap_or_else(|| "nemotron-3-nano:30b".to_string());
let fallback_model = std::env::var("OLLAMA_FALLBACK_MODEL").ok();
.unwrap_or_else(|| ollama_cfg.primary_model.clone());
let mut ollama = OllamaClient::new(
primary_url.clone(),
fallback_url,
ollama_cfg.fallback_url.clone(),
primary_model.clone(),
fallback_model,
ollama_cfg.fallback_model.clone(),
)
.with_request_timeout(args.timeout_secs);
src/bin/reembed_embeddings.rs
+1 -1
View File
@@ -284,7 +284,7 @@ async fn main() -> Result<()> {
let mut conn = SqliteConnection::establish(&database_url)
.with_context(|| format!("connecting to {}", database_url))?;
let llm = LocalLlm::from_env();
let llm = LocalLlm::from_config();
let model_version = llm.embedding_model_version();
println!("Embedding via '{}'", model_version);
if args.dry_run {
src/bin/test_daily_summary.rs
+9 -20
View File
@@ -66,6 +66,7 @@ async fn main() -> Result<()> {
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
let args = Args::parse();
let ollama_cfg = &image_api::ai::config::llm_config().ollama;
// Parse dates
let start_date = NaiveDate::parse_from_str(&args.start, "%Y-%m-%d")
@@ -82,12 +83,7 @@ async fn main() -> Result<()> {
if let Some(ref model) = args.model {
println!("Model: {}", model);
} else {
println!(
"Model: {} (from env)",
env::var("OLLAMA_PRIMARY_MODEL")
.or_else(|_| env::var("OLLAMA_MODEL"))
.unwrap_or_else(|_| "nemotron-3-nano:30b".to_string())
);
println!("Model: {} (from llm config)", ollama_cfg.primary_model);
}
if args.test_mode {
println!("⚠ TEST MODE: Results will NOT be saved to database");
@@ -96,22 +92,15 @@ async fn main() -> Result<()> {
println!();
// Initialize AI clients
let ollama_primary_url = env::var("OLLAMA_PRIMARY_URL")
.or_else(|_| env::var("OLLAMA_URL"))
.unwrap_or_else(|_| "http://localhost:11434".to_string());
let ollama_fallback_url = env::var("OLLAMA_FALLBACK_URL").ok();
// Use provided model or fallback to env
let model_to_use = args.model.clone().unwrap_or_else(|| {
env::var("OLLAMA_PRIMARY_MODEL")
.or_else(|_| env::var("OLLAMA_MODEL"))
.unwrap_or_else(|_| "nemotron-3-nano:30b".to_string())
});
// Use provided model or fall back to the configured one
let model_to_use = args
.model
.clone()
.unwrap_or_else(|| ollama_cfg.primary_model.clone());
let mut ollama = OllamaClient::new(
ollama_primary_url,
ollama_fallback_url.clone(),
ollama_cfg.primary_url.clone(),
ollama_cfg.fallback_url.clone(),
model_to_use.clone(),
Some(model_to_use), // Use same model for fallback
);
src/state.rs
+38 -87
View File
@@ -1,5 +1,6 @@
use crate::ai::apollo_client::ApolloClient;
use crate::ai::clip_client::ClipClient;
use crate::ai::config::{LlmConfig, OllamaConfig, OpenRouterConfig, llm_config};
use crate::ai::face_client::FaceClient;
use crate::ai::insight_chat::{ChatLockMap, InsightChatService};
use crate::ai::llamacpp::LlamaCppClient;
@@ -224,13 +225,14 @@ impl AppState {
impl Default for AppState {
fn default() -> Self {
// Initialize AI clients
let ollama = build_ollama_from_env();
let llm = llm_config();
let ollama = build_ollama(&llm.ollama);
let openrouter = build_openrouter_from_env();
let openrouter_allowed_models = parse_openrouter_allowed_models();
let openrouter = build_openrouter(&llm.openrouter);
let openrouter_allowed_models = llm.openrouter.allowed.clone();
let llamacpp = build_llamacpp_from_env();
let llamacpp_allowed_models = parse_llamacpp_allowed_models();
let llamacpp = build_llamacpp(llm);
let llamacpp_allowed_models = llm.llamacpp.picker.include.clone().unwrap_or_default();
let sms_api_url =
env::var("SMS_API_URL").unwrap_or_else(|_| "http://localhost:8000".to_string());
@@ -346,10 +348,7 @@ impl Default for AppState {
// Turn registry for reconnectable chat turns. 5-minute timeout for
// stale turns (background cleaner drops entries older than this).
let timeout_secs: u64 = env::var("INSIGHT_CHAT_TURN_TIMEOUT_SECS")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(300);
let timeout_secs = llm_config().agentic.chat_turn_timeout_secs;
let turn_registry = Arc::new(TurnRegistry::new(timeout_secs));
// Ensure preview clips directory exists
@@ -387,94 +386,46 @@ impl Default for AppState {
}
}
/// Build an `OpenRouterClient` from environment variables. Returns `None`
/// when `OPENROUTER_API_KEY` is unset (the hybrid backend is then
/// unavailable and requests for it return a clear error).
fn build_openrouter_from_env() -> Option<Arc<OpenRouterClient>> {
/// Build an `OpenRouterClient` from `[openrouter]`. Returns `None` when
/// `OPENROUTER_API_KEY` (env-only, it's a secret) is unset — the hybrid
/// backend is then unavailable and requests for it return a clear error.
fn build_openrouter(cfg: &OpenRouterConfig) -> Option<Arc<OpenRouterClient>> {
let api_key = env::var("OPENROUTER_API_KEY").ok()?;
let base_url = env::var("OPENROUTER_BASE_URL").ok();
let default_model = env::var("OPENROUTER_DEFAULT_MODEL")
.unwrap_or_else(|_| "anthropic/claude-sonnet-4".to_string());
let mut client = OpenRouterClient::new(api_key, base_url, default_model);
client.set_attribution(
env::var("OPENROUTER_HTTP_REFERER").ok(),
env::var("OPENROUTER_APP_TITLE").ok(),
);
if let Ok(model) = env::var("OPENROUTER_EMBEDDING_MODEL") {
client.set_embedding_model(model);
}
// OPENROUTER_MAX_TOKENS=0 omits the cap (provider default).
if let Some(n) = env::var("OPENROUTER_MAX_TOKENS")
.ok()
.and_then(|v| v.trim().parse::<u32>().ok())
{
client.set_max_tokens((n > 0).then_some(n));
let mut client =
OpenRouterClient::new(api_key, cfg.base_url.clone(), cfg.default_model.clone());
client.set_attribution(cfg.http_referer.clone(), cfg.app_title.clone());
if let Some(model) = &cfg.embedding_model {
client.set_embedding_model(model.clone());
}
client.set_max_tokens(cfg.max_tokens_cap());
Some(Arc::new(client))
}
/// Parse `OPENROUTER_ALLOWED_MODELS` (comma-separated) into a vec. Returns
/// empty when unset, in which case `/insights/openrouter/models` reports no
/// curated picks and the server falls back to `OPENROUTER_DEFAULT_MODEL`.
fn parse_openrouter_allowed_models() -> Vec<String> {
env::var("OPENROUTER_ALLOWED_MODELS")
.unwrap_or_default()
.split(',')
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
.collect()
/// Build the `OllamaClient` from `[ollama]` — shared by the server
/// (`AppState::default`) and the standalone binaries.
pub fn build_ollama(cfg: &OllamaConfig) -> OllamaClient {
OllamaClient::new(
cfg.primary_url.clone(),
cfg.fallback_url.clone(),
cfg.primary_model.clone(),
cfg.fallback_model.clone(),
)
}
/// Build the `OllamaClient` from environment variables — the canonical
/// `OLLAMA_*` wiring shared by the server (`AppState::default`) and the
/// standalone binaries (which predate this helper and used to copy it).
pub fn build_ollama_from_env() -> OllamaClient {
let primary_url = env::var("OLLAMA_PRIMARY_URL").unwrap_or_else(|_| {
env::var("OLLAMA_URL").unwrap_or_else(|_| "http://localhost:11434".to_string())
});
let fallback_url = env::var("OLLAMA_FALLBACK_URL").ok();
let primary_model = env::var("OLLAMA_PRIMARY_MODEL")
.or_else(|_| env::var("OLLAMA_MODEL"))
.unwrap_or_else(|_| "nemotron-3-nano:30b".to_string());
let fallback_model = env::var("OLLAMA_FALLBACK_MODEL").ok();
OllamaClient::new(primary_url, fallback_url, primary_model, fallback_model)
}
/// Build a `LlamaCppClient` from environment variables. Returns `None` when
/// `LLAMA_SWAP_URL` is unset. The client is constructed unconditionally
/// when the URL is set (so it's available even under `LLM_BACKEND=ollama`
/// for ad-hoc tooling), but the agentic / chat paths only route through it
/// when `LLM_BACKEND=llamacpp`. Slot ids default to the names the bundled
/// `llama-swap/config.yaml` uses — `chat` / `vision` / `embed`.
pub fn build_llamacpp_from_env() -> Option<Arc<LlamaCppClient>> {
let base_url = env::var("LLAMA_SWAP_URL").ok()?;
let primary_model = env::var("LLAMA_SWAP_PRIMARY_MODEL").ok();
let mut client = LlamaCppClient::new(Some(base_url), primary_model);
if let Ok(model) = env::var("LLAMA_SWAP_EMBEDDING_MODEL") {
client.set_embedding_model(model);
}
if let Ok(model) = env::var("LLAMA_SWAP_VISION_MODEL") {
client.set_vision_model(model);
}
if let Ok(model) = env::var("LLAMA_SWAP_TTS_MODEL") {
client.set_tts_model(model);
}
/// Build a `LlamaCppClient` from `[llamacpp]` (+ the embed and TTS model
/// ids). Returns `None` when `llamacpp.url` is unset. The client is built
/// whenever the URL is set (TTS and ad-hoc tooling use it even under
/// `backend = "ollama"`), but chat / embeddings only route through it when
/// `backend = "llamacpp"`.
pub fn build_llamacpp(cfg: &LlmConfig) -> Option<Arc<LlamaCppClient>> {
let base_url = cfg.llamacpp.url.clone()?;
let mut client = LlamaCppClient::new(Some(base_url), Some(cfg.llamacpp.chat.clone()));
client.set_embedding_model(cfg.embedding.model.clone());
client.set_vision_model(cfg.llamacpp.vision_model().to_string());
client.set_tts_model(cfg.tts.model.clone());
Some(Arc::new(client))
}
/// Parse `LLAMA_SWAP_ALLOWED_MODELS` (comma-separated) into a vec. Used to
/// populate the model picker when `LLM_BACKEND=llamacpp` — `/insights/models`
/// surfaces these slots with capabilities. Empty when unset.
fn parse_llamacpp_allowed_models() -> Vec<String> {
env::var("LLAMA_SWAP_ALLOWED_MODELS")
.unwrap_or_default()
.split(',')
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
.collect()
}
#[cfg(test)]
impl AppState {
/// Creates an AppState instance for testing with temporary directories
src/unified_search.rs
+4 -4
View File
@@ -639,15 +639,15 @@ async fn translate<TagD: TagDao>(
// llama-swap); local only, no hybrid, per the feature's design.
//
// Translation-model precedence:
// 1. UNIFIED_SEARCH_MODEL env — pin a small, fast model that can stay
// 1. `unified_search_model` (llm.toml) — pin a small, fast model that can stay
// co-resident with CLIP (and the chat model) so translation never
// evicts them. This is the recommended setup on a tight VRAM budget.
// 2. the client-selected model — routes translation to whatever the user
// already has loaded (no swap) when no dedicated model is pinned.
// 3. None → resolve_backend uses the configured default local model.
let translation_model = std::env::var("UNIFIED_SEARCH_MODEL")
.ok()
.filter(|m| !m.trim().is_empty())
let translation_model = crate::ai::config::llm_config()
.unified_search_model
.clone()
.or_else(|| query.model.clone())
.filter(|m| !m.trim().is_empty());
let overrides = SamplingOverrides {