diff --git a/.env.example b/.env.example index 64c31d3..d9704d9 100644 --- a/.env.example +++ b/.env.example @@ -52,6 +52,10 @@ AGENTIC_CHAT_MAX_ITERATIONS=6 # OPENROUTER_EMBEDDING_MODEL=openai/text-embedding-3-small # OPENROUTER_HTTP_REFERER=https://your-site.example # OPENROUTER_APP_TITLE=ImageApi +# Cap on generated tokens per call (sent as max_tokens). OpenRouter reserves +# credit against the requested max, so leaving it uncapped makes pricey +# models fail with credit-limit errors. Default 16384; 0 = no cap. +# OPENROUTER_MAX_TOKENS=16384 # ── AI Insights — local backend switch ────────────────────────────────── # Picks which local LLM stack the server uses for chat, vision describe, diff --git a/CLAUDE.md b/CLAUDE.md index 4faec1c..bd06509 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -636,6 +636,9 @@ OPENROUTER_BASE_URL=https://openrouter.ai/api/v1 # Override base URL (option OPENROUTER_EMBEDDING_MODEL=openai/text-embedding-3-small # Optional, embeddings stay local today OPENROUTER_HTTP_REFERER=https://your-site.example # Optional attribution header OPENROUTER_APP_TITLE=ImageApi # Optional attribution header +OPENROUTER_MAX_TOKENS=16384 # Per-call max_tokens cap (default 16384; 0 = uncapped). + # OpenRouter reserves credit against this, so an uncapped + # request to a pricey model can fail on credit limits. # Local LLM backend switch. `ollama` (default) keeps the OLLAMA_* settings # above; `llamacpp` swaps the entire local stack (chat + vision describe + diff --git a/src/ai/openrouter.rs b/src/ai/openrouter.rs index 62a3cca..5ec420f 100644 --- a/src/ai/openrouter.rs +++ b/src/ai/openrouter.rs @@ -19,6 +19,11 @@ use futures::stream::{BoxStream, StreamExt}; const DEFAULT_BASE_URL: &str = "https://openrouter.ai/api/v1"; const DEFAULT_EMBEDDING_MODEL: &str = "openai/text-embedding-3-small"; const CACHE_DURATION_SECS: u64 = 15 * 60; +/// Default completion cap sent as `max_tokens`. OpenRouter pre-authorizes +/// credit against the requested max (defaulting to the model's full output +/// window when omitted), so expensive models fail with credit-limit errors +/// even when the actual reply is short. Override via `OPENROUTER_MAX_TOKENS`. +pub const DEFAULT_MAX_TOKENS: u32 = 16384; #[derive(Clone)] struct CachedEntry { @@ -64,6 +69,9 @@ pub struct OpenRouterClient { top_p: Option, top_k: Option, min_p: Option, + /// Upper bound on generated tokens per call, sent as `max_tokens`. + /// `None` omits the field (provider default). + max_tokens: Option, /// Optional `HTTP-Referer` header OpenRouter uses for attribution. pub referer: Option, /// Optional `X-Title` header OpenRouter uses for attribution. @@ -87,6 +95,7 @@ impl OpenRouterClient { top_p: None, top_k: None, min_p: None, + max_tokens: Some(DEFAULT_MAX_TOKENS), referer: None, app_title: None, } @@ -115,6 +124,10 @@ impl OpenRouterClient { self.min_p = min_p; } + pub fn set_max_tokens(&mut self, max_tokens: Option) { + self.max_tokens = max_tokens; + } + pub fn set_attribution(&mut self, referer: Option, app_title: Option) { self.referer = referer; self.app_title = app_title; @@ -288,6 +301,9 @@ impl OpenRouterClient { // directly transferable. Skip rather than silently mis-map. let _ = c; } + if let Some(m) = self.max_tokens { + v.push(("max_tokens", json!(m))); + } v } } diff --git a/src/state.rs b/src/state.rs index 3f456e7..0d09db2 100644 --- a/src/state.rs +++ b/src/state.rs @@ -402,6 +402,13 @@ fn build_openrouter_from_env() -> Option> { if let Ok(model) = env::var("OPENROUTER_EMBEDDING_MODEL") { client.set_embedding_model(model); } + // OPENROUTER_MAX_TOKENS=0 omits the cap (provider default). + if let Some(n) = env::var("OPENROUTER_MAX_TOKENS") + .ok() + .and_then(|v| v.trim().parse::().ok()) + { + client.set_max_tokens((n > 0).then_some(n)); + } Some(Arc::new(client)) }