feat(openrouter): cap max_tokens per request (OPENROUTER_MAX_TOKENS)

OpenRouter reserves credit against the requested max_tokens, falling
back to the model's full output window when omitted, so expensive
models failed with credit-limit errors even for short replies. Default
the cap to 16384; OPENROUTER_MAX_TOKENS overrides it and 0 disables it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Cameron CordesandClaude Opus 5.5 committed 2026-09-22 21:32:56 -04:00
1 parent 4e3ee1973c
commit 33fe7ce77c
4 files changed
+30

No files matched your search

+16
View File
@@ -19,6 +19,11 @@ use futures::stream::{BoxStream, StreamExt};
const DEFAULT_BASE_URL: &str = "https://openrouter.ai/api/v1";
const DEFAULT_EMBEDDING_MODEL: &str = "openai/text-embedding-3-small";
const CACHE_DURATION_SECS: u64 = 15 * 60;
/// Default completion cap sent as `max_tokens`. OpenRouter pre-authorizes
/// credit against the requested max (defaulting to the model's full output
/// window when omitted), so expensive models fail with credit-limit errors
/// even when the actual reply is short. Override via `OPENROUTER_MAX_TOKENS`.
pub const DEFAULT_MAX_TOKENS: u32 = 16384;
#[derive(Clone)]
struct CachedEntry<T> {
@@ -64,6 +69,9 @@ pub struct OpenRouterClient {
top_p: Option<f32>,
top_k: Option<i32>,
min_p: Option<f32>,
/// Upper bound on generated tokens per call, sent as `max_tokens`.
/// `None` omits the field (provider default).
max_tokens: Option<u32>,
/// Optional `HTTP-Referer` header OpenRouter uses for attribution.
pub referer: Option<String>,
/// Optional `X-Title` header OpenRouter uses for attribution.
@@ -87,6 +95,7 @@ impl OpenRouterClient {
top_p: None,
top_k: None,
min_p: None,
max_tokens: Some(DEFAULT_MAX_TOKENS),
referer: None,
app_title: None,
}
@@ -115,6 +124,10 @@ impl OpenRouterClient {
self.min_p = min_p;
}
pub fn set_max_tokens(&mut self, max_tokens: Option<u32>) {
self.max_tokens = max_tokens;
}
pub fn set_attribution(&mut self, referer: Option<String>, app_title: Option<String>) {
self.referer = referer;
self.app_title = app_title;
@@ -288,6 +301,9 @@ impl OpenRouterClient {
// directly transferable. Skip rather than silently mis-map.
let _ = c;
}
if let Some(m) = self.max_tokens {
v.push(("max_tokens", json!(m)));
}
v
}
}