AI: add enable_thinking reasoning toggle plumbed to llama.cpp
New optional SamplingOverride forwarded to llama-server as chat_template_kwargs.enable_thinking (gates Qwen3-style reasoning blocks). None leaves the template default; other backends ignore it. Wired through the agentic-insight and chat-turn request bodies/handlers. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
7684220f52
commit
475072810e
8 files changed
+55
No files matched your search
@@ -64,6 +64,12 @@ pub struct LlamaCppClient {
|
||||
top_p: Option<f32>,
|
||||
top_k: Option<i32>,
|
||||
min_p: Option<f32>,
|
||||
/// When `Some`, forwarded to llama-server as
|
||||
/// `chat_template_kwargs: {"enable_thinking": <bool>}`. The Jinja chat
|
||||
/// template (e.g. Qwen3) reads this to gate its reasoning block. `None`
|
||||
/// omits the key entirely, leaving the template's own default. Templates
|
||||
/// that don't reference the key ignore it, so sending it is harmless.
|
||||
enable_thinking: Option<bool>,
|
||||
}
|
||||
|
||||
impl LlamaCppClient {
|
||||
@@ -89,6 +95,7 @@ impl LlamaCppClient {
|
||||
top_p: None,
|
||||
top_k: None,
|
||||
min_p: None,
|
||||
enable_thinking: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -104,6 +111,12 @@ impl LlamaCppClient {
|
||||
self.num_ctx = num_ctx;
|
||||
}
|
||||
|
||||
/// Set the reasoning toggle forwarded as `chat_template_kwargs.enable_thinking`.
|
||||
/// `None` leaves the chat template's own default in place.
|
||||
pub fn set_enable_thinking(&mut self, enable_thinking: Option<bool>) {
|
||||
self.enable_thinking = enable_thinking;
|
||||
}
|
||||
|
||||
pub fn set_sampling_params(
|
||||
&mut self,
|
||||
temperature: Option<f32>,
|
||||
@@ -458,6 +471,12 @@ impl LlamaCppClient {
|
||||
// via -c, so we silently drop the override here. The config.yaml
|
||||
// entry is the source of truth for context size.
|
||||
let _ = self.num_ctx;
|
||||
// Reasoning toggle for thinking-capable templates (Qwen3 et al.).
|
||||
// llama-server forwards chat_template_kwargs into the Jinja render
|
||||
// (requires --jinja); templates that ignore the key are unaffected.
|
||||
if let Some(think) = self.enable_thinking {
|
||||
v.push(("chat_template_kwargs", json!({ "enable_thinking": think })));
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user