AI: add enable_thinking reasoning toggle plumbed to llama.cpp

New optional SamplingOverride forwarded to llama-server as
chat_template_kwargs.enable_thinking (gates Qwen3-style reasoning
blocks). None leaves the template default; other backends ignore it.
Wired through the agentic-insight and chat-turn request bodies/handlers.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Cameron CordesandClaude Opus 4.8 authored and Cameron committed 2026-06-29 12:39:16 -04:00
1 parent 7684220f52
commit 475072810e
8 files changed
+55

No files matched your search

+19
View File
@@ -64,6 +64,12 @@ pub struct LlamaCppClient {
top_p: Option<f32>,
top_k: Option<i32>,
min_p: Option<f32>,
/// When `Some`, forwarded to llama-server as
/// `chat_template_kwargs: {"enable_thinking": <bool>}`. The Jinja chat
/// template (e.g. Qwen3) reads this to gate its reasoning block. `None`
/// omits the key entirely, leaving the template's own default. Templates
/// that don't reference the key ignore it, so sending it is harmless.
enable_thinking: Option<bool>,
}
impl LlamaCppClient {
@@ -89,6 +95,7 @@ impl LlamaCppClient {
top_p: None,
top_k: None,
min_p: None,
enable_thinking: None,
}
}
@@ -104,6 +111,12 @@ impl LlamaCppClient {
self.num_ctx = num_ctx;
}
/// Set the reasoning toggle forwarded as `chat_template_kwargs.enable_thinking`.
/// `None` leaves the chat template's own default in place.
pub fn set_enable_thinking(&mut self, enable_thinking: Option<bool>) {
self.enable_thinking = enable_thinking;
}
pub fn set_sampling_params(
&mut self,
temperature: Option<f32>,
@@ -458,6 +471,12 @@ impl LlamaCppClient {
// via -c, so we silently drop the override here. The config.yaml
// entry is the source of truth for context size.
let _ = self.num_ctx;
// Reasoning toggle for thinking-capable templates (Qwen3 et al.).
// llama-server forwards chat_template_kwargs into the Jinja render
// (requires --jinja); templates that ignore the key are unaffected.
if let Some(think) = self.enable_thinking {
v.push(("chat_template_kwargs", json!({ "enable_thinking": think })));
}
v
}