Add GPU lease coordinating LLM and TTS requests through llama-swap
llama-swap runs chat/vision/Chatterbox as a mutually-exclusive set on one GPU and HOLDS a request for a non-resident model until the resident model drains, then swaps. That hold burned the holder's reqwest timeout (measured: a queued TTS lost 77s behind one LLM turn; an LLM request behind a synthesis waited the entire remaining synth), so concurrent insight + read-aloud timed out instead of queueing. ai::gpu adds a fair RwLock lease acquired before each request is sent, so cross-model waits happen before the HTTP timeout starts: chat/vision share the read lease, TTS synthesis and voice-library ops (which spin Chatterbox up) take the write lease, and embeddings take none (the embed slot is in llama-swap's always-resident group). Speech jobs now flip queued->running only after acquiring the GPU, letting the client anchor its poll deadline to that transition. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
03699f7413
commit
0accc4ef2f
4 files changed
+125
-1
No files matched your search
@@ -142,6 +142,11 @@ impl LlamaCppClient {
|
||||
/// Chatterbox generation knobs are forwarded when set (caller is expected
|
||||
/// to have range-clamped them): `exaggeration` (0.25–2.0, emotion),
|
||||
/// `cfg_weight` (0.0–1.0, pace), `temperature` (0.05–5.0, randomness).
|
||||
///
|
||||
/// Callers must hold the GPU write lease (`ai::gpu::tts_lease`) across
|
||||
/// this call. It is taken at the call sites in `ai::tts` rather than here
|
||||
/// so the speech-job path can flip its job to `running` between acquiring
|
||||
/// the GPU and sending the request.
|
||||
pub async fn text_to_speech(
|
||||
&self,
|
||||
input: &str,
|
||||
@@ -204,6 +209,9 @@ impl LlamaCppClient {
|
||||
/// List voices in the Chatterbox voice library (raw JSON passthrough).
|
||||
pub async fn list_voices(&self) -> Result<Value> {
|
||||
let url = format!("{}/upstream/{}/voices", self.swap_root(), self.tts_model);
|
||||
// The /upstream passthrough spins Chatterbox up (evicting the LLM),
|
||||
// so it takes the exclusive GPU lease like synthesis does.
|
||||
let _gpu = crate::ai::gpu::tts_lease().await;
|
||||
let resp = self
|
||||
.client
|
||||
.get(&url)
|
||||
@@ -237,6 +245,9 @@ impl LlamaCppClient {
|
||||
.text("voice_name", voice_name.to_string())
|
||||
.part("voice_file", part);
|
||||
|
||||
// The /upstream passthrough spins Chatterbox up (evicting the LLM),
|
||||
// so it takes the exclusive GPU lease like synthesis does.
|
||||
let _gpu = crate::ai::gpu::tts_lease().await;
|
||||
let resp = self
|
||||
.client
|
||||
.post(&url)
|
||||
@@ -262,6 +273,9 @@ impl LlamaCppClient {
|
||||
self.tts_model,
|
||||
voice_name
|
||||
);
|
||||
// The /upstream passthrough spins Chatterbox up (evicting the LLM),
|
||||
// so it takes the exclusive GPU lease like synthesis does.
|
||||
let _gpu = crate::ai::gpu::tts_lease().await;
|
||||
let resp = self
|
||||
.client
|
||||
.delete(&url)
|
||||
@@ -481,6 +495,9 @@ impl LlamaCppClient {
|
||||
body.insert(k.into(), v);
|
||||
}
|
||||
|
||||
// Wait for any TTS synthesis to release the GPU before the request
|
||||
// timeout starts (see ai::gpu).
|
||||
let _gpu = crate::ai::gpu::llm_lease().await;
|
||||
let resp = self
|
||||
.client
|
||||
.post(&url)
|
||||
@@ -599,6 +616,10 @@ impl LlmClient for LlamaCppClient {
|
||||
body.insert(k.into(), v);
|
||||
}
|
||||
|
||||
// Wait for any TTS synthesis to release the GPU before the request
|
||||
// timeout starts (see ai::gpu). The guard is moved into the stream
|
||||
// below so the lease spans the whole generation, not just the send.
|
||||
let gpu = crate::ai::gpu::llm_lease().await;
|
||||
let resp = self
|
||||
.client
|
||||
.post(&url)
|
||||
@@ -615,6 +636,7 @@ impl LlmClient for LlamaCppClient {
|
||||
|
||||
let byte_stream = resp.bytes_stream();
|
||||
let stream = async_stream::stream! {
|
||||
let _gpu = gpu;
|
||||
let mut byte_stream = byte_stream;
|
||||
let mut buf: Vec<u8> = Vec::new();
|
||||
let mut accumulated_content = String::new();
|
||||
@@ -730,6 +752,9 @@ impl LlmClient for LlamaCppClient {
|
||||
}
|
||||
|
||||
async fn generate_embeddings(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
|
||||
// Deliberately NO GPU lease: the embed slot sits in llama-swap's
|
||||
// always-resident group and never participates in a model swap, so
|
||||
// leasing here would only stall searches behind a queued synthesis.
|
||||
let url = format!("{}/embeddings", self.base_url);
|
||||
let body = json!({
|
||||
"model": self.embedding_model,
|
||||
|
||||
Reference in new issue
Block a user