Tracing: carry span into insight panic-isolation tasks; root the watcher boot probe (#143)
Found by the parity capture: both insight-generation handlers run the generation inside a nested tokio::task::spawn (panic isolation) that was not instrumented, so the background ai.insight.* trace still split off from its 202 request. The watcher's boot-time availability probe ran before the first tick span, leaving its count_for_library DB spans as roots; it now opens `watcher.initial_probe`. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
983102c1a5
commit
f510940819
2 files changed
+58
-49
No files matched your search
+57
-49
@@ -503,29 +503,33 @@ pub async fn generate_insight_handler(
|
||||
|
||||
let path_for_task = path.clone();
|
||||
let generator_for_task = generator.clone();
|
||||
let result = tokio::task::spawn(async move {
|
||||
// Cross-model barrier: if a TTS synthesis holds the GPU, wait it
|
||||
// out BEFORE the generation wall-clock starts. The per-request
|
||||
// lease keeps reqwest budgets honest, but this job-level timeout
|
||||
// would otherwise burn while the first chat call queues behind a
|
||||
// multi-minute synthesis. Dropped immediately — holding it across
|
||||
// the generation would deadlock the chat calls' own leases.
|
||||
drop(crate::ai::gpu::llm_lease().await);
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(timeout_secs),
|
||||
generator_for_task.generate_insight_for_photo_with_config(
|
||||
&path_for_task,
|
||||
request.model.clone(),
|
||||
request.system_prompt.clone(),
|
||||
request.num_ctx,
|
||||
request.temperature,
|
||||
request.top_p,
|
||||
request.top_k,
|
||||
request.min_p,
|
||||
),
|
||||
)
|
||||
.await
|
||||
})
|
||||
let result = tokio::task::spawn(
|
||||
async move {
|
||||
// Cross-model barrier: if a TTS synthesis holds the GPU, wait it
|
||||
// out BEFORE the generation wall-clock starts. The per-request
|
||||
// lease keeps reqwest budgets honest, but this job-level timeout
|
||||
// would otherwise burn while the first chat call queues behind a
|
||||
// multi-minute synthesis. Dropped immediately — holding it across
|
||||
// the generation would deadlock the chat calls' own leases.
|
||||
drop(crate::ai::gpu::llm_lease().await);
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(timeout_secs),
|
||||
generator_for_task.generate_insight_for_photo_with_config(
|
||||
&path_for_task,
|
||||
request.model.clone(),
|
||||
request.system_prompt.clone(),
|
||||
request.num_ctx,
|
||||
request.temperature,
|
||||
request.top_p,
|
||||
request.top_k,
|
||||
request.min_p,
|
||||
),
|
||||
)
|
||||
.await
|
||||
}
|
||||
// Inner panic-isolation task: carry the job span in too.
|
||||
.instrument(tracing::Span::current()),
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut dao = job_dao.lock().expect("Unable to lock InsightJobDao");
|
||||
@@ -883,32 +887,36 @@ pub async fn generate_agentic_insight_handler(
|
||||
|
||||
let path_for_task = path.clone();
|
||||
let generator_for_task = generator.clone();
|
||||
let result = tokio::task::spawn(async move {
|
||||
// Cross-model barrier — see generate_insight_handler: wait out any
|
||||
// running TTS synthesis before the generation wall-clock starts.
|
||||
drop(crate::ai::gpu::llm_lease().await);
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(timeout_secs),
|
||||
generator_for_task.generate_agentic_insight_for_photo(
|
||||
&path_for_task,
|
||||
request.model.clone(),
|
||||
system_prompt,
|
||||
request.num_ctx,
|
||||
request.temperature,
|
||||
request.top_p,
|
||||
request.top_k,
|
||||
request.min_p,
|
||||
request.enable_thinking,
|
||||
max_iterations,
|
||||
request.backend.clone(),
|
||||
fewshot_examples,
|
||||
fewshot_ids,
|
||||
user_id,
|
||||
persona_id,
|
||||
),
|
||||
)
|
||||
.await
|
||||
})
|
||||
let result = tokio::task::spawn(
|
||||
async move {
|
||||
// Cross-model barrier — see generate_insight_handler: wait out any
|
||||
// running TTS synthesis before the generation wall-clock starts.
|
||||
drop(crate::ai::gpu::llm_lease().await);
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(timeout_secs),
|
||||
generator_for_task.generate_agentic_insight_for_photo(
|
||||
&path_for_task,
|
||||
request.model.clone(),
|
||||
system_prompt,
|
||||
request.num_ctx,
|
||||
request.temperature,
|
||||
request.top_p,
|
||||
request.top_k,
|
||||
request.min_p,
|
||||
request.enable_thinking,
|
||||
max_iterations,
|
||||
request.backend.clone(),
|
||||
fewshot_examples,
|
||||
fewshot_ids,
|
||||
user_id,
|
||||
persona_id,
|
||||
),
|
||||
)
|
||||
.await
|
||||
}
|
||||
// Inner panic-isolation task: carry the job span in too.
|
||||
.instrument(tracing::Span::current()),
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut dao = job_dao.lock().expect("Unable to lock InsightJobDao");
|
||||
|
||||
@@ -394,6 +394,7 @@ pub fn watch_files(
|
||||
// Disabled libraries skip the probe entirely — they should
|
||||
// never enter the health map (treated as out-of-scope).
|
||||
{
|
||||
let _probe = tracing::info_span!("watcher.initial_probe").entered();
|
||||
let libs = libs_lock.read().unwrap_or_else(|e| e.into_inner());
|
||||
for lib in libs.iter() {
|
||||
if !lib.enabled {
|
||||
|
||||
Reference in new issue
Block a user