From f510940819f8f7b9b88cf2f1543780c1dc88c44c Mon Sep 17 00:00:00 2001 From: Cameron Cordes Date: Tue, 29 Sep 2026 17:13:36 -0400 Subject: [PATCH] Tracing: carry span into insight panic-isolation tasks; root the watcher boot probe (#143) Found by the parity capture: both insight-generation handlers run the generation inside a nested tokio::task::spawn (panic isolation) that was not instrumented, so the background ai.insight.* trace still split off from its 202 request. The watcher's boot-time availability probe ran before the first tick span, leaving its count_for_library DB spans as roots; it now opens `watcher.initial_probe`. Co-Authored-By: Claude Opus 5.5 --- src/ai/handlers.rs | 106 ++++++++++++++++++++++++--------------------- src/watcher.rs | 1 + 2 files changed, 58 insertions(+), 49 deletions(-) diff --git a/src/ai/handlers.rs b/src/ai/handlers.rs index 7918bea..7482058 100644 --- a/src/ai/handlers.rs +++ b/src/ai/handlers.rs @@ -503,29 +503,33 @@ pub async fn generate_insight_handler( let path_for_task = path.clone(); let generator_for_task = generator.clone(); - let result = tokio::task::spawn(async move { - // Cross-model barrier: if a TTS synthesis holds the GPU, wait it - // out BEFORE the generation wall-clock starts. The per-request - // lease keeps reqwest budgets honest, but this job-level timeout - // would otherwise burn while the first chat call queues behind a - // multi-minute synthesis. Dropped immediately — holding it across - // the generation would deadlock the chat calls' own leases. - drop(crate::ai::gpu::llm_lease().await); - tokio::time::timeout( - std::time::Duration::from_secs(timeout_secs), - generator_for_task.generate_insight_for_photo_with_config( - &path_for_task, - request.model.clone(), - request.system_prompt.clone(), - request.num_ctx, - request.temperature, - request.top_p, - request.top_k, - request.min_p, - ), - ) - .await - }) + let result = tokio::task::spawn( + async move { + // Cross-model barrier: if a TTS synthesis holds the GPU, wait it + // out BEFORE the generation wall-clock starts. The per-request + // lease keeps reqwest budgets honest, but this job-level timeout + // would otherwise burn while the first chat call queues behind a + // multi-minute synthesis. Dropped immediately — holding it across + // the generation would deadlock the chat calls' own leases. + drop(crate::ai::gpu::llm_lease().await); + tokio::time::timeout( + std::time::Duration::from_secs(timeout_secs), + generator_for_task.generate_insight_for_photo_with_config( + &path_for_task, + request.model.clone(), + request.system_prompt.clone(), + request.num_ctx, + request.temperature, + request.top_p, + request.top_k, + request.min_p, + ), + ) + .await + } + // Inner panic-isolation task: carry the job span in too. + .instrument(tracing::Span::current()), + ) .await; let mut dao = job_dao.lock().expect("Unable to lock InsightJobDao"); @@ -883,32 +887,36 @@ pub async fn generate_agentic_insight_handler( let path_for_task = path.clone(); let generator_for_task = generator.clone(); - let result = tokio::task::spawn(async move { - // Cross-model barrier — see generate_insight_handler: wait out any - // running TTS synthesis before the generation wall-clock starts. - drop(crate::ai::gpu::llm_lease().await); - tokio::time::timeout( - std::time::Duration::from_secs(timeout_secs), - generator_for_task.generate_agentic_insight_for_photo( - &path_for_task, - request.model.clone(), - system_prompt, - request.num_ctx, - request.temperature, - request.top_p, - request.top_k, - request.min_p, - request.enable_thinking, - max_iterations, - request.backend.clone(), - fewshot_examples, - fewshot_ids, - user_id, - persona_id, - ), - ) - .await - }) + let result = tokio::task::spawn( + async move { + // Cross-model barrier — see generate_insight_handler: wait out any + // running TTS synthesis before the generation wall-clock starts. + drop(crate::ai::gpu::llm_lease().await); + tokio::time::timeout( + std::time::Duration::from_secs(timeout_secs), + generator_for_task.generate_agentic_insight_for_photo( + &path_for_task, + request.model.clone(), + system_prompt, + request.num_ctx, + request.temperature, + request.top_p, + request.top_k, + request.min_p, + request.enable_thinking, + max_iterations, + request.backend.clone(), + fewshot_examples, + fewshot_ids, + user_id, + persona_id, + ), + ) + .await + } + // Inner panic-isolation task: carry the job span in too. + .instrument(tracing::Span::current()), + ) .await; let mut dao = job_dao.lock().expect("Unable to lock InsightJobDao"); diff --git a/src/watcher.rs b/src/watcher.rs index f10f243..f226f2f 100644 --- a/src/watcher.rs +++ b/src/watcher.rs @@ -394,6 +394,7 @@ pub fn watch_files( // Disabled libraries skip the probe entirely — they should // never enter the health map (treated as out-of-scope). { + let _probe = tracing::info_span!("watcher.initial_probe").entered(); let libs = libs_lock.read().unwrap_or_else(|e| e.into_inner()); for lib in libs.iter() { if !lib.enabled {