Tracing: carry span into insight panic-isolation tasks; root the watcher boot probe (#143)

Found by the parity capture: both insight-generation handlers run the
generation inside a nested tokio::task::spawn (panic isolation) that was
not instrumented, so the background ai.insight.* trace still split off
from its 202 request. The watcher's boot-time availability probe ran
before the first tick span, leaving its count_for_library DB spans as
roots; it now opens `watcher.initial_probe`.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Cameron CordesandClaude Opus 5.5 committed 2026-09-29 17:13:36 -04:00
1 parent 983102c1a5
commit f510940819
2 files changed
+58 -49

No files matched your search

+57 -49
View File
@@ -503,29 +503,33 @@ pub async fn generate_insight_handler(
let path_for_task = path.clone(); let path_for_task = path.clone();
let generator_for_task = generator.clone(); let generator_for_task = generator.clone();
let result = tokio::task::spawn(async move { let result = tokio::task::spawn(
// Cross-model barrier: if a TTS synthesis holds the GPU, wait it async move {
// out BEFORE the generation wall-clock starts. The per-request // Cross-model barrier: if a TTS synthesis holds the GPU, wait it
// lease keeps reqwest budgets honest, but this job-level timeout // out BEFORE the generation wall-clock starts. The per-request
// would otherwise burn while the first chat call queues behind a // lease keeps reqwest budgets honest, but this job-level timeout
// multi-minute synthesis. Dropped immediately — holding it across // would otherwise burn while the first chat call queues behind a
// the generation would deadlock the chat calls' own leases. // multi-minute synthesis. Dropped immediately — holding it across
drop(crate::ai::gpu::llm_lease().await); // the generation would deadlock the chat calls' own leases.
tokio::time::timeout( drop(crate::ai::gpu::llm_lease().await);
std::time::Duration::from_secs(timeout_secs), tokio::time::timeout(
generator_for_task.generate_insight_for_photo_with_config( std::time::Duration::from_secs(timeout_secs),
&path_for_task, generator_for_task.generate_insight_for_photo_with_config(
request.model.clone(), &path_for_task,
request.system_prompt.clone(), request.model.clone(),
request.num_ctx, request.system_prompt.clone(),
request.temperature, request.num_ctx,
request.top_p, request.temperature,
request.top_k, request.top_p,
request.min_p, request.top_k,
), request.min_p,
) ),
.await )
}) .await
}
// Inner panic-isolation task: carry the job span in too.
.instrument(tracing::Span::current()),
)
.await; .await;
let mut dao = job_dao.lock().expect("Unable to lock InsightJobDao"); let mut dao = job_dao.lock().expect("Unable to lock InsightJobDao");
@@ -883,32 +887,36 @@ pub async fn generate_agentic_insight_handler(
let path_for_task = path.clone(); let path_for_task = path.clone();
let generator_for_task = generator.clone(); let generator_for_task = generator.clone();
let result = tokio::task::spawn(async move { let result = tokio::task::spawn(
// Cross-model barrier — see generate_insight_handler: wait out any async move {
// running TTS synthesis before the generation wall-clock starts. // Cross-model barrier — see generate_insight_handler: wait out any
drop(crate::ai::gpu::llm_lease().await); // running TTS synthesis before the generation wall-clock starts.
tokio::time::timeout( drop(crate::ai::gpu::llm_lease().await);
std::time::Duration::from_secs(timeout_secs), tokio::time::timeout(
generator_for_task.generate_agentic_insight_for_photo( std::time::Duration::from_secs(timeout_secs),
&path_for_task, generator_for_task.generate_agentic_insight_for_photo(
request.model.clone(), &path_for_task,
system_prompt, request.model.clone(),
request.num_ctx, system_prompt,
request.temperature, request.num_ctx,
request.top_p, request.temperature,
request.top_k, request.top_p,
request.min_p, request.top_k,
request.enable_thinking, request.min_p,
max_iterations, request.enable_thinking,
request.backend.clone(), max_iterations,
fewshot_examples, request.backend.clone(),
fewshot_ids, fewshot_examples,
user_id, fewshot_ids,
persona_id, user_id,
), persona_id,
) ),
.await )
}) .await
}
// Inner panic-isolation task: carry the job span in too.
.instrument(tracing::Span::current()),
)
.await; .await;
let mut dao = job_dao.lock().expect("Unable to lock InsightJobDao"); let mut dao = job_dao.lock().expect("Unable to lock InsightJobDao");
+1
View File
@@ -394,6 +394,7 @@ pub fn watch_files(
// Disabled libraries skip the probe entirely — they should // Disabled libraries skip the probe entirely — they should
// never enter the health map (treated as out-of-scope). // never enter the health map (treated as out-of-scope).
{ {
let _probe = tracing::info_span!("watcher.initial_probe").entered();
let libs = libs_lock.read().unwrap_or_else(|e| e.into_inner()); let libs = libs_lock.read().unwrap_or_else(|e| e.into_inner());
for lib in libs.iter() { for lib in libs.iter() {
if !lib.enabled { if !lib.enabled {