Tracing: carry span into insight panic-isolation tasks; root the watcher boot probe (#143)
Found by the parity capture: both insight-generation handlers run the generation inside a nested tokio::task::spawn (panic isolation) that was not instrumented, so the background ai.insight.* trace still split off from its 202 request. The watcher's boot-time availability probe ran before the first tick span, leaving its count_for_library DB spans as roots; it now opens `watcher.initial_probe`. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
983102c1a5
commit
f510940819
2 files changed
+58
-49
No files matched your search
+57
-49
@@ -503,29 +503,33 @@ pub async fn generate_insight_handler(
|
|||||||
|
|
||||||
let path_for_task = path.clone();
|
let path_for_task = path.clone();
|
||||||
let generator_for_task = generator.clone();
|
let generator_for_task = generator.clone();
|
||||||
let result = tokio::task::spawn(async move {
|
let result = tokio::task::spawn(
|
||||||
// Cross-model barrier: if a TTS synthesis holds the GPU, wait it
|
async move {
|
||||||
// out BEFORE the generation wall-clock starts. The per-request
|
// Cross-model barrier: if a TTS synthesis holds the GPU, wait it
|
||||||
// lease keeps reqwest budgets honest, but this job-level timeout
|
// out BEFORE the generation wall-clock starts. The per-request
|
||||||
// would otherwise burn while the first chat call queues behind a
|
// lease keeps reqwest budgets honest, but this job-level timeout
|
||||||
// multi-minute synthesis. Dropped immediately — holding it across
|
// would otherwise burn while the first chat call queues behind a
|
||||||
// the generation would deadlock the chat calls' own leases.
|
// multi-minute synthesis. Dropped immediately — holding it across
|
||||||
drop(crate::ai::gpu::llm_lease().await);
|
// the generation would deadlock the chat calls' own leases.
|
||||||
tokio::time::timeout(
|
drop(crate::ai::gpu::llm_lease().await);
|
||||||
std::time::Duration::from_secs(timeout_secs),
|
tokio::time::timeout(
|
||||||
generator_for_task.generate_insight_for_photo_with_config(
|
std::time::Duration::from_secs(timeout_secs),
|
||||||
&path_for_task,
|
generator_for_task.generate_insight_for_photo_with_config(
|
||||||
request.model.clone(),
|
&path_for_task,
|
||||||
request.system_prompt.clone(),
|
request.model.clone(),
|
||||||
request.num_ctx,
|
request.system_prompt.clone(),
|
||||||
request.temperature,
|
request.num_ctx,
|
||||||
request.top_p,
|
request.temperature,
|
||||||
request.top_k,
|
request.top_p,
|
||||||
request.min_p,
|
request.top_k,
|
||||||
),
|
request.min_p,
|
||||||
)
|
),
|
||||||
.await
|
)
|
||||||
})
|
.await
|
||||||
|
}
|
||||||
|
// Inner panic-isolation task: carry the job span in too.
|
||||||
|
.instrument(tracing::Span::current()),
|
||||||
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
let mut dao = job_dao.lock().expect("Unable to lock InsightJobDao");
|
let mut dao = job_dao.lock().expect("Unable to lock InsightJobDao");
|
||||||
@@ -883,32 +887,36 @@ pub async fn generate_agentic_insight_handler(
|
|||||||
|
|
||||||
let path_for_task = path.clone();
|
let path_for_task = path.clone();
|
||||||
let generator_for_task = generator.clone();
|
let generator_for_task = generator.clone();
|
||||||
let result = tokio::task::spawn(async move {
|
let result = tokio::task::spawn(
|
||||||
// Cross-model barrier — see generate_insight_handler: wait out any
|
async move {
|
||||||
// running TTS synthesis before the generation wall-clock starts.
|
// Cross-model barrier — see generate_insight_handler: wait out any
|
||||||
drop(crate::ai::gpu::llm_lease().await);
|
// running TTS synthesis before the generation wall-clock starts.
|
||||||
tokio::time::timeout(
|
drop(crate::ai::gpu::llm_lease().await);
|
||||||
std::time::Duration::from_secs(timeout_secs),
|
tokio::time::timeout(
|
||||||
generator_for_task.generate_agentic_insight_for_photo(
|
std::time::Duration::from_secs(timeout_secs),
|
||||||
&path_for_task,
|
generator_for_task.generate_agentic_insight_for_photo(
|
||||||
request.model.clone(),
|
&path_for_task,
|
||||||
system_prompt,
|
request.model.clone(),
|
||||||
request.num_ctx,
|
system_prompt,
|
||||||
request.temperature,
|
request.num_ctx,
|
||||||
request.top_p,
|
request.temperature,
|
||||||
request.top_k,
|
request.top_p,
|
||||||
request.min_p,
|
request.top_k,
|
||||||
request.enable_thinking,
|
request.min_p,
|
||||||
max_iterations,
|
request.enable_thinking,
|
||||||
request.backend.clone(),
|
max_iterations,
|
||||||
fewshot_examples,
|
request.backend.clone(),
|
||||||
fewshot_ids,
|
fewshot_examples,
|
||||||
user_id,
|
fewshot_ids,
|
||||||
persona_id,
|
user_id,
|
||||||
),
|
persona_id,
|
||||||
)
|
),
|
||||||
.await
|
)
|
||||||
})
|
.await
|
||||||
|
}
|
||||||
|
// Inner panic-isolation task: carry the job span in too.
|
||||||
|
.instrument(tracing::Span::current()),
|
||||||
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
let mut dao = job_dao.lock().expect("Unable to lock InsightJobDao");
|
let mut dao = job_dao.lock().expect("Unable to lock InsightJobDao");
|
||||||
|
|||||||
@@ -394,6 +394,7 @@ pub fn watch_files(
|
|||||||
// Disabled libraries skip the probe entirely — they should
|
// Disabled libraries skip the probe entirely — they should
|
||||||
// never enter the health map (treated as out-of-scope).
|
// never enter the health map (treated as out-of-scope).
|
||||||
{
|
{
|
||||||
|
let _probe = tracing::info_span!("watcher.initial_probe").entered();
|
||||||
let libs = libs_lock.read().unwrap_or_else(|e| e.into_inner());
|
let libs = libs_lock.read().unwrap_or_else(|e| e.into_inner());
|
||||||
for lib in libs.iter() {
|
for lib in libs.iter() {
|
||||||
if !lib.enabled {
|
if !lib.enabled {
|
||||||
|
|||||||
Reference in new issue
Block a user