Tracing: root spans for background jobs, retire raw OTel spans (#142)

Background work has no request parent, so each job now opens a named
tracing root and carries it into its fan-out:

- watcher: `watcher.tick` (scan=quick|full) entered per tick on the
  watcher's std thread, `watcher.library` per library, and
  `watcher.playlist_cleanup` per orphaned-HLS pass. face_watch /
  clip_watch carry the ambient span into every per-candidate
  tokio::spawn and give each call its own `face_watch.detect` /
  `clip_watch.encode` span.
- thumbnails: `creating thumbnails` via #[instrument]; per-video spans on
  rayon workers name their parent explicitly.
- video actors: `playlistgenerator.generate_playlist` and
  `preview_clip.generate` are tracing roots that instrument their futures.
- reels: `reel.job` (created before the spawn, so on-demand jobs stay in
  the request trace), `reel.pregen_batch`, `reel.cache_sweep`.
- RefreshThumbnailsMessage and the dead-code GIF job move to tracing.

No raw OpenTelemetry span creation is left outside otel.rs;
`global_tracer` is removed.

tests/otel_background.rs pins the std-thread tick -> block_on -> spawn
shape and the rayon explicit-parent shape.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Cameron CordesandClaude Opus 5.5 committed 2026-09-29 17:02:40 -04:00
1 parent b6deca2a52
commit 26115b3378
11 files changed
+330 -179

No files matched your search

+41 -28
View File
@@ -28,6 +28,7 @@ use anyhow::{Context, anyhow};
use chrono::{DateTime, Datelike, Timelike};
use serde::{Deserialize, Serialize};
use serde_json::json;
use tracing::Instrument;
use uuid::Uuid;
use crate::data::Claims;
@@ -557,34 +558,41 @@ pub async fn create_reel_handler(
let state = app_state.clone();
let insight_dao = insight_dao.clone();
let exif_dao = exif_dao.clone();
let handle = tokio::spawn(async move {
match run_reel_job(
&state,
&insight_dao,
&exif_dao,
job_id,
planned,
meta,
voice,
&key,
)
.await
{
Ok((title, path)) => {
finish_job(job_id, ReelJobStatus::Done, Some(title), Some(path), None)
}
Err(e) => {
log::error!("reel job {job_id} failed: {e:?}");
finish_job(
job_id,
ReelJobStatus::Error,
None,
None,
Some(format!("{e}")),
)
// Created before the spawn so the job parents on the dispatching request
// (or runs as its own root from the pre-gen scheduler), then carried into
// the task — tracing context doesn't cross `tokio::spawn` on its own.
let job_span = tracing::info_span!("reel.job", job_id = %job_id);
let handle = tokio::spawn(
async move {
match run_reel_job(
&state,
&insight_dao,
&exif_dao,
job_id,
planned,
meta,
voice,
&key,
)
.await
{
Ok((title, path)) => {
finish_job(job_id, ReelJobStatus::Done, Some(title), Some(path), None)
}
Err(e) => {
log::error!("reel job {job_id} failed: {e:?}");
finish_job(
job_id,
ReelJobStatus::Error,
None,
None,
Some(format!("{e}")),
)
}
}
}
});
.instrument(job_span),
);
with_job(job_id, |job| job.abort = Some(handle.abort_handle()));
// Capture params for passive prefs mirror (best-effort, never fails).
@@ -1056,7 +1064,11 @@ pub(crate) async fn spawn_pregen_scheduler(app_state: web::Data<AppState>) {
log::debug!("Next pre-gen run in {}s", sleep_secs);
tokio::time::sleep(std::time::Duration::from_secs(sleep_secs)).await;
if let Err(e) = run_pregen_batch(&app_state).await {
// One root span per nightly batch.
if let Err(e) = run_pregen_batch(&app_state)
.instrument(tracing::info_span!("reel.pregen_batch"))
.await
{
log::error!("Reel pre-generation batch failed: {}", e);
}
}
@@ -1267,7 +1279,8 @@ pub(crate) async fn spawn_reel_cache_sweeper(app_state: web::Data<AppState>) {
// Settle after startup, then sweep on a fixed cadence.
tokio::time::sleep(Duration::from_secs(300)).await;
loop {
let removed = sweep_reel_cache(&app_state);
let removed =
tracing::info_span!("reel.cache_sweep").in_scope(|| sweep_reel_cache(&app_state));
if removed > 0 {
log::info!("Reel cache sweep removed {removed} stale file(s)");
}