Tracing: root spans for background jobs, retire raw OTel spans (#142)

Background work has no request parent, so each job now opens a named
tracing root and carries it into its fan-out:

- watcher: `watcher.tick` (scan=quick|full) entered per tick on the
  watcher's std thread, `watcher.library` per library, and
  `watcher.playlist_cleanup` per orphaned-HLS pass. face_watch /
  clip_watch carry the ambient span into every per-candidate
  tokio::spawn and give each call its own `face_watch.detect` /
  `clip_watch.encode` span.
- thumbnails: `creating thumbnails` via #[instrument]; per-video spans on
  rayon workers name their parent explicitly.
- video actors: `playlistgenerator.generate_playlist` and
  `preview_clip.generate` are tracing roots that instrument their futures.
- reels: `reel.job` (created before the spawn, so on-demand jobs stay in
  the request trace), `reel.pregen_batch`, `reel.cache_sweep`.
- RefreshThumbnailsMessage and the dead-code GIF job move to tracing.

No raw OpenTelemetry span creation is left outside otel.rs;
`global_tracer` is removed.

tests/otel_background.rs pins the std-thread tick -> block_on -> spawn
shape and the rayon explicit-parent shape.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/otel_background.rs

@@ -0,0 +1,106 @@
+//! Verifies background-job root spans (#142 / map #135).
+//!
+//! Background work has no request parent, so each job opens its own root
+//! span and must carry it into whatever concurrency it fans out to. Two
+//! production shapes are pinned here:
+//!
+//! 1. **Watcher tick → per-pass runtime.** `watcher::watch_files` runs on a
+//!    plain std thread and enters a `watcher.tick` span; `face_watch` /
+//!    `clip_watch` then build a multi-thread runtime, `block_on` it, and
+//!    `tokio::spawn` one task per candidate. Each task must carry the
+//!    tick's span via `.instrument(parent.clone())`.
+//! 2. **Rayon fan-out.** `thumbnails::create_thumbnails` walks files on
+//!    rayon workers, where the ambient span isn't current, so child spans
+//!    name their parent explicitly (`info_span!(parent: &span, ...)`).
+//!
+//! One process-global subscriber per test binary, so this file holds a
+//! single test.
+
+use opentelemetry::trace::{SpanId, TracerProvider};
+use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider, SpanData};
+use rayon::prelude::*;
+use tracing::Instrument;
+use tracing_subscriber::layer::SubscriberExt;
+use tracing_subscriber::registry::Registry;
+use tracing_subscriber::util::SubscriberInitExt;
+
+fn find<'a>(spans: &'a [SpanData], name: &str) -> &'a SpanData {
+    spans
+        .iter()
+        .find(|s| s.name == name)
+        .unwrap_or_else(|| panic!("span {name} not exported"))
+}
+
+#[tracing::instrument(name = "test.detect_one", skip_all)]
+async fn detect_one() {
+    tokio::task::yield_now().await;
+}
+
+#[test]
+fn background_work_nests_under_job_root() {
+    let exporter = InMemorySpanExporter::default();
+    let provider = SdkTracerProvider::builder()
+        .with_simple_exporter(exporter.clone())
+        .build();
+    Registry::default()
+        .with(tracing_opentelemetry::OpenTelemetryLayer::new(
+            provider.tracer("background-test"),
+        ))
+        .try_init()
+        .expect("global subscriber not yet set");
+
+    // Shape 1: std-thread tick with an entered root, per-pass runtime, spawns.
+    std::thread::spawn(|| {
+        let _tick = tracing::info_span!("test.tick").entered();
+        let rt = tokio::runtime::Builder::new_multi_thread()
+            .worker_threads(2)
+            .enable_all()
+            .build()
+            .unwrap();
+        let parent = tracing::Span::current();
+        rt.block_on(async move {
+            let handles: Vec<_> = (0..3)
+                .map(|_| tokio::spawn(detect_one().instrument(parent.clone())))
+                .collect();
+            for h in handles {
+                h.await.unwrap();
+            }
+        });
+    })
+    .join()
+    .unwrap();
+
+    // Shape 2: rayon fan-out with an explicit parent.
+    {
+        let root = tracing::info_span!("test.thumbnails");
+        (0..4).into_par_iter().for_each(|_| {
+            let _child = tracing::info_span!(parent: &root, "test.thumbnail_one");
+        });
+    }
+    let _ = provider.force_flush();
+
+    let spans = exporter.get_finished_spans().unwrap();
+
+    let tick = find(&spans, "test.tick");
+    assert_eq!(tick.parent_span_id, SpanId::INVALID, "tick is a root span");
+    let detects: Vec<_> = spans
+        .iter()
+        .filter(|s| s.name == "test.detect_one")
+        .collect();
+    assert_eq!(detects.len(), 3);
+    for d in detects {
+        assert_eq!(d.span_context.trace_id(), tick.span_context.trace_id());
+        assert_eq!(d.parent_span_id, tick.span_context.span_id());
+    }
+
+    let root = find(&spans, "test.thumbnails");
+    let children: Vec<_> = spans
+        .iter()
+        .filter(|s| s.name == "test.thumbnail_one")
+        .collect();
+    assert_eq!(children.len(), 4);
+    for c in children {
+        assert_eq!(c.span_context.trace_id(), root.span_context.trace_id());
+        assert_eq!(c.parent_span_id, root.span_context.span_id());
+    }
+}

tests/otel_db_spans.rs

@@ -14,8 +14,8 @@
 //! One process-global subscriber per test binary, so this file holds a
 //! single test.
 
-use opentelemetry::trace::{Status, TracerProvider};
 use opentelemetry::Value;
+use opentelemetry::trace::{Status, TracerProvider};
 use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider, SpanData};
 use tracing::Instrument;
 use tracing_opentelemetry::OpenTelemetrySpanExt;
@@ -85,7 +85,10 @@ fn db_spans_nest_under_ambient_span() {
     let del = find(&spans, "db.delete.remove_tag");
 
     for (span, qt, op) in [(get, "query", "get_tags"), (del, "delete", "remove_tag")] {
-        assert_eq!(span.span_context.trace_id(), handler.span_context.trace_id());
+        assert_eq!(
+            span.span_context.trace_id(),
+            handler.span_context.trace_id()
+        );
         assert_eq!(
             span.parent_span_id,
             handler.span_context.span_id(),