src/clip_watch.rs

@@ -29,6 +29,7 @@ use log::{debug, info, warn};
 use std::path::Path;
 use std::sync::{Arc, Mutex};
 use tokio::sync::Semaphore;
+use tracing::Instrument;
 
 /// One file the watcher would like to CLIP-encode. Built from the DAO
 /// `list_clip_unencoded_candidates` result — needs the `content_hash`
@@ -90,6 +91,9 @@ pub fn run_clip_encoding_pass(
 
     let library_id = library.id;
     let library_root = library.root_path.clone();
+    // The detect tasks run on this runtime's worker threads; carry the
+    // watcher's span into each so their spans stay in the tick's trace.
+    let parent = tracing::Span::current();
     rt.block_on(async move {
         let sem = Arc::new(Semaphore::new(concurrency));
         let mut handles = Vec::with_capacity(filtered.len());
@@ -98,10 +102,14 @@ pub fn run_clip_encoding_pass(
             let clip_client = clip_client.clone();
             let exif_dao = exif_dao.clone();
             let library_root = library_root.clone();
-            handles.push(tokio::spawn(async move {
+            let task_span = parent.clone();
+            handles.push(tokio::spawn(
+                async move {
                     let _permit = permit_sem.acquire().await.expect("clip semaphore");
                     process_one(library_id, &library_root, cand, &clip_client, exif_dao).await;
-            }));
+                }
+                .instrument(task_span),
+            ));
         }
         for h in handles {
             let _ = h.await;
@@ -109,6 +117,7 @@ pub fn run_clip_encoding_pass(
     });
 }
 
+#[tracing::instrument(name = "clip_watch.encode", skip_all, fields(rel_path = %cand.rel_path))]
 async fn process_one(
     library_id: i32,
     library_root: &str,

src/face_watch.rs

@@ -25,6 +25,7 @@ use log::{debug, info, warn};
 use std::path::Path;
 use std::sync::{Arc, Mutex};
 use tokio::sync::Semaphore;
+use tracing::Instrument;
 
 /// One file the watcher would like to face-scan. Built by the caller from
 /// the EXIF batch (we need `content_hash` to key everything against).
@@ -89,6 +90,9 @@ pub fn run_face_detection_pass(
 
     let library_id = library.id;
     let library_root = library.root_path.clone();
+    // The detect tasks run on this runtime's worker threads; carry the
+    // watcher's span into each so their spans stay in the tick's trace.
+    let parent = tracing::Span::current();
     rt.block_on(async move {
         let sem = Arc::new(Semaphore::new(concurrency));
         let mut handles = Vec::with_capacity(filtered.len());
@@ -98,7 +102,9 @@ pub fn run_face_detection_pass(
             let face_dao = face_dao.clone();
             let tag_dao = tag_dao.clone();
             let library_root = library_root.clone();
-            handles.push(tokio::spawn(async move {
+            let task_span = parent.clone();
+            handles.push(tokio::spawn(
+                async move {
                     // acquire_owned would let us drop the permit explicitly
                     // before await points; for a one-shot call into Apollo
                     // the simpler bounded acquire is enough.
@@ -112,7 +118,9 @@ pub fn run_face_detection_pass(
                         tag_dao,
                     )
                     .await;
-            }));
+                }
+                .instrument(task_span),
+            ));
         }
         for h in handles {
             // join; per-task panics are logged inside process_one before
@@ -122,6 +130,7 @@ pub fn run_face_detection_pass(
     });
 }
 
+#[tracing::instrument(name = "face_watch.detect", skip_all, fields(rel_path = %cand.rel_path))]
 async fn process_one(
     library_id: i32,
     library_root: &str,

src/files.rs

@@ -30,11 +30,10 @@ use actix_web::{
 use chrono::{DateTime, Utc};
 use log::{debug, error, info, trace, warn};
 use opentelemetry::KeyValue;
-use opentelemetry::trace::{Status, Tracer};
+use opentelemetry::trace::Status;
 
 use crate::data::SortType::NameAsc;
 use crate::error::IntoHttpError;
-use crate::otel::global_tracer;
 use crate::tags::{FileWithTagCount, TagDao};
 use crate::video::actors::StreamActor;
 use path_absolutize::*;
@@ -1373,9 +1372,8 @@ impl Message for RefreshThumbnailsMessage {
 impl Handler<RefreshThumbnailsMessage> for StreamActor {
     type Result = ();
 
+    #[tracing::instrument(name = "RefreshThumbnailsMessage", skip_all)]
     fn handle(&mut self, _msg: RefreshThumbnailsMessage, _ctx: &mut Self::Context) -> Self::Result {
-        let tracer = global_tracer();
-        let _ = tracer.start("RefreshThumbnailsMessage");
         info!("Refreshing thumbnails after upload");
         // The stub in lib.rs is a no-op; the real generation is driven by
         // the file watcher tick in main.rs, which has access to the

src/otel.rs

@@ -1,4 +1,3 @@
-use opentelemetry::global::BoxedTracer;
 use opentelemetry::propagation::TextMapPropagator;
 use opentelemetry::trace::{Status, TraceContextExt, TracerProvider};
 use opentelemetry::{Context, KeyValue, global};
@@ -10,10 +9,6 @@ use opentelemetry_sdk::propagation::TraceContextPropagator;
 use tracing_subscriber::layer::SubscriberExt;
 use tracing_subscriber::util::SubscriberInitExt;
 
-pub fn global_tracer() -> BoxedTracer {
-    global::tracer("image-server")
-}
-
 /// Install the process-wide `tracing` subscriber plus the OTel span pipeline.
 ///
 /// Called in BOTH debug and release builds (the old split made spans a no-op
@@ -54,9 +49,9 @@ pub fn init_tracing() {
     }
     let tracer_provider = provider_builder.build();
 
-    // The rest of the codebase resolves spans through the global provider
-    // (`global_tracer()`, `global::tracer("db")`), so install it there; the
-    // subscriber's OTel layer uses the same provider's tracer.
+    // Install the provider globally too (flush/shutdown and any direct
+    // `global::tracer` users); the subscriber's OTel layer uses the same
+    // provider's tracer.
     let tracer = tracer_provider.tracer("image-server");
     global::set_tracer_provider(tracer_provider);
 

src/reels/mod.rs

@@ -28,6 +28,7 @@ use anyhow::{Context, anyhow};
 use chrono::{DateTime, Datelike, Timelike};
 use serde::{Deserialize, Serialize};
 use serde_json::json;
+use tracing::Instrument;
 use uuid::Uuid;
 
 use crate::data::Claims;
@@ -557,7 +558,12 @@ pub async fn create_reel_handler(
     let state = app_state.clone();
     let insight_dao = insight_dao.clone();
     let exif_dao = exif_dao.clone();
-    let handle = tokio::spawn(async move {
+    // Created before the spawn so the job parents on the dispatching request
+    // (or runs as its own root from the pre-gen scheduler), then carried into
+    // the task — tracing context doesn't cross `tokio::spawn` on its own.
+    let job_span = tracing::info_span!("reel.job", job_id = %job_id);
+    let handle = tokio::spawn(
+        async move {
             match run_reel_job(
                 &state,
                 &insight_dao,
@@ -584,7 +590,9 @@ pub async fn create_reel_handler(
                     )
                 }
             }
-    });
+        }
+        .instrument(job_span),
+    );
     with_job(job_id, |job| job.abort = Some(handle.abort_handle()));
 
     // Capture params for passive prefs mirror (best-effort, never fails).
@@ -1056,7 +1064,11 @@ pub(crate) async fn spawn_pregen_scheduler(app_state: web::Data<AppState>) {
             log::debug!("Next pre-gen run in {}s", sleep_secs);
             tokio::time::sleep(std::time::Duration::from_secs(sleep_secs)).await;
 
-            if let Err(e) = run_pregen_batch(&app_state).await {
+            // One root span per nightly batch.
+            if let Err(e) = run_pregen_batch(&app_state)
+                .instrument(tracing::info_span!("reel.pregen_batch"))
+                .await
+            {
                 log::error!("Reel pre-generation batch failed: {}", e);
             }
         }
@@ -1267,7 +1279,8 @@ pub(crate) async fn spawn_reel_cache_sweeper(app_state: web::Data<AppState>) {
         // Settle after startup, then sweep on a fixed cadence.
         tokio::time::sleep(Duration::from_secs(300)).await;
         loop {
-            let removed = sweep_reel_cache(&app_state);
+            let removed =
+                tracing::info_span!("reel.cache_sweep").in_scope(|| sweep_reel_cache(&app_state));
             if removed > 0 {
                 log::info!("Reel cache sweep removed {removed} stale file(s)");
             }

src/thumbnails.rs

@@ -14,19 +14,15 @@ use image::GenericImageView;
 use image::codecs::jpeg::JpegEncoder;
 use lazy_static::lazy_static;
 use log::{debug, error, info, warn};
-use opentelemetry::{
-    KeyValue,
-    trace::{Span, TraceContextExt, Tracer},
-};
 use prometheus::IntGauge;
 use rayon::prelude::*;
+use tracing_opentelemetry::OpenTelemetrySpanExt;
 use walkdir::DirEntry;
 
 use crate::content_hash;
 use crate::exif;
 use crate::file_types;
 use crate::libraries;
-use crate::otel::global_tracer;
 use crate::video::actors::{generate_image_thumbnail_ffmpeg, generate_video_thumbnail};
 
 /// Maximum long-edge size (px) for the large preview tier. Tuned to look
@@ -327,9 +323,9 @@ fn generate_xlarge_preview_ffmpeg(
     Ok(())
 }
 
+#[tracing::instrument(name = "creating thumbnails", skip_all)]
 pub fn create_thumbnails(libs: &[libraries::Library], excluded_dirs: &[String]) {
-    let tracer = global_tracer();
-    let span = tracer.start("creating thumbnails");
+    let span = tracing::Span::current();
 
     let thumbs = &dotenv::var("THUMBNAILS").expect("THUMBNAILS not defined");
     let thumbnail_directory: &Path = Path::new(thumbs);
@@ -387,16 +383,12 @@ pub fn create_thumbnails(libs: &[libraries::Library], excluded_dirs: &[String])
                 }
 
                 if is_video(&entry) {
-                    let mut video_span = tracer.start_with_context(
-                        "generate_video_thumbnail",
-                        &opentelemetry::Context::new()
-                            .with_remote_span_context(span.span_context().clone()),
-                    );
-                    video_span.set_attributes(vec![
-                        KeyValue::new("type", "video"),
-                        KeyValue::new("file-name", thumb_path.display().to_string()),
-                        KeyValue::new("library", lib.name.clone()),
-                    ]);
+                    // Explicit parent: this closure runs on rayon worker
+                    // threads, where the ambient span isn't current.
+                    let video_span = tracing::info_span!(parent: &span, "generate_video_thumbnail");
+                    video_span.set_attribute("type", "video");
+                    video_span.set_attribute("file-name", thumb_path.display().to_string());
+                    video_span.set_attribute("library", lib.name.clone());
 
                     debug!("Generating video thumbnail: {:?}", thumb_path);
                     if let Err(e) = generate_video_thumbnail(src, &thumb_path) {
@@ -409,7 +401,7 @@ pub fn create_thumbnails(libs: &[libraries::Library], excluded_dirs: &[String])
                             warn!("Failed to write sentinel {:?}: {}", sentinel, se);
                         }
                     }
-                    video_span.end();
+                    drop(video_span);
                 } else if is_image(&entry) {
                     match generate_image_thumbnail(src, &thumb_path) {
                         Ok(_) => info!("Saved thumbnail: {:?}", thumb_path),

src/video/actors.rs

@@ -1,18 +1,19 @@
 use crate::content_hash;
 use crate::database::PreviewDao;
 use crate::libraries::Library;
-use crate::otel::global_tracer;
 use crate::video::ffmpeg::{generate_preview_clip, get_duration_seconds_blocking};
 use crate::video::hls_paths;
 use actix::prelude::*;
 use log::{debug, error, info, warn};
 use opentelemetry::KeyValue;
-use opentelemetry::trace::{Span, Status, Tracer};
+use opentelemetry::trace::Status;
 use std::io::Result;
 use std::path::{Path, PathBuf};
 use std::process::{Command, Stdio};
 use std::sync::{Arc, Mutex};
 use tokio::sync::Semaphore;
+use tracing::Instrument;
+use tracing_opentelemetry::OpenTelemetrySpanExt;
 // ffmpeg -i test.mp4 -c:v h264 -flags +cgop -g 30 -hls_time 3 out.m3u8
 // ffmpeg -i "filename.mp4" -preset veryfast -c:v libx264 -f hls -hls_list_size 100 -hls_time 2 -crf 24 -vf scale=1080:-2,setsar=1:1 attempt/vid_out.m3u8
 
@@ -448,15 +449,16 @@ impl Handler<GeneratePlaylistMessage> for PlaylistGenerator {
         let playlist_file = playlist_path.to_string_lossy().to_string();
         let segment_pattern = segment_template.to_string_lossy().to_string();
 
-        let tracer = global_tracer();
-        let mut span = tracer
-            .span_builder("playlistgenerator.generate_playlist")
-            .with_attributes(vec![
-                KeyValue::new("video_file", video_file.clone()),
-                KeyValue::new("content_hash", content_hash_str.clone()),
-                KeyValue::new("playlist_file", playlist_file.clone()),
-            ])
-            .start(&tracer);
+        // Job root: actix messages don't carry the sender's span, so each
+        // playlist generation is its own trace. The future is instrumented
+        // so any DB / child spans inside nest under it.
+        let span = tracing::info_span!(
+            "playlistgenerator.generate_playlist",
+            video_file = %video_file,
+            content_hash = %content_hash_str,
+            playlist_file = %playlist_file,
+        );
+        let fut_span = span.clone();
 
         Box::pin(async move {
             let wait_start = std::time::Instant::now();
@@ -691,7 +693,8 @@ impl Handler<GeneratePlaylistMessage> for PlaylistGenerator {
                 span.set_status(Status::error(detail.clone()));
                 Err(std::io::Error::other(detail))
             }
-        })
+        }
+        .instrument(fut_span))
     }
 }
 
@@ -818,7 +821,12 @@ impl Handler<GeneratePreviewClipMessage> for PreviewClipGenerator {
 
             let video_path_owned = video_path.clone();
             let relative_path_owned = relative_path.clone();
-            tokio::spawn(async move {
+            let clip_span = tracing::info_span!(
+                "preview_clip.generate",
+                rel_path = %relative_path_owned,
+            );
+            tokio::spawn(
+                async move {
                     match generate_preview_clip(&video_path_owned, &output_str).await {
                         Ok((duration, size)) => {
                             info!(
@@ -850,7 +858,9 @@ impl Handler<GeneratePreviewClipMessage> for PreviewClipGenerator {
                         }
                     }
                     drop(permit);
-            });
+                }
+                .instrument(clip_span),
+            );
         })
     }
 }

src/video/mod.rs

@@ -1,10 +1,9 @@
-use crate::otel::global_tracer;
 use crate::thumbnails::{is_video, update_media_counts};
 use crate::video::ffmpeg::{Ffmpeg, GifType};
 use log::info;
-use opentelemetry::trace::Tracer;
 use std::fs;
 use std::path::{Path, PathBuf};
+use tracing::Instrument;
 use walkdir::WalkDir;
 
 pub mod actors;
@@ -14,12 +13,11 @@ pub mod legacy_migration;
 
 #[allow(dead_code)]
 pub async fn generate_video_gifs() {
-    tokio::spawn(async {
+    tokio::spawn(
+        async {
             info!("Starting to make video gifs");
 
             let start = std::time::Instant::now();
-        let tracer = global_tracer();
-        tracer.start("creating video gifs");
 
             let gif_base_path = &dotenv::var("GIFS_DIRECTORY").unwrap_or(String::from("gifs"));
             let gif_directory: &Path = Path::new(gif_base_path);
@@ -76,5 +74,7 @@ pub async fn generate_video_gifs() {
             info!("Finished making video gifs in {:?}", start.elapsed());
 
             update_media_counts(&files, &excluded_dirs);
-    });
+        }
+        .instrument(tracing::info_span!("video.create_gifs")),
+    );
 }

src/watcher.rs

@@ -125,6 +125,7 @@ pub fn cleanup_orphaned_playlists(
             }
 
             info!("Running orphaned HLS cleanup");
+            let _pass = tracing::info_span!("watcher.playlist_cleanup").entered();
             let start = std::time::Instant::now();
 
             // Snapshot every live content_hash currently in image_exif.
@@ -418,6 +419,15 @@ pub fn watch_files(
 
             let is_full_scan = since_last_full.as_secs() >= full_interval_secs;
 
+            // Root span per tick: this loop runs on a plain std thread, so a
+            // synchronous `entered()` guard is the right shape. Everything the
+            // tick does (probes, drains, ingest, maintenance) nests under it.
+            let _tick = tracing::info_span!(
+                "watcher.tick",
+                scan = if is_full_scan { "full" } else { "quick" },
+            )
+            .entered();
+
             // Fresh snapshot per tick — picks up PATCH /libraries/{id}
             // mutations to `enabled` / `excluded_dirs` without restart.
             let libs: Vec<libraries::Library> =
@@ -436,6 +446,12 @@ pub fn watch_files(
                     );
                     continue;
                 }
+                let _lib_span = tracing::info_span!(
+                    "watcher.library",
+                    library = %lib.name,
+                    library_id = lib.id,
+                )
+                .entered();
 
                 // Availability probe: every tick checks that the
                 // library's mount is reachable, is a directory, is

tests/otel_background.rs

@@ -0,0 +1,106 @@
+//! Verifies background-job root spans (#142 / map #135).
+//!
+//! Background work has no request parent, so each job opens its own root
+//! span and must carry it into whatever concurrency it fans out to. Two
+//! production shapes are pinned here:
+//!
+//! 1. **Watcher tick → per-pass runtime.** `watcher::watch_files` runs on a
+//!    plain std thread and enters a `watcher.tick` span; `face_watch` /
+//!    `clip_watch` then build a multi-thread runtime, `block_on` it, and
+//!    `tokio::spawn` one task per candidate. Each task must carry the
+//!    tick's span via `.instrument(parent.clone())`.
+//! 2. **Rayon fan-out.** `thumbnails::create_thumbnails` walks files on
+//!    rayon workers, where the ambient span isn't current, so child spans
+//!    name their parent explicitly (`info_span!(parent: &span, ...)`).
+//!
+//! One process-global subscriber per test binary, so this file holds a
+//! single test.
+
+use opentelemetry::trace::{SpanId, TracerProvider};
+use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider, SpanData};
+use rayon::prelude::*;
+use tracing::Instrument;
+use tracing_subscriber::layer::SubscriberExt;
+use tracing_subscriber::registry::Registry;
+use tracing_subscriber::util::SubscriberInitExt;
+
+fn find<'a>(spans: &'a [SpanData], name: &str) -> &'a SpanData {
+    spans
+        .iter()
+        .find(|s| s.name == name)
+        .unwrap_or_else(|| panic!("span {name} not exported"))
+}
+
+#[tracing::instrument(name = "test.detect_one", skip_all)]
+async fn detect_one() {
+    tokio::task::yield_now().await;
+}
+
+#[test]
+fn background_work_nests_under_job_root() {
+    let exporter = InMemorySpanExporter::default();
+    let provider = SdkTracerProvider::builder()
+        .with_simple_exporter(exporter.clone())
+        .build();
+    Registry::default()
+        .with(tracing_opentelemetry::OpenTelemetryLayer::new(
+            provider.tracer("background-test"),
+        ))
+        .try_init()
+        .expect("global subscriber not yet set");
+
+    // Shape 1: std-thread tick with an entered root, per-pass runtime, spawns.
+    std::thread::spawn(|| {
+        let _tick = tracing::info_span!("test.tick").entered();
+        let rt = tokio::runtime::Builder::new_multi_thread()
+            .worker_threads(2)
+            .enable_all()
+            .build()
+            .unwrap();
+        let parent = tracing::Span::current();
+        rt.block_on(async move {
+            let handles: Vec<_> = (0..3)
+                .map(|_| tokio::spawn(detect_one().instrument(parent.clone())))
+                .collect();
+            for h in handles {
+                h.await.unwrap();
+            }
+        });
+    })
+    .join()
+    .unwrap();
+
+    // Shape 2: rayon fan-out with an explicit parent.
+    {
+        let root = tracing::info_span!("test.thumbnails");
+        (0..4).into_par_iter().for_each(|_| {
+            let _child = tracing::info_span!(parent: &root, "test.thumbnail_one");
+        });
+    }
+    let _ = provider.force_flush();
+
+    let spans = exporter.get_finished_spans().unwrap();
+
+    let tick = find(&spans, "test.tick");
+    assert_eq!(tick.parent_span_id, SpanId::INVALID, "tick is a root span");
+    let detects: Vec<_> = spans
+        .iter()
+        .filter(|s| s.name == "test.detect_one")
+        .collect();
+    assert_eq!(detects.len(), 3);
+    for d in detects {
+        assert_eq!(d.span_context.trace_id(), tick.span_context.trace_id());
+        assert_eq!(d.parent_span_id, tick.span_context.span_id());
+    }
+
+    let root = find(&spans, "test.thumbnails");
+    let children: Vec<_> = spans
+        .iter()
+        .filter(|s| s.name == "test.thumbnail_one")
+        .collect();
+    assert_eq!(children.len(), 4);
+    for c in children {
+        assert_eq!(c.span_context.trace_id(), root.span_context.trace_id());
+        assert_eq!(c.parent_span_id, root.span_context.span_id());
+    }
+}

tests/otel_db_spans.rs

@@ -14,8 +14,8 @@
 //! One process-global subscriber per test binary, so this file holds a
 //! single test.
 
-use opentelemetry::trace::{Status, TracerProvider};
 use opentelemetry::Value;
+use opentelemetry::trace::{Status, TracerProvider};
 use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider, SpanData};
 use tracing::Instrument;
 use tracing_opentelemetry::OpenTelemetrySpanExt;
@@ -85,7 +85,10 @@ fn db_spans_nest_under_ambient_span() {
     let del = find(&spans, "db.delete.remove_tag");
 
     for (span, qt, op) in [(get, "query", "get_tags"), (del, "delete", "remove_tag")] {
-        assert_eq!(span.span_context.trace_id(), handler.span_context.trace_id());
+        assert_eq!(
+            span.span_context.trace_id(),
+            handler.span_context.trace_id()
+        );
         assert_eq!(
             span.parent_span_id,
             handler.span_context.span_id(),