feat(otel): export logs through the tracing pipeline, correlated with traces (#147)

Logs were a separate pipeline: env_logger in debug (never exported) and
opentelemetry-appender-log in release (exported without trace context,
so SigNoz could not link logs to traces).

- tracing-subscriber `tracing-log` feature: try_init installs LogTracer,
  so log:: records become tracing events: one console format, and
  RUST_LOG per-module directives still apply to them.
- opentelemetry-appender-tracing exports events as OTel log records when
  OTLP_OTLS_ENDPOINT is set (any build), stamped with the enclosing
  span's trace_id/span_id. A per-layer filter drops the exporter's own
  crates.
- The OTel span layer keeps only WARN/ERROR events as span events; the
  info/debug copies duplicated the correlated logs.
- Remove env_logger init in the server, init_logs, and the unused
  opentelemetry-appender-log / opentelemetry-stdout deps.
- tests/otel_log_correlation.rs pins correlation (native + bridged),
  the span-event filter, and per-module filtering of bridged records.
- Constitution 1.2.0 (Principle V logging rule), CLAUDE.md, README,
  .env.example.

Closes #147

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/main.rs

@@ -111,17 +111,9 @@ fn main() -> std::io::Result<()> {
     let system = actix::System::new();
     system.block_on(async {
         // One `tracing` subscriber in every build: env-driven, so dev prints
-        // spans to the console and release exports them over OTLP. The `log`
-        // pipeline stays separate — env_logger in debug, OTel logs in release.
-        otel::init_tracing();
-        #[cfg(debug_assertions)]
-        {
-            env_logger::init();
-        }
-        #[cfg(not(debug_assertions))]
-        {
-            otel::init_logs();
-        }
+        // spans + logs to the console and OTLP_OTLS_ENDPOINT exports both.
+        // `log::` records are bridged into it.
+        let _logger_provider = otel::init_tracing();
 
         // AppState construction loads (and seeds if needed) the libraries
         // table; we use that list to drive the initial thumbnail sweep.

src/otel.rs

@@ -1,29 +1,33 @@
 use opentelemetry::propagation::TextMapPropagator;
 use opentelemetry::trace::{Status, TraceContextExt, TracerProvider};
 use opentelemetry::{Context, KeyValue, global};
-use opentelemetry_appender_log::OpenTelemetryLogBridge;
+use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge;
 use opentelemetry_otlp::WithExportConfig;
 use opentelemetry_sdk::Resource;
-use opentelemetry_sdk::logs::{BatchLogProcessor, SdkLoggerProvider};
+use opentelemetry_sdk::logs::SdkLoggerProvider;
 use opentelemetry_sdk::propagation::TraceContextPropagator;
+use tracing_subscriber::Layer;
 use tracing_subscriber::layer::SubscriberExt;
 use tracing_subscriber::util::SubscriberInitExt;
 
-/// Install the process-wide `tracing` subscriber plus the OTel span pipeline.
+/// Install the process-wide `tracing` subscriber plus the OTel span and log
+/// pipelines.
 ///
-/// Called in BOTH debug and release builds (the old split made spans a no-op
-/// in dev). Behaviour is driven by the environment, not the build profile:
-/// - `OTLP_OTLS_ENDPOINT` set -> spans are batch-exported over OTLP (the
-///   release / staging path).
-/// - unset                    -> spans are created with valid contexts (so
-///   outgoing `traceparent` injection still works) but exported nowhere; the
-///   console `fmt` layer is what shows them.
+/// One subscriber handles everything, in every build; behaviour comes from
+/// the environment, not the build profile:
+/// - `OTLP_OTLS_ENDPOINT` set -> spans AND log records are batch-exported
+///   over OTLP. Log records carry the trace_id / span_id of the span they
+///   were emitted in, so SigNoz correlates logs with traces.
+/// - unset                    -> spans get valid contexts (outgoing
+///   `traceparent` injection still works) but nothing is exported.
 ///
-/// The console `fmt` layer is always present, so a release build run locally
-/// also prints spans to stdout. `log` records are NOT routed through this
-/// subscriber (the `tracing-log` bridge is off) — they keep their existing
-/// path: `env_logger` in debug, OTel logs in release.
-pub fn init_tracing() {
+/// The console `fmt` layer is always present. `log::` records are bridged
+/// into `tracing` (the `tracing-log` feature makes `try_init` install
+/// `LogTracer`), so they print in the same format and export the same way.
+///
+/// Returns the logger provider when log export is on; the caller keeps it
+/// alive and calls `shutdown()` on exit to flush the batch.
+pub fn init_tracing() -> Option<SdkLoggerProvider> {
     // W3C propagator for the whole process: the request middleware
     // (tracing-actix-web) and the outgoing `traceparent` helpers both read
     // the *global* propagator, so it must be set before any request lands.
@@ -36,10 +40,11 @@ pub fn init_tracing() {
         ])
         .build();
 
-    let mut provider_builder =
-        opentelemetry_sdk::trace::SdkTracerProvider::builder().with_resource(resources);
+    let endpoint = std::env::var("OTLP_OTLS_ENDPOINT").ok();
 
-    if let Ok(endpoint) = std::env::var("OTLP_OTLS_ENDPOINT") {
+    let mut provider_builder =
+        opentelemetry_sdk::trace::SdkTracerProvider::builder().with_resource(resources.clone());
+    if let Some(endpoint) = &endpoint {
         let span_exporter = opentelemetry_otlp::SpanExporter::builder()
             .with_tonic()
             .with_endpoint(endpoint)
@@ -55,7 +60,27 @@ pub fn init_tracing() {
     let tracer = tracer_provider.tracer("image-server");
     global::set_tracer_provider(tracer_provider);
 
-    // One EnvFilter in front of BOTH layers: `RUST_LOG` also decides which
+    let logger_provider = endpoint.map(|endpoint| {
+        let log_exporter = opentelemetry_otlp::LogExporter::builder()
+            .with_tonic()
+            .with_endpoint(endpoint)
+            .build()
+            .expect("OTLP log exporter");
+        SdkLoggerProvider::builder()
+            .with_batch_exporter(log_exporter)
+            .with_resource(resources)
+            .build()
+    });
+
+    // The exporter's own crates log through `tracing` too; exporting those
+    // records would feed back into the exporter, so this layer drops them.
+    let log_layer = logger_provider.as_ref().map(|provider| {
+        OpenTelemetryTracingBridge::new(provider).with_filter(tracing_subscriber::EnvFilter::new(
+            "trace,opentelemetry=off,opentelemetry_sdk=off,opentelemetry_otlp=off,tonic=off,h2=off,hyper=off,tower=off",
+        ))
+    });
+
+    // One EnvFilter in front of every layer: `RUST_LOG` also decides which
     // spans exist for export. All spans are `info`, so keep it at info/debug
     // (see CLAUDE.md "Tracing"); `warn` turns span export off.
     tracing_subscriber::registry()
@@ -63,39 +88,27 @@ pub fn init_tracing() {
             tracing_subscriber::EnvFilter::try_from_default_env()
                 .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
         )
-        .with(tracing_opentelemetry::OpenTelemetryLayer::new(tracer))
-        .with(tracing_subscriber::fmt::layer().with_target(false))
+        .with(
+            tracing_opentelemetry::OpenTelemetryLayer::new(tracer).with_filter(span_layer_filter()),
+        )
+        .with(log_layer)
+        .with(tracing_subscriber::fmt::layer())
         .try_init()
         .expect("failed to install tracing subscriber");
+
+    logger_provider
 }
 
-#[allow(dead_code)]
-pub fn init_logs() {
-    let otlp_exporter = opentelemetry_otlp::LogExporter::builder()
-        .with_tonic()
-        .with_endpoint(std::env::var("OTLP_OTLS_ENDPOINT").unwrap())
-        .build()
-        .unwrap();
-
-    let exporter = opentelemetry_stdout::LogExporter::default();
-
-    let resources = Resource::builder()
-        .with_attributes([
-            KeyValue::new("service.name", "image-server"),
-            KeyValue::new("service.version", env!("CARGO_PKG_VERSION")),
-        ])
-        .build();
-
-    let log_provider = SdkLoggerProvider::builder()
-        .with_log_processor(BatchLogProcessor::builder(exporter).build())
-        .with_log_processor(BatchLogProcessor::builder(otlp_exporter).build())
-        .with_resource(resources)
-        .build();
-
-    let otel_log_appender = OpenTelemetryLogBridge::new(&log_provider);
-    log::set_boxed_logger(Box::new(otel_log_appender)).expect("Unable to set boxed logger");
-    //TODO: Still set this with the env? Ideally we still have a clean/simple local logger for local dev
-    log::set_max_level(log::LevelFilter::Info);
+/// Per-layer filter for the OTel span layer: every span, but only WARN and
+/// ERROR events. `tracing-opentelemetry` copies each event it sees onto its
+/// span as a span event; with logs exported and correlated separately, the
+/// info/debug copies are duplicate noise in the trace view. WARN/ERROR stay
+/// because error events set the span's status.
+pub fn span_layer_filter()
+-> tracing_subscriber::filter::FilterFn<impl Fn(&tracing::Metadata<'_>) -> bool> {
+    tracing_subscriber::filter::filter_fn(|meta| {
+        meta.is_span() || *meta.level() <= tracing::Level::WARN
+    })
 }
 
 /// Start a client span for a single LLM call. Every provider funnels its