From ecd49fd0535b49cf7569f8a4e24e67ed6718b324 Mon Sep 17 00:00:00 2001
From: Cameron Cordes <cameronc.dev@gmail.com>
Date: Wed, 6 May 2026 18:33:37 -0400
Subject: [PATCH] otel: revert HTTP transport, keep gRPC
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The HTTP/protobuf exporter never sent any traffic in prod (tcpdump
on port 4318 showed nothing) despite the receiver path being correct
and the bridge wiring being intact (logs reached journalctl via the
stdout exporter). Likely the BatchLogProcessor + reqwest-client combo
isn't getting the right runtime context, but debugging that on a live
deployment isn't worth holding up the rest of the speedups.

Restoring grpc-tonic transport so prod observability comes back. The
remaining build-time wins on this branch (mold linker, system sqlite3,
profile.dev tweaks, lockfile-only dep refresh) deliver most of the
original savings without touching telemetry. Operator: revert
OTLP_OTLS_ENDPOINT in prod from port 4318 back to 4317.

HTTP transport remains a viable follow-up — needs to be debugged
against a local SigNoz instance with internal SDK error visibility
enabled, on its own branch.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 Cargo.lock  | 26 ++++++++++++++++++++++++++
 Cargo.toml  |  6 +++---
 src/otel.rs |  4 ++--
 3 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index eee335a..3f6d0c2 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1826,6 +1826,19 @@ dependencies = [
  "tower-service",
 ]
 
+[[package]]
+name = "hyper-timeout"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0"
+dependencies = [
+ "hyper",
+ "hyper-util",
+ "pin-project-lite",
+ "tokio",
+ "tower-service",
+]
+
 [[package]]
 name = "hyper-tls"
 version = "0.6.0"
@@ -2732,6 +2745,9 @@ dependencies = [
  "prost",
  "reqwest",
  "thiserror 2.0.18",
+ "tokio",
+ "tonic",
+ "tracing",
 ]
 
 [[package]]
@@ -3377,6 +3393,7 @@ dependencies = [
  "base64",
  "bytes",
  "encoding_rs",
+ "futures-channel",
  "futures-core",
  "futures-util",
  "h2 0.4.14",
@@ -4162,10 +4179,15 @@ dependencies = [
  "http 1.4.0",
  "http-body",
  "http-body-util",
+ "hyper",
+ "hyper-timeout",
+ "hyper-util",
  "percent-encoding",
  "pin-project",
  "sync_wrapper",
+ "tokio",
  "tokio-stream",
+ "tower",
  "tower-layer",
  "tower-service",
  "tracing",
@@ -4190,11 +4212,15 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4"
 dependencies = [
  "futures-core",
  "futures-util",
+ "indexmap",
  "pin-project-lite",
+ "slab",
  "sync_wrapper",
  "tokio",
+ "tokio-util",
  "tower-layer",
  "tower-service",
+ "tracing",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
index 657c3e0..7713970 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -44,9 +44,9 @@ prometheus = "0.13"
 lazy_static = "1.5"
 anyhow = "1.0"
 rand = "0.8.5"
-opentelemetry = { version = "0.31.0", features = ["metrics", "trace"] }
-opentelemetry_sdk = { version = "0.31.0", features = ["rt-tokio-current-thread", "metrics", "logs"] }
-opentelemetry-otlp = { version = "0.31.0", default-features = false, features = ["trace", "metrics", "logs", "http-proto", "reqwest-client"] }
+opentelemetry = { version = "0.31.0", features = ["default", "metrics", "tracing"] }
+opentelemetry_sdk = { version = "0.31.0", features = ["default", "rt-tokio-current-thread", "metrics"] }
+opentelemetry-otlp = { version = "0.31.0", features = ["default", "metrics", "tracing", "grpc-tonic"] }
 opentelemetry-stdout = "0.31.0"
 opentelemetry-appender-log = "0.31.0"
 tempfile = "3.20.0"
diff --git a/src/otel.rs b/src/otel.rs
index 4b9fb8a..110a2d1 100644
--- a/src/otel.rs
+++ b/src/otel.rs
@@ -24,7 +24,7 @@ pub fn init_tracing() {
         .build();
 
     let span_exporter = opentelemetry_otlp::SpanExporter::builder()
-        .with_http()
+        .with_tonic()
         .with_endpoint(std::env::var("OTLP_OTLS_ENDPOINT").unwrap())
         .build()
         .unwrap();
@@ -40,7 +40,7 @@ pub fn init_tracing() {
 #[allow(dead_code)]
 pub fn init_logs() {
     let otlp_exporter = opentelemetry_otlp::LogExporter::builder()
-        .with_http()
+        .with_tonic()
         .with_endpoint(std::env::var("OTLP_OTLS_ENDPOINT").unwrap())
         .build()
         .unwrap();