Support protobuf over HTTP for OTLP export

.github/workflows/ci_build_production.yml

@@ -63,6 +63,15 @@ jobs:
       - name: Clippy (production features)
         run: cargo clippy --workspace --all-targets --features production --no-deps -- -D warnings
 
+      # The Test Suite workflow builds with default features, so tests behind a production-only
+      # gate like `#[cfg(feature = "otel")]` never run there. This job is the only one that
+      # enables them. The filter limits the step to lib unit tests needing no server, so a
+      # feature-gated test whose path does not match it stays uncovered: widen the filter when
+      # adding one elsewhere. oxen-py is excluded because its test binary needs the Python
+      # runtime installed by the maturin step below.
+      - name: Test (production features)
+        run: cargo test --workspace --exclude oxen-py --features production --lib telemetry
+
       - name: Build oxen-python with production features
         run: |
           cd oxen-python

Cargo.toml

@@ -105,12 +105,10 @@ mockito = "1.1.0"
 mp4 = "0.14.0"
 num_cpus = "1.16.0"
 opentelemetry = "0.31"
-opentelemetry-otlp = { version = "0.31", features = [
-    "grpc-tonic",
-    "http-json",
-    "tls-roots",
-    "trace",
-] }
+# The OTLP/HTTP encoding is chosen by this feature list, not at runtime. Binary protobuf comes
+# from the default "http-proto"; adding "http-json" back would make JSON the exporter's default
+# encoding, which is why the builder names its encoding instead of inheriting it.
+opentelemetry-otlp = { version = "0.31", features = ["grpc-tonic", "gzip-http", "gzip-tonic", "tls-roots", "trace"] }
 opentelemetry_sdk = { version = "0.31", features = ["rt-tokio"] }
 os_path = "0.8.0"
 page_size = "0.6.0"

bin/otel-metrics-test

@@ -9,8 +9,21 @@
 # Prerequisites:
 #   cargo build --workspace --features production
 #   docker (running)
-#   uv, maturin (for oxen-python)
+#   uv, maturin (for oxen-python). The bindings must be built with production features, or the
+#     oxen-python trace assertion fails while every other one passes.
 #   jq
+#
+# Environment:
+#   OXEN_SERVER_PORT   port for the oxen-server this starts (default 3000)
+#   OXEN_METRICS_PORT  port for its Prometheus endpoint (default 9090)
+#
+# SYNC_DIR is not read. This always uses a temp data directory of its own, removed on exit.
+#
+# Set the ports to run alongside a development server holding the defaults:
+#   OXEN_SERVER_PORT=3010 OXEN_METRICS_PORT=9091 bin/otel-metrics-test
+#
+# Jaeger's ports (16686, 4317, 4318, 5778, 9411) are still fixed, so stop any container
+# already publishing them first.
 
 set -euo pipefail
 
@@ -22,6 +35,19 @@ SERVER="$REPO_ROOT/target/debug/oxen-server"
 SERVER_PID=""
 JAEGER_NAME=""
 
+# Overridable so a run can step around a development server already on these ports. A server
+# holding 3000 answers the readiness probe below, and the workflow then runs against it with no
+# telemetry configured, failing every assertion for a reason unrelated to telemetry.
+OXEN_SERVER_PORT="${OXEN_SERVER_PORT:-3000}"
+OXEN_METRICS_PORT="${OXEN_METRICS_PORT:-9090}"
+SERVER_URL="http://localhost:$OXEN_SERVER_PORT"
+
+# Overrides any SYNC_DIR in the environment, which a developer's shell commonly exports for their
+# own server. The workflow below creates a remote repository that nothing deletes, so inheriting
+# that value writes into data this must not touch and leaves the next run to fail on leftovers.
+SYNC_DIR="$(mktemp -d)"
+export SYNC_DIR
+
 # Track which config files we modified so cleanup can restore/remove them
 MODIFIED_HOME_CONFIG=false
 HOME_CONFIG_BACKUP=""
@@ -89,7 +115,7 @@ cleanup() {
         fi
     fi
     # Remove only the temp paths this run created.
-    rm -rf "$REPO_A_DIR" "$REPO_B_PARENT" "$CI_USER_TOML"
+    rm -rf "$REPO_A_DIR" "$REPO_B_PARENT" "$CI_USER_TOML" "$SYNC_DIR"
 }
 trap cleanup EXIT INT TERM
 
@@ -163,17 +189,25 @@ if [ -f "$AUTH_CONFIG" ]; then
     AUTH_CONFIG_BACKUP=$(mktemp)
     cp "$AUTH_CONFIG" "$AUTH_CONFIG_BACKUP"
 fi
-"$OXEN" config --auth localhost:3000 "$TOKEN"
+"$OXEN" config --auth "localhost:$OXEN_SERVER_PORT" "$TOKEN"
 MODIFIED_AUTH_CONFIG=true
 
 # ── Start oxen-server with OTel export and Prometheus metrics ──
 # Deliberately no RUST_LOG anywhere in this script. Span export is filtered independently of the
 # log destinations, so every trace asserted below arrives at the stock log level — which for the
 # CLI and the Python bindings is "off". Setting RUST_LOG=info here would hide a regression that
 # re-couples the two.
-OXEN_OTEL_ENDPOINT=http://localhost:4317 \
-OXEN_METRICS_PORT=9090 \
-"$SERVER" start &
+#
+# The server exports over OTLP/HTTP while the CLI and the Python bindings below stay on gRPC, so
+# one run covers both transports against a real collector. The server is the one on HTTP because
+# that is what a deployment uses: it is the binary built with the otel feature, and hosted
+# collectors commonly accept OTLP/HTTP with binary protobuf and nothing else. The trace
+# assertions below read the server's spans, so a break in HTTP export shows up there as missing
+# or unparented spans rather than as an obvious export failure.
+OXEN_OTEL_ENDPOINT=http://localhost:4318 \
+OXEN_OTEL_PROTOCOL=http \
+OXEN_METRICS_PORT="$OXEN_METRICS_PORT" \
+"$SERVER" start -p "$OXEN_SERVER_PORT" &
 SERVER_PID=$!
 
 # Wait for server to be ready (bounded: 30 attempts, 1s apart)
@@ -184,7 +218,7 @@ for _i in $(seq 1 30); do
         echo "FAIL: oxen-server exited before becoming ready"
         exit 1
     fi
-    if curl -sf --max-time 2 http://localhost:3000/api/health > /dev/null 2>&1; then
+    if curl -sf --max-time 2 "$SERVER_URL/api/health" > /dev/null 2>&1; then
         echo "oxen-server is ready"
         server_ready=true
         break
@@ -205,20 +239,20 @@ TRACEPARENT_SPAN_ID="00f067aa0ba902b7"
 curl -sf --max-time 10 \
   -H "Authorization: Bearer $TOKEN" \
   -H "traceparent: 00-$TRACEPARENT_TRACE_ID-$TRACEPARENT_SPAN_ID-01" \
-  http://localhost:3000/api/version > /dev/null
+  "$SERVER_URL/api/version" > /dev/null
 
 # ── oxen client: Full push / clone / modify / push / pull workflow ──
 
 # Create remote repository (namespace "ox", name "otel-metrics-test")
 curl -sf --max-time 30 -H "Authorization: Bearer $TOKEN" \
   -H "Content-Type: application/json" \
   -X POST -d '{"namespace":"ox","name":"otel-metrics-test"}' \
-  http://localhost:3000/api/repos
+  "$SERVER_URL/api/repos"
 
 # Repo A: init, commit, push (REPO_A_DIR already created by mktemp -d)
 cd "$REPO_A_DIR"
 "$OXEN" init
-"$OXEN" config --set-remote origin http://localhost:3000/ox/otel-metrics-test
+"$OXEN" config --set-remote origin "$SERVER_URL/ox/otel-metrics-test"
 echo "hello from CI" > test-file.txt
 "$OXEN" add test-file.txt
 "$OXEN" commit -m "initial commit"
@@ -227,7 +261,7 @@ echo "hello from CI" > test-file.txt
 # Clone to Repo B (oxen clone rejects absolute paths, so cd into a parent
 # tmpdir and clone using the relative "repo-b" name)
 cd "$REPO_B_PARENT"
-"$OXEN" clone http://localhost:3000/ox/otel-metrics-test repo-b
+"$OXEN" clone "$SERVER_URL/ox/otel-metrics-test" repo-b
 
 # Repo B: modify, commit, push
 cd "$REPO_B_DIR"
@@ -335,9 +369,9 @@ fi
 
 # ── Verify Prometheus metrics ──
 
-METRICS_OUTPUT=$(curl -sf --max-time 5 http://localhost:9090/metrics 2>/dev/null || true)
+METRICS_OUTPUT=$(curl -sf --max-time 5 "http://localhost:$OXEN_METRICS_PORT/metrics" 2>/dev/null || true)
 if [ -z "$METRICS_OUTPUT" ]; then
-  echo "FAIL: Could not fetch /metrics from Prometheus endpoint on port 9090"
+  echo "FAIL: Could not fetch /metrics from Prometheus endpoint on port $OXEN_METRICS_PORT"
   did_fail=true
 else
   echo "--- Prometheus /metrics sample ---"