From f7c27c4cfb0e52bea34a4e9b7d873d67dc674ec3 Mon Sep 17 00:00:00 2001 From: Joshua Elliott Date: Tue, 11 Aug 2026 17:48:55 -0600 Subject: [PATCH 01/10] Encode OTLP/HTTP span exports as binary protobuf The exporter chooses its OTLP/HTTP encoding at compile time from the feature set and never reads OTEL_EXPORTER_OTLP_PROTOCOL, so the "http-json" feature made JSON the encoding of every export under OXEN_OTEL_PROTOCOL=http, with no runtime setting able to override it. OTLP/HTTP endpoints commonly accept only binary protobuf, and none reject it. Dropping the feature leaves the crate's default "http-proto" in place, so the HTTP path posts application/x-protobuf. The gRPC path already did and is unchanged. Serde support in the OTLP protocol types goes with it, taking proptest out of the dependency tree. --- Cargo.lock | 47 ------------------------------------ Cargo.toml | 10 +++----- crates/oxen-cli/README.md | 2 +- crates/oxen-server/README.md | 2 +- 4 files changed, 6 insertions(+), 55 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 30af9fde9d..4f307ae9eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2331,18 +2331,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "const-hex" -version = "1.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "531185e432bb31db1ecda541e9e7ab21468d4d844ad7505e0546a49b4945d49b" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "proptest", - "serde_core", -] - [[package]] name = "const-oid" version = "0.9.6" @@ -5572,7 +5560,6 @@ dependencies = [ "opentelemetry_sdk", "prost", "reqwest 0.12.28", - "serde_json", "thiserror 2.0.18", "tokio", "tonic", @@ -5585,13 +5572,9 @@ version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" dependencies = [ - "base64", - "const-hex", "opentelemetry", "opentelemetry_sdk", "prost", - "serde", - "serde_json", "tonic", "tonic-prost", ] @@ -6697,21 +6680,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "proptest" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" -dependencies = [ - "bitflags 2.11.0", - "num-traits", - "rand 0.9.2", - "rand_chacha 0.9.0", - "rand_xorshift", - "regex-syntax", - "unarray", -] - [[package]] name = "prost" version = "0.14.3" @@ -7101,15 +7069,6 @@ dependencies = [ "rand 0.8.5", ] -[[package]] -name = "rand_xorshift" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" -dependencies = [ - "rand_core 0.9.5", -] - [[package]] name = "rand_xoshiro" version = "0.7.0" @@ -9316,12 +9275,6 @@ dependencies = [ "libc", ] -[[package]] -name = "unarray" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" - [[package]] name = "unicase" version = "2.9.0" diff --git a/Cargo.toml b/Cargo.toml index 00cba9e5f8..349fa9ed5a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -105,12 +105,10 @@ mockito = "1.1.0" mp4 = "0.14.0" num_cpus = "1.16.0" opentelemetry = "0.31" -opentelemetry-otlp = { version = "0.31", features = [ - "grpc-tonic", - "http-json", - "tls-roots", - "trace", -] } +# The HTTP path's encoding is chosen at compile time, not from the environment: the exporter never +# reads `OTEL_EXPORTER_OTLP_PROTOCOL`, and enabling "http-json" would make JSON the default encoding +# for every OTLP/HTTP export. Binary protobuf comes from the crate's default "http-proto". +opentelemetry-otlp = { version = "0.31", features = ["grpc-tonic", "tls-roots", "trace"] } opentelemetry_sdk = { version = "0.31", features = ["rt-tokio"] } os_path = "0.8.0" page_size = "0.6.0" diff --git a/crates/oxen-cli/README.md b/crates/oxen-cli/README.md index 586e9d5e84..59da957df0 100644 --- a/crates/oxen-cli/README.md +++ b/crates/oxen-cli/README.md @@ -60,7 +60,7 @@ to be exported. | Variable | Description | Default | |---|---|---| | `OXEN_OTEL_ENDPOINT` | Collector endpoint URL. Absent = disabled. | *(none)* | -| `OXEN_OTEL_PROTOCOL` | Transport: `grpc` or `http` | `grpc` | +| `OXEN_OTEL_PROTOCOL` | Transport: `grpc` or `http` (binary protobuf either way) | `grpc` | | `RUST_LOG` | Must include `info` level for spans to be exported | `off` | ### Quick start with Jaeger diff --git a/crates/oxen-server/README.md b/crates/oxen-server/README.md index f51622deed..4849212759 100644 --- a/crates/oxen-server/README.md +++ b/crates/oxen-server/README.md @@ -237,7 +237,7 @@ OXEN_OTEL_ENDPOINT=https://otlp.vendor.example:443 oxen-server start | Variable | Description | Default | |---|---|---| | `OXEN_OTEL_ENDPOINT` | Collector endpoint: an `http://` or `https://` URL, or a bare `host:port` (which gets `http://`). Absent = export disabled. | *(none)* | -| `OXEN_OTEL_PROTOCOL` | Transport: `grpc` or `http`. Under `http` the OTLP signal path `/v1/traces` is appended to the endpoint unless it already names one. | `grpc` | +| `OXEN_OTEL_PROTOCOL` | Transport: `grpc` or `http`. Under `http` the OTLP signal path `/v1/traces` is appended to the endpoint unless it already names one, and the payload is binary protobuf. | `grpc` | | `OXEN_OTEL_FILTER` | Which spans and events are exported. Same syntax as `RUST_LOG`, and independent of it. | `info` | An `https://` endpoint is verified against the platform's root certificate From 693c93af309092345f3f599c32783634cf1aa237 Mon Sep 17 00:00:00 2001 From: Joshua Elliott Date: Tue, 11 Aug 2026 19:41:06 -0600 Subject: [PATCH 02/10] Support gzip on OTLP export and the standard protocol variable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compressing an export is a compile-time capability, so without the gzip features OTEL_EXPORTER_OTLP_COMPRESSION=gzip failed the exporter build, and a failed build starts the process with export disabled. Enabling them makes the standard variable work: roughly 8x on a full batch of spans, for a few milliseconds of CPU on the exporter's own thread rather than a request thread. Compression stays off unless the variable asks for it, because the SDK's parser accepts only algorithm names and has no value meaning "off" — a compiled-in default could not be turned back off. gzip pulls in no new dependency; flate2 is already built here, and keeps its faster zlib-ng backend. The transport now falls back to OTEL_EXPORTER_OTLP_PROTOCOL where OXEN_OTEL_PROTOCOL is unset, so a deployment configured through the standard variables alone exports over the transport it names instead of defaulting to gRPC against an endpoint that may accept only HTTP. That variable's http/protobuf and http/json both select the HTTP transport; the encoding is binary protobuf either way, which every OTLP/HTTP endpoint accepts. --- Cargo.lock | 2 + Cargo.toml | 2 +- crates/liboxen/src/util/telemetry.rs | 96 +++++++++++++++++++++++----- crates/oxen-cli/README.md | 2 +- crates/oxen-server/README.md | 9 ++- 5 files changed, 92 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4f307ae9eb..1f12828fc1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5553,6 +5553,7 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" dependencies = [ + "flate2", "http 1.5.0", "opentelemetry", "opentelemetry-http", @@ -9033,6 +9034,7 @@ dependencies = [ "async-trait", "base64", "bytes", + "flate2", "http 1.5.0", "http-body 1.0.1", "http-body-util", diff --git a/Cargo.toml b/Cargo.toml index 349fa9ed5a..0528e464b2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -108,7 +108,7 @@ opentelemetry = "0.31" # The HTTP path's encoding is chosen at compile time, not from the environment: the exporter never # reads `OTEL_EXPORTER_OTLP_PROTOCOL`, and enabling "http-json" would make JSON the default encoding # for every OTLP/HTTP export. Binary protobuf comes from the crate's default "http-proto". -opentelemetry-otlp = { version = "0.31", features = ["grpc-tonic", "tls-roots", "trace"] } +opentelemetry-otlp = { version = "0.31", features = ["grpc-tonic", "gzip-http", "gzip-tonic", "tls-roots", "trace"] } opentelemetry_sdk = { version = "0.31", features = ["rt-tokio"] } os_path = "0.8.0" page_size = "0.6.0" diff --git a/crates/liboxen/src/util/telemetry.rs b/crates/liboxen/src/util/telemetry.rs index 6b46701d11..fde483979c 100644 --- a/crates/liboxen/src/util/telemetry.rs +++ b/crates/liboxen/src/util/telemetry.rs @@ -22,7 +22,9 @@ pub enum TelemetryError { #[error("Failed to initialize tracing: {0}")] InitFail(#[from] TryInitError), #[cfg(feature = "otel")] - #[error("Unknown OXEN_OTEL_PROTOCOL value: {0}")] + #[error( + "Unknown OXEN_OTEL_PROTOCOL / OTEL_EXPORTER_OTLP_PROTOCOL value: {0} (expected grpc, http, http/protobuf, or http/json)" + )] UnknownProtocol(String), #[cfg(feature = "otel")] #[error( @@ -168,7 +170,9 @@ mod atexit_flush { /// env var `OTEL_EXPORTER_OTLP_ENDPOINT`, but checks `OXEN_OTEL_ENDPOINT` first. /// /// The `OXEN_OTEL_PROTOCOL` env var is either `"http"` or `"grpc"`: it controls -/// the protocol used for OTLP exports. If not set, defaults to `"grpc"`. +/// the transport used for OTLP exports. If not set, the standard +/// `OTEL_EXPORTER_OTLP_PROTOCOL` is read instead, and absent both it defaults to +/// `"grpc"`. Spans are encoded as binary protobuf under either transport. /// /// **Filtering** is per-layer, not global. `RUST_LOG` (falling back to `default`) gates the log /// destinations — stderr, the JSON file, and the caller-supplied layer. Span export is gated @@ -260,17 +264,7 @@ pub fn init_tracing_with_layer( Some(endpoint) => { let endpoint = normalize_otel_endpoint(&endpoint)?; - let protocol = match std::env::var("OXEN_OTEL_PROTOCOL") - .map(|x| x.to_lowercase()) - .ok() - .as_deref() - { - Some("http") => Protocol::Http, - Some("grpc") | None => Protocol::Grpc, - Some(unknown) => { - return Err(TelemetryError::UnknownProtocol(unknown.to_string())); - } - }; + let protocol = otel_protocol()?; match build_otel_layer(app_name, &protocol, &endpoint) { (Some(layer), Some(provider)) => { @@ -477,6 +471,32 @@ impl std::fmt::Display for Protocol { } } +/// The transport OTLP export uses: `OXEN_OTEL_PROTOCOL`, or the standard +/// `OTEL_EXPORTER_OTLP_PROTOCOL` where that is unset. +#[cfg(feature = "otel")] +fn otel_protocol() -> Result { + let configured = non_empty_env("OXEN_OTEL_PROTOCOL") + .or_else(|| non_empty_env("OTEL_EXPORTER_OTLP_PROTOCOL")); + parse_otel_protocol(configured.as_deref()) +} + +/// Read a configured protocol as a transport, defaulting to gRPC where nothing is configured. +/// +/// The standard variable's `http/protobuf` and `http/json` name an encoding as well as a transport, +/// and both select HTTP here. The encoding is fixed to binary protobuf at compile time, which every +/// OTLP/HTTP endpoint accepts, so honoring the transport is more useful than rejecting the value. +#[cfg(feature = "otel")] +fn parse_otel_protocol(configured: Option<&str>) -> Result { + match configured + .map(|value| value.trim().to_lowercase()) + .as_deref() + { + None | Some("grpc") => Ok(Protocol::Grpc), + Some("http" | "http/protobuf" | "http/json") => Ok(Protocol::Http), + Some(unknown) => Err(TelemetryError::UnknownProtocol(unknown.to_string())), + } +} + /// Turn a configured OTLP endpoint into an absolute URL the exporter can dial. /// /// `http://` and `https://` URLs are taken as given — TLS is what every hosted collector speaks. A @@ -789,10 +809,56 @@ mod tests { #[cfg(feature = "otel")] mod otel_tests { use super::super::{ - TelemetryError, env_names_a_service, http_traces_endpoint, normalize_otel_endpoint, - otel_filter_directives, + Protocol, TelemetryError, env_names_a_service, http_traces_endpoint, + normalize_otel_endpoint, otel_filter_directives, parse_otel_protocol, }; + /// Nothing configured is gRPC, the transport a collector reached at a bare `host:port` + /// speaks. + #[test] + fn no_configured_protocol_is_grpc() { + assert_eq!(parse_otel_protocol(None).unwrap(), Protocol::Grpc); + } + + /// The values the standard `OTEL_EXPORTER_OTLP_PROTOCOL` carries name an encoding along + /// with the transport. Both HTTP spellings select HTTP rather than being rejected, so a + /// deployment configured only through the standard variable exports over the transport it + /// asked for instead of silently falling back to gRPC. + #[test] + fn standard_protocol_values_select_a_transport() { + for value in ["grpc", "GRPC", " grpc "] { + assert_eq!( + parse_otel_protocol(Some(value)).unwrap(), + Protocol::Grpc, + "{value} should select gRPC" + ); + } + for value in [ + "http", + "http/protobuf", + "http/json", + "HTTP/protobuf", + " http ", + ] { + assert_eq!( + parse_otel_protocol(Some(value)).unwrap(), + Protocol::Http, + "{value} should select HTTP" + ); + } + } + + #[test] + fn rejects_an_unknown_protocol() { + for value in ["https", "http/proto", "tcp"] { + let result = parse_otel_protocol(Some(value)); + assert!( + matches!(result, Err(TelemetryError::UnknownProtocol(_))), + "{value} should be rejected, got {result:?}" + ); + } + } + /// A service the operator named through either variable is kept, and the app's own name is /// used only when neither does. Overriding a configured `service.name` would rename the /// deployment's traces out from under whoever set it. diff --git a/crates/oxen-cli/README.md b/crates/oxen-cli/README.md index 59da957df0..915107c658 100644 --- a/crates/oxen-cli/README.md +++ b/crates/oxen-cli/README.md @@ -60,7 +60,7 @@ to be exported. | Variable | Description | Default | |---|---|---| | `OXEN_OTEL_ENDPOINT` | Collector endpoint URL. Absent = disabled. | *(none)* | -| `OXEN_OTEL_PROTOCOL` | Transport: `grpc` or `http` (binary protobuf either way) | `grpc` | +| `OXEN_OTEL_PROTOCOL` | Transport: `grpc` or `http` (binary protobuf either way). Falls back to `OTEL_EXPORTER_OTLP_PROTOCOL` | `grpc` | | `RUST_LOG` | Must include `info` level for spans to be exported | `off` | ### Quick start with Jaeger diff --git a/crates/oxen-server/README.md b/crates/oxen-server/README.md index 4849212759..5f20d97195 100644 --- a/crates/oxen-server/README.md +++ b/crates/oxen-server/README.md @@ -245,8 +245,11 @@ store under both transports, so a collector behind a publicly trusted certificate needs no further configuration. A private CA has to be installed in that store. -The standard `OTEL_EXPORTER_OTLP_ENDPOINT` variable is also respected as a -fallback if `OXEN_OTEL_ENDPOINT` is not set. +The standard `OTEL_EXPORTER_OTLP_ENDPOINT` and `OTEL_EXPORTER_OTLP_PROTOCOL` +variables are also respected as fallbacks where `OXEN_OTEL_ENDPOINT` and +`OXEN_OTEL_PROTOCOL` are not set, so a vendor's stock configuration snippet +works as given. The protocol variable's `http/protobuf` and `http/json` both +select the HTTP transport; spans are encoded as binary protobuf either way. These standard `OTEL_*` variables are read by the SDK itself: @@ -257,6 +260,8 @@ These standard `OTEL_*` variables are read by the SDK itself: | `OTEL_TRACES_SAMPLER` | `always_on`, `always_off`, `traceidratio`, `parentbased_always_on`, `parentbased_always_off`, `parentbased_traceidratio`. | `parentbased_always_on` | | `OTEL_TRACES_SAMPLER_ARG` | Sampling probability, `0.0`–`1.0`, for the ratio samplers. | `1.0` | | `OTEL_BSP_MAX_QUEUE_SIZE`, `OTEL_BSP_SCHEDULE_DELAY`, `OTEL_BSP_MAX_EXPORT_BATCH_SIZE`, `OTEL_BSP_EXPORT_TIMEOUT` | Batch-processor tuning: queue depth, how often a batch drains, batch size, and how long the processor waits on one export. | `4096`, `2000` ms, `512`, `30000` ms | +| `OTEL_EXPORTER_OTLP_COMPRESSION` | `gzip` to compress export payloads, which is worth roughly 8x on a full batch of spans for a few milliseconds of CPU on the exporter's own thread. Only `gzip` is compiled in; any other value fails the exporter build, which disables export. Unset sends payloads uncompressed. | *(none)* | +| `OTEL_EXPORTER_OTLP_TRACES_COMPRESSION` | The same setting for span exports alone, and takes precedence over `OTEL_EXPORTER_OTLP_COMPRESSION` where both are set. | *(whatever `OTEL_EXPORTER_OTLP_COMPRESSION` resolves to)* | | `OTEL_EXPORTER_OTLP_TIMEOUT` | How long one export request to the collector may take, for every signal. Distinct from `OTEL_BSP_EXPORT_TIMEOUT` above, which bounds the batch processor rather than the request. | `10000` ms | | `OTEL_EXPORTER_OTLP_TRACES_TIMEOUT` | The same bound for span exports alone, and takes precedence over `OTEL_EXPORTER_OTLP_TIMEOUT` where both are set. | *(whatever `OTEL_EXPORTER_OTLP_TIMEOUT` resolves to)* | From db3857d2a562e87795d5ce694e58a0093a2c37ae Mon Sep 17 00:00:00 2001 From: Joshua Elliott Date: Wed, 12 Aug 2026 18:08:32 -0600 Subject: [PATCH 03/10] Resolve the OTLP endpoint and transport through the standard variable tiers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both settings are handed to the exporter builder rather than read by the SDK, so the SDK's own variable precedence never applied to them: a deployment configured with OTEL_EXPORTER_OTLP_TRACES_ENDPOINT alone exported nothing, and nothing said why. Each now resolves as OXEN_OTEL_*, then the traces-specific standard variable, then the general one, through a helper both sites share. A blank value names nothing and falls through rather than shadowing the next variable, which is what the two settings previously disagreed about — a blank endpoint failed startup where a blank protocol fell through. The OTLP/HTTP exporter names binary protobuf explicitly instead of inheriting whichever encoding the enabled cargo features imply, so a crate elsewhere in the graph enabling "http-json" can no longer turn every export into JSON without a compile error. The startup line names the encoding, because an operator who asks for http/json is given protobuf. --- crates/liboxen/src/util/telemetry.rs | 82 ++++++++++++++++++---------- crates/oxen-cli/README.md | 2 +- crates/oxen-server/README.md | 19 +++++-- 3 files changed, 67 insertions(+), 36 deletions(-) diff --git a/crates/liboxen/src/util/telemetry.rs b/crates/liboxen/src/util/telemetry.rs index fde483979c..425e21e2df 100644 --- a/crates/liboxen/src/util/telemetry.rs +++ b/crates/liboxen/src/util/telemetry.rs @@ -169,10 +169,12 @@ mod atexit_flush { /// the server README for configuration details. Also accepts the standard named /// env var `OTEL_EXPORTER_OTLP_ENDPOINT`, but checks `OXEN_OTEL_ENDPOINT` first. /// -/// The `OXEN_OTEL_PROTOCOL` env var is either `"http"` or `"grpc"`: it controls -/// the transport used for OTLP exports. If not set, the standard -/// `OTEL_EXPORTER_OTLP_PROTOCOL` is read instead, and absent both it defaults to -/// `"grpc"`. Spans are encoded as binary protobuf under either transport. +/// The `OXEN_OTEL_PROTOCOL` env var selects the transport OTLP exports use, and +/// accepts `"grpc"`, `"http"`, `"http/protobuf"`, or `"http/json"` — the last +/// three all meaning HTTP. If not set, the standard +/// `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL` and `OTEL_EXPORTER_OTLP_PROTOCOL` are +/// read in that order, and absent all three it defaults to `"grpc"`. Spans are +/// encoded as binary protobuf under either transport. /// /// **Filtering** is per-layer, not global. `RUST_LOG` (falling back to `default`) gates the log /// destinations — stderr, the JSON file, and the caller-supplied layer. Span export is gated @@ -257,10 +259,11 @@ pub fn init_tracing_with_layer( // concrete subscriber type (`S`) changes with each `.with()` call and // `OpenTelemetryLayer` must match the exact inner subscriber. #[cfg(feature = "otel")] - let (m_otel_layer, m_tracer_provider, m_endpoint_p) = match std::env::var("OXEN_OTEL_ENDPOINT") - .or_else(|_| std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT")) - .ok() - { + let (m_otel_layer, m_tracer_provider, m_endpoint_p) = match otel_env( + "OXEN_OTEL_ENDPOINT", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "OTEL_EXPORTER_OTLP_ENDPOINT", + ) { Some(endpoint) => { let endpoint = normalize_otel_endpoint(&endpoint)?; @@ -272,7 +275,7 @@ pub fn init_tracing_with_layer( ( Some(layer), Some(provider), - Some(format!("{protocol} -> {endpoint}")), + Some(format!("{protocol} (protobuf) -> {endpoint}")), ) } _ => (None, None, None), @@ -303,13 +306,15 @@ pub fn init_tracing_with_layer( { registry.try_init()?; - if std::env::var("OXEN_OTEL_ENDPOINT").is_ok() { - log::error!("OXEN_OTEL_ENDPOINT is set but otel feature is not enabled! (Ignoring)") - } - - if std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").is_ok() { + if otel_env( + "OXEN_OTEL_ENDPOINT", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "OTEL_EXPORTER_OTLP_ENDPOINT", + ) + .is_some() + { log::error!( - "OTEL_EXPORTER_OTLP_ENDPOINT is set but otel feature is not enabled! (Ignoring)" + "An OTLP endpoint is configured but the otel feature is not enabled! (Ignoring)" ) } } @@ -365,6 +370,22 @@ fn env_names_a_service( .any(|name| !name.trim().is_empty()) } +/// The first of three variables to name something: this project's own, then the traces-specific +/// standard variable, then the general standard one. +/// +/// That is the precedence the OpenTelemetry SDK applies to its own variables, but it applies it +/// only to settings it reads for itself. The endpoint and the transport are handed to the exporter +/// builder instead, which bypasses the SDK's lookup — so a value configured through a standard +/// variable reaches the exporter only by being read here. +/// +/// A blank value names nothing and falls through, so blanking a variable leaves the next one +/// standing rather than shadowing it with an empty string. +fn otel_env(oxen: &str, traces: &str, general: &str) -> Option { + non_empty_env(oxen) + .or_else(|| non_empty_env(traces)) + .or_else(|| non_empty_env(general)) +} + /// The value of `name`, or `None` when it is unset or blank. fn non_empty_env(name: &str) -> Option { std::env::var(name) @@ -471,20 +492,22 @@ impl std::fmt::Display for Protocol { } } -/// The transport OTLP export uses: `OXEN_OTEL_PROTOCOL`, or the standard -/// `OTEL_EXPORTER_OTLP_PROTOCOL` where that is unset. +/// The transport OTLP export uses, from the first of the protocol variables to name one. #[cfg(feature = "otel")] fn otel_protocol() -> Result { - let configured = non_empty_env("OXEN_OTEL_PROTOCOL") - .or_else(|| non_empty_env("OTEL_EXPORTER_OTLP_PROTOCOL")); + let configured = otel_env( + "OXEN_OTEL_PROTOCOL", + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", + "OTEL_EXPORTER_OTLP_PROTOCOL", + ); parse_otel_protocol(configured.as_deref()) } /// Read a configured protocol as a transport, defaulting to gRPC where nothing is configured. /// -/// The standard variable's `http/protobuf` and `http/json` name an encoding as well as a transport, -/// and both select HTTP here. The encoding is fixed to binary protobuf at compile time, which every -/// OTLP/HTTP endpoint accepts, so honoring the transport is more useful than rejecting the value. +/// The standard variables' `http/protobuf` and `http/json` name an encoding as well as a transport, +/// and both select HTTP. Spans are encoded as binary protobuf either way, which every OTLP/HTTP +/// endpoint accepts. #[cfg(feature = "otel")] fn parse_otel_protocol(configured: Option<&str>) -> Result { match configured @@ -587,8 +610,13 @@ where let exporter = match protocol { Protocol::Http => { + // Named rather than left to the exporter's default, which is whichever encoding the + // enabled cargo features imply: any crate in the graph turning on "http-json" would + // otherwise switch every export to JSON, silently and without a compile error. + // `opentelemetry_otlp::Protocol` is qualified to keep it apart from this module's own. match opentelemetry_otlp::SpanExporter::builder() .with_http() + .with_protocol(opentelemetry_otlp::Protocol::HttpBinary) .with_endpoint(http_traces_endpoint(endpoint)) .build() { @@ -826,13 +854,9 @@ mod tests { /// asked for instead of silently falling back to gRPC. #[test] fn standard_protocol_values_select_a_transport() { - for value in ["grpc", "GRPC", " grpc "] { - assert_eq!( - parse_otel_protocol(Some(value)).unwrap(), - Protocol::Grpc, - "{value} should select gRPC" - ); - } + assert_eq!(parse_otel_protocol(Some("grpc")).unwrap(), Protocol::Grpc); + // Case and surrounding whitespace are normalized before the match, so the HTTP + // spellings cover both for every value. for value in [ "http", "http/protobuf", diff --git a/crates/oxen-cli/README.md b/crates/oxen-cli/README.md index 915107c658..0b0b0dd410 100644 --- a/crates/oxen-cli/README.md +++ b/crates/oxen-cli/README.md @@ -60,7 +60,7 @@ to be exported. | Variable | Description | Default | |---|---|---| | `OXEN_OTEL_ENDPOINT` | Collector endpoint URL. Absent = disabled. | *(none)* | -| `OXEN_OTEL_PROTOCOL` | Transport: `grpc` or `http` (binary protobuf either way). Falls back to `OTEL_EXPORTER_OTLP_PROTOCOL` | `grpc` | +| `OXEN_OTEL_PROTOCOL` | Transport: `grpc`, or `http` / `http/protobuf` / `http/json` for HTTP (binary protobuf either way). Falls back to `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`, then `OTEL_EXPORTER_OTLP_PROTOCOL` | `grpc` | | `RUST_LOG` | Must include `info` level for spans to be exported | `off` | ### Quick start with Jaeger diff --git a/crates/oxen-server/README.md b/crates/oxen-server/README.md index 5f20d97195..9858871553 100644 --- a/crates/oxen-server/README.md +++ b/crates/oxen-server/README.md @@ -237,7 +237,7 @@ OXEN_OTEL_ENDPOINT=https://otlp.vendor.example:443 oxen-server start | Variable | Description | Default | |---|---|---| | `OXEN_OTEL_ENDPOINT` | Collector endpoint: an `http://` or `https://` URL, or a bare `host:port` (which gets `http://`). Absent = export disabled. | *(none)* | -| `OXEN_OTEL_PROTOCOL` | Transport: `grpc` or `http`. Under `http` the OTLP signal path `/v1/traces` is appended to the endpoint unless it already names one, and the payload is binary protobuf. | `grpc` | +| `OXEN_OTEL_PROTOCOL` | Transport: `grpc`, or `http` / `http/protobuf` / `http/json` for HTTP. Under HTTP the OTLP signal path `/v1/traces` is appended to the endpoint unless it already names one, and the payload is binary protobuf whichever of the three spellings is used. | `grpc` | | `OXEN_OTEL_FILTER` | Which spans and events are exported. Same syntax as `RUST_LOG`, and independent of it. | `info` | An `https://` endpoint is verified against the platform's root certificate @@ -245,11 +245,18 @@ store under both transports, so a collector behind a publicly trusted certificate needs no further configuration. A private CA has to be installed in that store. -The standard `OTEL_EXPORTER_OTLP_ENDPOINT` and `OTEL_EXPORTER_OTLP_PROTOCOL` -variables are also respected as fallbacks where `OXEN_OTEL_ENDPOINT` and -`OXEN_OTEL_PROTOCOL` are not set, so a vendor's stock configuration snippet -works as given. The protocol variable's `http/protobuf` and `http/json` both -select the HTTP transport; spans are encoded as binary protobuf either way. +The endpoint and the transport each fall back to their standard variables where +the `OXEN_` one is not set, so a vendor's stock configuration snippet works as +given. Each is resolved in the order `OXEN_OTEL_*`, then the traces-specific +standard variable, then the general one: + +| Setting | Resolved in this order | +|---|---| +| Endpoint | `OXEN_OTEL_ENDPOINT`, `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, `OTEL_EXPORTER_OTLP_ENDPOINT` | +| Transport | `OXEN_OTEL_PROTOCOL`, `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`, `OTEL_EXPORTER_OTLP_PROTOCOL` | + +A variable set to a blank value names nothing and falls through to the next, +rather than shadowing it. These standard `OTEL_*` variables are read by the SDK itself: From 482e4681cab3ee9814482e4703329b08c31c77fa Mon Sep 17 00:00:00 2001 From: Joshua Elliott Date: Wed, 12 Aug 2026 19:02:29 -0600 Subject: [PATCH 04/10] Post spans to a traces endpoint exactly as it is configured MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A traces endpoint names the signal, not the collector, so the OTLP signal path is not this crate's to append to it — appending one posts spans to a URL the operator never named, and the endpoint that most often carries a full path is the one copied from a vendor's own snippet. Only the two base-endpoint variables gain `/v1/traces` now, so `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` is dialed as given while `OXEN_OTEL_ENDPOINT` and `OTEL_EXPORTER_OTLP_ENDPOINT` still name the collector alone. Which variable supplied the endpoint decides this, so resolution keeps that distinction rather than reducing the three to one string. --- Cargo.toml | 7 ++- crates/liboxen/src/util/telemetry.rs | 91 +++++++++++++++++++++++----- crates/oxen-cli/README.md | 2 +- crates/oxen-server/README.md | 7 ++- 4 files changed, 86 insertions(+), 21 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0528e464b2..e33d7409a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -105,9 +105,10 @@ mockito = "1.1.0" mp4 = "0.14.0" num_cpus = "1.16.0" opentelemetry = "0.31" -# The HTTP path's encoding is chosen at compile time, not from the environment: the exporter never -# reads `OTEL_EXPORTER_OTLP_PROTOCOL`, and enabling "http-json" would make JSON the default encoding -# for every OTLP/HTTP export. Binary protobuf comes from the crate's default "http-proto". +# The protocol env vars select the transport; the OTLP/HTTP encoding is a compile-time choice this +# feature list makes. "http-proto" (a default feature) is what supplies binary protobuf, and adding +# "http-json" back would make JSON the encoding the exporter defaults to — which is why the builder +# names its encoding rather than inheriting that default. opentelemetry-otlp = { version = "0.31", features = ["grpc-tonic", "gzip-http", "gzip-tonic", "tls-roots", "trace"] } opentelemetry_sdk = { version = "0.31", features = ["rt-tokio"] } os_path = "0.8.0" diff --git a/crates/liboxen/src/util/telemetry.rs b/crates/liboxen/src/util/telemetry.rs index 425e21e2df..63b0d3a9ae 100644 --- a/crates/liboxen/src/util/telemetry.rs +++ b/crates/liboxen/src/util/telemetry.rs @@ -259,23 +259,20 @@ pub fn init_tracing_with_layer( // concrete subscriber type (`S`) changes with each `.with()` call and // `OpenTelemetryLayer` must match the exact inner subscriber. #[cfg(feature = "otel")] - let (m_otel_layer, m_tracer_provider, m_endpoint_p) = match otel_env( - "OXEN_OTEL_ENDPOINT", - "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", - "OTEL_EXPORTER_OTLP_ENDPOINT", - ) { - Some(endpoint) => { - let endpoint = normalize_otel_endpoint(&endpoint)?; + let (m_otel_layer, m_tracer_provider, m_endpoint_p) = match otel_endpoint() { + Some(mut endpoint) => { + endpoint.url = normalize_otel_endpoint(&endpoint.url)?; let protocol = otel_protocol()?; match build_otel_layer(app_name, &protocol, &endpoint) { (Some(layer), Some(provider)) => { atexit_flush::register(provider.clone()); + let url = &endpoint.url; ( Some(layer), Some(provider), - Some(format!("{protocol} (protobuf) -> {endpoint}")), + Some(format!("{protocol} (protobuf) -> {url}")), ) } _ => (None, None, None), @@ -370,13 +367,42 @@ fn env_names_a_service( .any(|name| !name.trim().is_empty()) } +/// An OTLP endpoint as configured, and whether it names the collector or the traces signal. +#[cfg(feature = "otel")] +struct OtlpEndpoint { + url: String, + /// Whether the OTLP signal path is this crate's to append. A base endpoint names the collector, + /// so `/v1/traces` is appended under HTTP; a signal-specific endpoint already names the signal + /// and is dialed exactly as configured, which is what the OpenTelemetry specification requires + /// of `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`. + is_base: bool, +} + +/// The OTLP endpoint, from the first of the endpoint variables to name one. +/// +/// Resolved here rather than through [`otel_env`] because the tier that matched decides whether the +/// signal path is appended, and that distinction is lost once the value is just a string. +#[cfg(feature = "otel")] +fn otel_endpoint() -> Option { + if let Some(url) = non_empty_env("OXEN_OTEL_ENDPOINT") { + return Some(OtlpEndpoint { url, is_base: true }); + } + if let Some(url) = non_empty_env("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") { + return Some(OtlpEndpoint { + url, + is_base: false, + }); + } + non_empty_env("OTEL_EXPORTER_OTLP_ENDPOINT").map(|url| OtlpEndpoint { url, is_base: true }) +} + /// The first of three variables to name something: this project's own, then the traces-specific /// standard variable, then the general standard one. /// /// That is the precedence the OpenTelemetry SDK applies to its own variables, but it applies it -/// only to settings it reads for itself. The endpoint and the transport are handed to the exporter -/// builder instead, which bypasses the SDK's lookup — so a value configured through a standard -/// variable reaches the exporter only by being read here. +/// only to settings it reads for itself. The transport is handed to the exporter builder instead, +/// which bypasses the SDK's lookup — so a value configured through a standard variable reaches the +/// exporter only by being read here. /// /// A blank value names nothing and falls through, so blanking a variable leaves the next one /// standing rather than shadowing it with an empty string. @@ -561,6 +587,17 @@ fn require_host(url: &str, configured: &str) -> Result<(), TelemetryError> { #[cfg(feature = "otel")] const OTLP_HTTP_TRACES_PATH: &str = "/v1/traces"; +/// The URL the OTLP/HTTP exporter posts spans to, which only a base endpoint has the signal path +/// appended to. See [`OtlpEndpoint::is_base`]. +#[cfg(feature = "otel")] +fn http_endpoint_url(endpoint: &OtlpEndpoint) -> String { + if endpoint.is_base { + http_traces_endpoint(&endpoint.url) + } else { + endpoint.url.clone() + } +} + /// The URL OTLP/HTTP posts spans to for a configured base endpoint. /// /// The exporter appends the signal path only to an endpoint it reads from the environment itself; @@ -590,7 +627,7 @@ fn http_traces_endpoint(endpoint: &str) -> String { fn build_otel_layer( app_name: &str, protocol: &Protocol, - endpoint: &str, + endpoint: &OtlpEndpoint, ) -> ( Option>, Option, @@ -617,7 +654,7 @@ where match opentelemetry_otlp::SpanExporter::builder() .with_http() .with_protocol(opentelemetry_otlp::Protocol::HttpBinary) - .with_endpoint(http_traces_endpoint(endpoint)) + .with_endpoint(http_endpoint_url(endpoint)) .build() { Ok(e) => e, @@ -632,7 +669,7 @@ where // against. An `http://` endpoint ignores it. match opentelemetry_otlp::SpanExporter::builder() .with_tonic() - .with_endpoint(endpoint) + .with_endpoint(&endpoint.url) .with_tls_config(ClientTlsConfig::new().with_native_roots()) .build() { @@ -837,8 +874,9 @@ mod tests { #[cfg(feature = "otel")] mod otel_tests { use super::super::{ - Protocol, TelemetryError, env_names_a_service, http_traces_endpoint, - normalize_otel_endpoint, otel_filter_directives, parse_otel_protocol, + OtlpEndpoint, Protocol, TelemetryError, env_names_a_service, http_endpoint_url, + http_traces_endpoint, normalize_otel_endpoint, otel_filter_directives, + parse_otel_protocol, }; /// Nothing configured is gRPC, the transport a collector reached at a bare `host:port` @@ -1023,6 +1061,27 @@ mod tests { ); } + /// Only a base endpoint names the collector, so only a base endpoint gets the signal path. + /// A traces endpoint is posted to as configured even when it names no path of its own — + /// appending one would send spans somewhere the operator never pointed us. + #[test] + fn only_a_base_endpoint_carries_the_signal_path() { + let url = "https://vendor.example/otlp"; + let base = OtlpEndpoint { + url: url.to_string(), + is_base: true, + }; + let traces = OtlpEndpoint { + url: url.to_string(), + is_base: false, + }; + assert_eq!( + http_endpoint_url(&base), + "https://vendor.example/otlp/v1/traces" + ); + assert_eq!(http_endpoint_url(&traces), url); + } + /// Span export must not be silenced by the stock log level: `#[tracing::instrument]` and /// the HTTP root span record at `INFO`, and the server logs at `WARN` by default. #[test] diff --git a/crates/oxen-cli/README.md b/crates/oxen-cli/README.md index 0b0b0dd410..5119b8cbb6 100644 --- a/crates/oxen-cli/README.md +++ b/crates/oxen-cli/README.md @@ -59,7 +59,7 @@ to be exported. | Variable | Description | Default | |---|---|---| -| `OXEN_OTEL_ENDPOINT` | Collector endpoint URL. Absent = disabled. | *(none)* | +| `OXEN_OTEL_ENDPOINT` | Collector endpoint URL. Falls back to `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` (posted to as-is under HTTP, so include `/v1/traces`), then `OTEL_EXPORTER_OTLP_ENDPOINT`. Absent from all three = disabled. | *(none)* | | `OXEN_OTEL_PROTOCOL` | Transport: `grpc`, or `http` / `http/protobuf` / `http/json` for HTTP (binary protobuf either way). Falls back to `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`, then `OTEL_EXPORTER_OTLP_PROTOCOL` | `grpc` | | `RUST_LOG` | Must include `info` level for spans to be exported | `off` | diff --git a/crates/oxen-server/README.md b/crates/oxen-server/README.md index 9858871553..28e1f020e3 100644 --- a/crates/oxen-server/README.md +++ b/crates/oxen-server/README.md @@ -236,7 +236,7 @@ OXEN_OTEL_ENDPOINT=https://otlp.vendor.example:443 oxen-server start | Variable | Description | Default | |---|---|---| -| `OXEN_OTEL_ENDPOINT` | Collector endpoint: an `http://` or `https://` URL, or a bare `host:port` (which gets `http://`). Absent = export disabled. | *(none)* | +| `OXEN_OTEL_ENDPOINT` | Collector endpoint: an `http://` or `https://` URL, or a bare `host:port` (which gets `http://`). Absent = export disabled, unless a standard endpoint variable below names one. | *(none)* | | `OXEN_OTEL_PROTOCOL` | Transport: `grpc`, or `http` / `http/protobuf` / `http/json` for HTTP. Under HTTP the OTLP signal path `/v1/traces` is appended to the endpoint unless it already names one, and the payload is binary protobuf whichever of the three spellings is used. | `grpc` | | `OXEN_OTEL_FILTER` | Which spans and events are exported. Same syntax as `RUST_LOG`, and independent of it. | `info` | @@ -258,6 +258,11 @@ standard variable, then the general one: A variable set to a blank value names nothing and falls through to the next, rather than shadowing it. +`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` names the traces signal rather than the +collector, so under HTTP it is posted to exactly as configured — include +`/v1/traces` in it. The other two name the collector, and `/v1/traces` is +appended to them. + These standard `OTEL_*` variables are read by the SDK itself: | Variable | Description | Default | From 159bb22b4f9ac4fae529287a86f59a22b5fccb6e Mon Sep 17 00:00:00 2001 From: Joshua Elliott Date: Wed, 12 Aug 2026 19:10:45 -0600 Subject: [PATCH 05/10] Correct the CLI's account of what gates span export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Span export is gated by OXEN_OTEL_FILTER, which defaults to info and is independent of RUST_LOG — the CLI exports spans with RUST_LOG unset, and logging nothing by default does not mean exporting nothing. The guide described the single-filter behavior that predates per-layer filtering, telling readers to set RUST_LOG=info for spans to leave the process at all. --- crates/oxen-cli/README.md | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/crates/oxen-cli/README.md b/crates/oxen-cli/README.md index 5119b8cbb6..3b6f563e14 100644 --- a/crates/oxen-cli/README.md +++ b/crates/oxen-cli/README.md @@ -39,29 +39,33 @@ The `oxen` CLI can export tracing spans to any OTLP-compatible collector cargo build -p oxen-cli --features otel ``` -At runtime, set **both** `OXEN_OTEL_ENDPOINT` and `RUST_LOG=info`: +At runtime, set `OXEN_OTEL_ENDPOINT`: ```bash -OXEN_OTEL_ENDPOINT=http://localhost:4317 RUST_LOG=info oxen pull +OXEN_OTEL_ENDPOINT=http://localhost:4317 oxen pull ``` -### Why `RUST_LOG=info` is required +### Filtering: logs and spans are separate -The CLI defaults to `LevelFilter::OFF` when `RUST_LOG` is not set. The -`RUST_LOG` filter is global — it gates what reaches **all** tracing outputs, -including the OpenTelemetry exporter. Since `#[tracing::instrument]` creates -spans at `INFO` level, they are silently dropped before the OTel layer ever -sees them unless the filter is at `INFO` or below. +`RUST_LOG` gates the CLI's log output. Span export is gated separately by +`OXEN_OTEL_FILTER`, which defaults to `info` — the level +`#[tracing::instrument]` records at. So spans export at the stock settings even +though the CLI logs nothing by default, and raising `RUST_LOG` for a debugging +session does not change what is exported. -Setting `OXEN_OTEL_ENDPOINT` alone is not enough — you must also set -`RUST_LOG=info` (or more targeted, e.g. `RUST_LOG=liboxen=info`) for spans -to be exported. +`OXEN_OTEL_FILTER` takes the same directive syntax as `RUST_LOG`, so export can +be narrowed or widened on its own: + +```bash +OXEN_OTEL_FILTER=warn,liboxen=debug OXEN_OTEL_ENDPOINT=http://localhost:4317 oxen pull +``` | Variable | Description | Default | |---|---|---| | `OXEN_OTEL_ENDPOINT` | Collector endpoint URL. Falls back to `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` (posted to as-is under HTTP, so include `/v1/traces`), then `OTEL_EXPORTER_OTLP_ENDPOINT`. Absent from all three = disabled. | *(none)* | | `OXEN_OTEL_PROTOCOL` | Transport: `grpc`, or `http` / `http/protobuf` / `http/json` for HTTP (binary protobuf either way). Falls back to `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`, then `OTEL_EXPORTER_OTLP_PROTOCOL` | `grpc` | -| `RUST_LOG` | Must include `info` level for spans to be exported | `off` | +| `OXEN_OTEL_FILTER` | Which spans and events are exported. Same syntax as `RUST_LOG`, and independent of it. | `info` | +| `RUST_LOG` | Log verbosity on stderr. Does not affect span export. | `off` | ### Quick start with Jaeger @@ -76,7 +80,7 @@ docker run --rm --name jaeger \ cr.jaegertracing.io/jaegertracing/jaeger:2.17.0 # Run a pull with tracing enabled -OXEN_OTEL_ENDPOINT=http://localhost:4317 RUST_LOG=info cargo run --features otel -p oxen-cli pull +OXEN_OTEL_ENDPOINT=http://localhost:4317 cargo run --features otel -p oxen-cli pull # View traces at http://localhost:16686 under service "oxen" ``` From 7d4e0812a2f07e4ece67772adf4d762853c5d1f7 Mon Sep 17 00:00:00 2001 From: Joshua Elliott Date: Wed, 12 Aug 2026 20:39:17 -0600 Subject: [PATCH 06/10] Name the endpoint rule the CLI guide only documented the exception to The guide said a traces endpoint is posted to as configured without saying what the other two endpoint variables do, leaving the exception documented and the rule implied. Both base-endpoint spellings now say that `/v1/traces` is appended to them under HTTP. --- crates/oxen-cli/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/oxen-cli/README.md b/crates/oxen-cli/README.md index 3b6f563e14..90f7e6104b 100644 --- a/crates/oxen-cli/README.md +++ b/crates/oxen-cli/README.md @@ -62,7 +62,7 @@ OXEN_OTEL_FILTER=warn,liboxen=debug OXEN_OTEL_ENDPOINT=http://localhost:4317 oxe | Variable | Description | Default | |---|---|---| -| `OXEN_OTEL_ENDPOINT` | Collector endpoint URL. Falls back to `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` (posted to as-is under HTTP, so include `/v1/traces`), then `OTEL_EXPORTER_OTLP_ENDPOINT`. Absent from all three = disabled. | *(none)* | +| `OXEN_OTEL_ENDPOINT` | Collector base URL, which under HTTP has `/v1/traces` appended to it. Falls back to `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` — a full traces URL, posted to exactly as configured, so include `/v1/traces` in it — then to `OTEL_EXPORTER_OTLP_ENDPOINT`, a collector base URL appended to like the first. Absent from all three = disabled. | *(none)* | | `OXEN_OTEL_PROTOCOL` | Transport: `grpc`, or `http` / `http/protobuf` / `http/json` for HTTP (binary protobuf either way). Falls back to `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`, then `OTEL_EXPORTER_OTLP_PROTOCOL` | `grpc` | | `OXEN_OTEL_FILTER` | Which spans and events are exported. Same syntax as `RUST_LOG`, and independent of it. | `info` | | `RUST_LOG` | Log verbosity on stderr. Does not affect span export. | `off` | From 7d328503afce373e50479405a896f3e67fa2fe33 Mon Sep 17 00:00:00 2001 From: Joshua Elliott Date: Thu, 13 Aug 2026 15:54:12 -0600 Subject: [PATCH 07/10] Run the production-feature unit tests in CI Tests behind a production-only feature gate were compiled by the production job's clippy step and run by nothing: the test suite builds with default features, so a `#[cfg(feature = "otel")]` test does not exist in the job that runs tests, and a wrong assertion in one could not fail CI. The production job is the only one that enables those features, and at roughly six minutes against a nine-minute critical path it has the room to run them. Restricted to lib unit tests needing no server, so the step costs about a minute. oxen-py is excluded because its test binary needs the Python runtime that a later step installs. --- .github/workflows/ci_build_production.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/ci_build_production.yml b/.github/workflows/ci_build_production.yml index 7095b35d0d..da3138e850 100644 --- a/.github/workflows/ci_build_production.yml +++ b/.github/workflows/ci_build_production.yml @@ -63,6 +63,20 @@ jobs: - name: Clippy (production features) run: cargo clippy --workspace --all-targets --features production --no-deps -- -D warnings + # Unit tests behind a production-only feature gate — `#[cfg(feature = "otel")]` and the + # like — are compiled by the clippy step above but run nowhere else: the Test Suite + # workflow builds with default features, so those tests do not exist in it. This job is + # the only one that enables them, and it has the schedule room to run them. + # + # The filter keeps the step to lib unit tests that need no server, which is what makes it + # cheap enough to live here. A feature-gated test whose path does not match the filter is + # still uncovered, so widen the filter when adding one elsewhere. + # + # oxen-py is excluded because its test binary needs the Python runtime that the maturin + # step below sets up, and it carries none of these tests. + - name: Test (production features) + run: cargo test --workspace --exclude oxen-py --features production --lib telemetry + - name: Build oxen-python with production features run: | cd oxen-python From 0be21a9b5e04ae7f32712af0fc375730508b1aa4 Mon Sep 17 00:00:00 2001 From: Joshua Elliott Date: Thu, 13 Aug 2026 16:17:22 -0600 Subject: [PATCH 08/10] Cover both OTLP transports in the telemetry smoke test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The server now exports over OTLP/HTTP while the CLI and the Python bindings stay on gRPC, so a single run exercises both transports and both wire formats against a real collector rather than leaving the HTTP path unexercised. The server is the one moved because it is the binary a deployment builds with the otel feature, and hosted collectors commonly accept OTLP/HTTP with binary protobuf and nothing else — the encoding a receiver actually parses is not something a unit test can check. Both collector ports were already published, so nothing else changes. --- bin/otel-metrics-test | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/bin/otel-metrics-test b/bin/otel-metrics-test index 13c6b1a201..5570d52b57 100755 --- a/bin/otel-metrics-test +++ b/bin/otel-metrics-test @@ -171,7 +171,15 @@ MODIFIED_AUTH_CONFIG=true # log destinations, so every trace asserted below arrives at the stock log level — which for the # CLI and the Python bindings is "off". Setting RUST_LOG=info here would hide a regression that # re-couples the two. -OXEN_OTEL_ENDPOINT=http://localhost:4317 \ +# +# The server exports over OTLP/HTTP while the CLI and the Python bindings below stay on gRPC, so +# one run covers both transports against a real collector. The server is the one on HTTP because +# that is the transport a deployed server uses: it is the binary built with the otel feature, and +# hosted collectors commonly accept OTLP/HTTP with binary protobuf and nothing else. Note that the +# trace assertions below read the server's spans, so a break in HTTP export surfaces here as +# missing or unparented spans rather than as an obvious export failure. +OXEN_OTEL_ENDPOINT=http://localhost:4318 \ +OXEN_OTEL_PROTOCOL=http \ OXEN_METRICS_PORT=9090 \ "$SERVER" start & SERVER_PID=$! From 1227ad93b77c301bfd912367c45523e26c7584fd Mon Sep 17 00:00:00 2001 From: Joshua Elliott Date: Thu, 13 Aug 2026 16:26:47 -0600 Subject: [PATCH 09/10] Make the telemetry smoke test runnable alongside a development server The script bound the default server port and wrote into whatever data directory the environment named, which locally is a developer's own server on both counts. A server already holding the port answers the readiness probe, so the workflow ran against a server with no telemetry configured and every assertion failed for a reason unrelated to telemetry; and the remote repository the workflow creates is deleted by nothing, so a second run failed on the first one's leftovers. The server port and the metrics port are now overridable, and the data directory is always a temporary one of the run's own, removed on exit. It overrides an inherited SYNC_DIR rather than honoring it, because the value a developer's shell exports points at the data this must not touch. --- bin/otel-metrics-test | 53 +++++++++++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/bin/otel-metrics-test b/bin/otel-metrics-test index 5570d52b57..41f4981653 100755 --- a/bin/otel-metrics-test +++ b/bin/otel-metrics-test @@ -9,8 +9,21 @@ # Prerequisites: # cargo build --workspace --features production # docker (running) -# uv, maturin (for oxen-python) +# uv, maturin (for oxen-python) — the bindings must be built with production features, or the +# oxen-python trace assertion fails while every other one passes # jq +# +# Environment: +# OXEN_SERVER_PORT port for the oxen-server this starts (default 3000) +# OXEN_METRICS_PORT port for its Prometheus endpoint (default 9090) +# +# SYNC_DIR is not read: this always runs against a temp data directory of its own, removed on exit. +# +# Set the ports to run alongside a development server holding the defaults: +# OXEN_SERVER_PORT=3010 OXEN_METRICS_PORT=9091 bin/otel-metrics-test +# +# Jaeger's ports (16686, 4317, 4318, 5778, 9411) are still fixed, so a container already +# publishing them has to be stopped first. set -euo pipefail @@ -22,6 +35,22 @@ SERVER="$REPO_ROOT/target/debug/oxen-server" SERVER_PID="" JAEGER_NAME="" +# Ports and data directory, overridable so a run can step around a development server already +# holding the defaults. On the defaults this run would bind nothing that machine does not expect +# to be free, but a developer's own oxen-server is commonly on 3000: its health endpoint answers +# the readiness probe below, and the workflow then runs against a server with no telemetry +# configured, failing every assertion here for a reason that has nothing to do with telemetry. +OXEN_SERVER_PORT="${OXEN_SERVER_PORT:-3000}" +OXEN_METRICS_PORT="${OXEN_METRICS_PORT:-9090}" +SERVER_URL="http://localhost:$OXEN_SERVER_PORT" + +# A data directory of this run's own, deliberately overriding any SYNC_DIR the environment +# supplies: a developer's shell commonly exports one for their own server, and the workflow below +# creates a remote repository that nothing deletes, so inheriting that value both writes into a +# data directory this has no business touching and leaves the next run to fail on the leftovers. +SYNC_DIR="$(mktemp -d)" +export SYNC_DIR + # Track which config files we modified so cleanup can restore/remove them MODIFIED_HOME_CONFIG=false HOME_CONFIG_BACKUP="" @@ -89,7 +118,7 @@ cleanup() { fi fi # Remove only the temp paths this run created. - rm -rf "$REPO_A_DIR" "$REPO_B_PARENT" "$CI_USER_TOML" + rm -rf "$REPO_A_DIR" "$REPO_B_PARENT" "$CI_USER_TOML" "$SYNC_DIR" } trap cleanup EXIT INT TERM @@ -163,7 +192,7 @@ if [ -f "$AUTH_CONFIG" ]; then AUTH_CONFIG_BACKUP=$(mktemp) cp "$AUTH_CONFIG" "$AUTH_CONFIG_BACKUP" fi -"$OXEN" config --auth localhost:3000 "$TOKEN" +"$OXEN" config --auth "localhost:$OXEN_SERVER_PORT" "$TOKEN" MODIFIED_AUTH_CONFIG=true # ── Start oxen-server with OTel export and Prometheus metrics ── @@ -180,8 +209,8 @@ MODIFIED_AUTH_CONFIG=true # missing or unparented spans rather than as an obvious export failure. OXEN_OTEL_ENDPOINT=http://localhost:4318 \ OXEN_OTEL_PROTOCOL=http \ -OXEN_METRICS_PORT=9090 \ -"$SERVER" start & +OXEN_METRICS_PORT="$OXEN_METRICS_PORT" \ +"$SERVER" start -p "$OXEN_SERVER_PORT" & SERVER_PID=$! # Wait for server to be ready (bounded: 30 attempts, 1s apart) @@ -192,7 +221,7 @@ for _i in $(seq 1 30); do echo "FAIL: oxen-server exited before becoming ready" exit 1 fi - if curl -sf --max-time 2 http://localhost:3000/api/health > /dev/null 2>&1; then + if curl -sf --max-time 2 "$SERVER_URL/api/health" > /dev/null 2>&1; then echo "oxen-server is ready" server_ready=true break @@ -213,7 +242,7 @@ TRACEPARENT_SPAN_ID="00f067aa0ba902b7" curl -sf --max-time 10 \ -H "Authorization: Bearer $TOKEN" \ -H "traceparent: 00-$TRACEPARENT_TRACE_ID-$TRACEPARENT_SPAN_ID-01" \ - http://localhost:3000/api/version > /dev/null + "$SERVER_URL/api/version" > /dev/null # ── oxen client: Full push / clone / modify / push / pull workflow ── @@ -221,12 +250,12 @@ curl -sf --max-time 10 \ curl -sf --max-time 30 -H "Authorization: Bearer $TOKEN" \ -H "Content-Type: application/json" \ -X POST -d '{"namespace":"ox","name":"otel-metrics-test"}' \ - http://localhost:3000/api/repos + "$SERVER_URL/api/repos" # Repo A: init, commit, push (REPO_A_DIR already created by mktemp -d) cd "$REPO_A_DIR" "$OXEN" init -"$OXEN" config --set-remote origin http://localhost:3000/ox/otel-metrics-test +"$OXEN" config --set-remote origin "$SERVER_URL/ox/otel-metrics-test" echo "hello from CI" > test-file.txt "$OXEN" add test-file.txt "$OXEN" commit -m "initial commit" @@ -235,7 +264,7 @@ echo "hello from CI" > test-file.txt # Clone to Repo B (oxen clone rejects absolute paths, so cd into a parent # tmpdir and clone using the relative "repo-b" name) cd "$REPO_B_PARENT" -"$OXEN" clone http://localhost:3000/ox/otel-metrics-test repo-b +"$OXEN" clone "$SERVER_URL/ox/otel-metrics-test" repo-b # Repo B: modify, commit, push cd "$REPO_B_DIR" @@ -343,9 +372,9 @@ fi # ── Verify Prometheus metrics ── -METRICS_OUTPUT=$(curl -sf --max-time 5 http://localhost:9090/metrics 2>/dev/null || true) +METRICS_OUTPUT=$(curl -sf --max-time 5 "http://localhost:$OXEN_METRICS_PORT/metrics" 2>/dev/null || true) if [ -z "$METRICS_OUTPUT" ]; then - echo "FAIL: Could not fetch /metrics from Prometheus endpoint on port 9090" + echo "FAIL: Could not fetch /metrics from Prometheus endpoint on port $OXEN_METRICS_PORT" did_fail=true else echo "--- Prometheus /metrics sample ---" From e047b1fabf0b4609ae81ccead0286a3d5b3a2bce Mon Sep 17 00:00:00 2001 From: Joshua Elliott Date: Thu, 13 Aug 2026 16:43:44 -0600 Subject: [PATCH 10/10] Tighten the comments added for OTLP transport selection Same content, fewer words: the comments explaining endpoint resolution, the compile-time encoding choice, the production-feature test step, and the smoke test's ports and data directory each said in several clauses what one says. Nine lines shorter overall, with no explanation dropped. --- .github/workflows/ci_build_production.yml | 17 +++++------- Cargo.toml | 7 +++-- bin/otel-metrics-test | 33 +++++++++++------------ crates/liboxen/src/util/telemetry.rs | 16 +++++------ crates/oxen-cli/README.md | 4 +-- crates/oxen-server/README.md | 2 +- 6 files changed, 35 insertions(+), 44 deletions(-) diff --git a/.github/workflows/ci_build_production.yml b/.github/workflows/ci_build_production.yml index da3138e850..5ca474814f 100644 --- a/.github/workflows/ci_build_production.yml +++ b/.github/workflows/ci_build_production.yml @@ -63,17 +63,12 @@ jobs: - name: Clippy (production features) run: cargo clippy --workspace --all-targets --features production --no-deps -- -D warnings - # Unit tests behind a production-only feature gate — `#[cfg(feature = "otel")]` and the - # like — are compiled by the clippy step above but run nowhere else: the Test Suite - # workflow builds with default features, so those tests do not exist in it. This job is - # the only one that enables them, and it has the schedule room to run them. - # - # The filter keeps the step to lib unit tests that need no server, which is what makes it - # cheap enough to live here. A feature-gated test whose path does not match the filter is - # still uncovered, so widen the filter when adding one elsewhere. - # - # oxen-py is excluded because its test binary needs the Python runtime that the maturin - # step below sets up, and it carries none of these tests. + # The Test Suite workflow builds with default features, so tests behind a production-only + # gate like `#[cfg(feature = "otel")]` never run there. This job is the only one that + # enables them. The filter limits the step to lib unit tests needing no server, so a + # feature-gated test whose path does not match it stays uncovered: widen the filter when + # adding one elsewhere. oxen-py is excluded because its test binary needs the Python + # runtime installed by the maturin step below. - name: Test (production features) run: cargo test --workspace --exclude oxen-py --features production --lib telemetry diff --git a/Cargo.toml b/Cargo.toml index e33d7409a4..133aed09b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -105,10 +105,9 @@ mockito = "1.1.0" mp4 = "0.14.0" num_cpus = "1.16.0" opentelemetry = "0.31" -# The protocol env vars select the transport; the OTLP/HTTP encoding is a compile-time choice this -# feature list makes. "http-proto" (a default feature) is what supplies binary protobuf, and adding -# "http-json" back would make JSON the encoding the exporter defaults to — which is why the builder -# names its encoding rather than inheriting that default. +# The OTLP/HTTP encoding is chosen by this feature list, not at runtime. Binary protobuf comes +# from the default "http-proto"; adding "http-json" back would make JSON the exporter's default +# encoding, which is why the builder names its encoding instead of inheriting it. opentelemetry-otlp = { version = "0.31", features = ["grpc-tonic", "gzip-http", "gzip-tonic", "tls-roots", "trace"] } opentelemetry_sdk = { version = "0.31", features = ["rt-tokio"] } os_path = "0.8.0" diff --git a/bin/otel-metrics-test b/bin/otel-metrics-test index 41f4981653..e3f25a1a04 100755 --- a/bin/otel-metrics-test +++ b/bin/otel-metrics-test @@ -9,21 +9,21 @@ # Prerequisites: # cargo build --workspace --features production # docker (running) -# uv, maturin (for oxen-python) — the bindings must be built with production features, or the -# oxen-python trace assertion fails while every other one passes +# uv, maturin (for oxen-python). The bindings must be built with production features, or the +# oxen-python trace assertion fails while every other one passes. # jq # # Environment: # OXEN_SERVER_PORT port for the oxen-server this starts (default 3000) # OXEN_METRICS_PORT port for its Prometheus endpoint (default 9090) # -# SYNC_DIR is not read: this always runs against a temp data directory of its own, removed on exit. +# SYNC_DIR is not read. This always uses a temp data directory of its own, removed on exit. # # Set the ports to run alongside a development server holding the defaults: # OXEN_SERVER_PORT=3010 OXEN_METRICS_PORT=9091 bin/otel-metrics-test # -# Jaeger's ports (16686, 4317, 4318, 5778, 9411) are still fixed, so a container already -# publishing them has to be stopped first. +# Jaeger's ports (16686, 4317, 4318, 5778, 9411) are still fixed, so stop any container +# already publishing them first. set -euo pipefail @@ -35,19 +35,16 @@ SERVER="$REPO_ROOT/target/debug/oxen-server" SERVER_PID="" JAEGER_NAME="" -# Ports and data directory, overridable so a run can step around a development server already -# holding the defaults. On the defaults this run would bind nothing that machine does not expect -# to be free, but a developer's own oxen-server is commonly on 3000: its health endpoint answers -# the readiness probe below, and the workflow then runs against a server with no telemetry -# configured, failing every assertion here for a reason that has nothing to do with telemetry. +# Overridable so a run can step around a development server already on these ports. A server +# holding 3000 answers the readiness probe below, and the workflow then runs against it with no +# telemetry configured, failing every assertion for a reason unrelated to telemetry. OXEN_SERVER_PORT="${OXEN_SERVER_PORT:-3000}" OXEN_METRICS_PORT="${OXEN_METRICS_PORT:-9090}" SERVER_URL="http://localhost:$OXEN_SERVER_PORT" -# A data directory of this run's own, deliberately overriding any SYNC_DIR the environment -# supplies: a developer's shell commonly exports one for their own server, and the workflow below -# creates a remote repository that nothing deletes, so inheriting that value both writes into a -# data directory this has no business touching and leaves the next run to fail on the leftovers. +# Overrides any SYNC_DIR in the environment, which a developer's shell commonly exports for their +# own server. The workflow below creates a remote repository that nothing deletes, so inheriting +# that value writes into data this must not touch and leaves the next run to fail on leftovers. SYNC_DIR="$(mktemp -d)" export SYNC_DIR @@ -203,10 +200,10 @@ MODIFIED_AUTH_CONFIG=true # # The server exports over OTLP/HTTP while the CLI and the Python bindings below stay on gRPC, so # one run covers both transports against a real collector. The server is the one on HTTP because -# that is the transport a deployed server uses: it is the binary built with the otel feature, and -# hosted collectors commonly accept OTLP/HTTP with binary protobuf and nothing else. Note that the -# trace assertions below read the server's spans, so a break in HTTP export surfaces here as -# missing or unparented spans rather than as an obvious export failure. +# that is what a deployment uses: it is the binary built with the otel feature, and hosted +# collectors commonly accept OTLP/HTTP with binary protobuf and nothing else. The trace +# assertions below read the server's spans, so a break in HTTP export shows up there as missing +# or unparented spans rather than as an obvious export failure. OXEN_OTEL_ENDPOINT=http://localhost:4318 \ OXEN_OTEL_PROTOCOL=http \ OXEN_METRICS_PORT="$OXEN_METRICS_PORT" \ diff --git a/crates/liboxen/src/util/telemetry.rs b/crates/liboxen/src/util/telemetry.rs index 63b0d3a9ae..21269bd0b1 100644 --- a/crates/liboxen/src/util/telemetry.rs +++ b/crates/liboxen/src/util/telemetry.rs @@ -170,7 +170,7 @@ mod atexit_flush { /// env var `OTEL_EXPORTER_OTLP_ENDPOINT`, but checks `OXEN_OTEL_ENDPOINT` first. /// /// The `OXEN_OTEL_PROTOCOL` env var selects the transport OTLP exports use, and -/// accepts `"grpc"`, `"http"`, `"http/protobuf"`, or `"http/json"` — the last +/// accepts `"grpc"`, `"http"`, `"http/protobuf"`, or `"http/json"`, the last /// three all meaning HTTP. If not set, the standard /// `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL` and `OTEL_EXPORTER_OTLP_PROTOCOL` are /// read in that order, and absent all three it defaults to `"grpc"`. Spans are @@ -372,7 +372,7 @@ fn env_names_a_service( struct OtlpEndpoint { url: String, /// Whether the OTLP signal path is this crate's to append. A base endpoint names the collector, - /// so `/v1/traces` is appended under HTTP; a signal-specific endpoint already names the signal + /// so `/v1/traces` is appended under HTTP. A signal-specific endpoint already names the signal /// and is dialed exactly as configured, which is what the OpenTelemetry specification requires /// of `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`. is_base: bool, @@ -399,10 +399,10 @@ fn otel_endpoint() -> Option { /// The first of three variables to name something: this project's own, then the traces-specific /// standard variable, then the general standard one. /// -/// That is the precedence the OpenTelemetry SDK applies to its own variables, but it applies it -/// only to settings it reads for itself. The transport is handed to the exporter builder instead, -/// which bypasses the SDK's lookup — so a value configured through a standard variable reaches the -/// exporter only by being read here. +/// That is the precedence the OpenTelemetry SDK applies to its own variables, but only to settings +/// it reads for itself. The transport is handed to the exporter builder instead, which bypasses +/// that lookup, so a value configured through a standard variable reaches the exporter only by +/// being read here. /// /// A blank value names nothing and falls through, so blanking a variable leaves the next one /// standing rather than shadowing it with an empty string. @@ -1062,8 +1062,8 @@ mod tests { } /// Only a base endpoint names the collector, so only a base endpoint gets the signal path. - /// A traces endpoint is posted to as configured even when it names no path of its own — - /// appending one would send spans somewhere the operator never pointed us. + /// A traces endpoint is posted to as configured even when it names no path of its own. + /// Appending one would send spans somewhere the operator never pointed us. #[test] fn only_a_base_endpoint_carries_the_signal_path() { let url = "https://vendor.example/otlp"; diff --git a/crates/oxen-cli/README.md b/crates/oxen-cli/README.md index 90f7e6104b..65bf691aaf 100644 --- a/crates/oxen-cli/README.md +++ b/crates/oxen-cli/README.md @@ -48,7 +48,7 @@ OXEN_OTEL_ENDPOINT=http://localhost:4317 oxen pull ### Filtering: logs and spans are separate `RUST_LOG` gates the CLI's log output. Span export is gated separately by -`OXEN_OTEL_FILTER`, which defaults to `info` — the level +`OXEN_OTEL_FILTER`, which defaults to `info`, the level `#[tracing::instrument]` records at. So spans export at the stock settings even though the CLI logs nothing by default, and raising `RUST_LOG` for a debugging session does not change what is exported. @@ -62,7 +62,7 @@ OXEN_OTEL_FILTER=warn,liboxen=debug OXEN_OTEL_ENDPOINT=http://localhost:4317 oxe | Variable | Description | Default | |---|---|---| -| `OXEN_OTEL_ENDPOINT` | Collector base URL, which under HTTP has `/v1/traces` appended to it. Falls back to `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` — a full traces URL, posted to exactly as configured, so include `/v1/traces` in it — then to `OTEL_EXPORTER_OTLP_ENDPOINT`, a collector base URL appended to like the first. Absent from all three = disabled. | *(none)* | +| `OXEN_OTEL_ENDPOINT` | Collector base URL, which under HTTP has `/v1/traces` appended to it. Falls back to `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` (a full traces URL, posted to exactly as configured, so include `/v1/traces` in it), then to `OTEL_EXPORTER_OTLP_ENDPOINT`, a collector base URL appended to like the first. Absent from all three = disabled. | *(none)* | | `OXEN_OTEL_PROTOCOL` | Transport: `grpc`, or `http` / `http/protobuf` / `http/json` for HTTP (binary protobuf either way). Falls back to `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`, then `OTEL_EXPORTER_OTLP_PROTOCOL` | `grpc` | | `OXEN_OTEL_FILTER` | Which spans and events are exported. Same syntax as `RUST_LOG`, and independent of it. | `info` | | `RUST_LOG` | Log verbosity on stderr. Does not affect span export. | `off` | diff --git a/crates/oxen-server/README.md b/crates/oxen-server/README.md index 28e1f020e3..85d9aa7a35 100644 --- a/crates/oxen-server/README.md +++ b/crates/oxen-server/README.md @@ -259,7 +259,7 @@ A variable set to a blank value names nothing and falls through to the next, rather than shadowing it. `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` names the traces signal rather than the -collector, so under HTTP it is posted to exactly as configured — include +collector, so under HTTP it is posted to exactly as configured. Include `/v1/traces` in it. The other two name the collector, and `/v1/traces` is appended to them.