diff --git a/.github/workflows/ci_build_production.yml b/.github/workflows/ci_build_production.yml index 7095b35d0..5ca474814 100644 --- a/.github/workflows/ci_build_production.yml +++ b/.github/workflows/ci_build_production.yml @@ -63,6 +63,15 @@ jobs: - name: Clippy (production features) run: cargo clippy --workspace --all-targets --features production --no-deps -- -D warnings + # The Test Suite workflow builds with default features, so tests behind a production-only + # gate like `#[cfg(feature = "otel")]` never run there. This job is the only one that + # enables them. The filter limits the step to lib unit tests needing no server, so a + # feature-gated test whose path does not match it stays uncovered: widen the filter when + # adding one elsewhere. oxen-py is excluded because its test binary needs the Python + # runtime installed by the maturin step below. + - name: Test (production features) + run: cargo test --workspace --exclude oxen-py --features production --lib telemetry + - name: Build oxen-python with production features run: | cd oxen-python diff --git a/Cargo.lock b/Cargo.lock index 30af9fde9..1f12828fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2331,18 +2331,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "const-hex" -version = "1.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "531185e432bb31db1ecda541e9e7ab21468d4d844ad7505e0546a49b4945d49b" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "proptest", - "serde_core", -] - [[package]] name = "const-oid" version = "0.9.6" @@ -5565,6 +5553,7 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" dependencies = [ + "flate2", "http 1.5.0", "opentelemetry", "opentelemetry-http", @@ -5572,7 +5561,6 @@ dependencies = [ "opentelemetry_sdk", "prost", "reqwest 0.12.28", - "serde_json", "thiserror 2.0.18", "tokio", "tonic", @@ -5585,13 +5573,9 @@ version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" dependencies = [ - "base64", - "const-hex", "opentelemetry", "opentelemetry_sdk", "prost", - "serde", - "serde_json", "tonic", "tonic-prost", ] @@ -6697,21 +6681,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "proptest" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" -dependencies = [ - "bitflags 2.11.0", - "num-traits", - "rand 0.9.2", - "rand_chacha 0.9.0", - "rand_xorshift", - "regex-syntax", - "unarray", -] - [[package]] name = "prost" version = "0.14.3" @@ -7101,15 +7070,6 @@ dependencies = [ "rand 0.8.5", ] -[[package]] -name = "rand_xorshift" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" -dependencies = [ - "rand_core 0.9.5", -] - [[package]] name = "rand_xoshiro" version = "0.7.0" @@ -9074,6 +9034,7 @@ dependencies = [ "async-trait", "base64", "bytes", + "flate2", "http 1.5.0", "http-body 1.0.1", "http-body-util", @@ -9316,12 +9277,6 @@ dependencies = [ "libc", ] -[[package]] -name = "unarray" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" - [[package]] name = "unicase" version = "2.9.0" diff --git a/Cargo.toml b/Cargo.toml index 00cba9e5f..133aed09b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -105,12 +105,10 @@ mockito = "1.1.0" mp4 = "0.14.0" num_cpus = "1.16.0" opentelemetry = "0.31" -opentelemetry-otlp = { version = "0.31", features = [ - "grpc-tonic", - "http-json", - "tls-roots", - "trace", -] } +# The OTLP/HTTP encoding is chosen by this feature list, not at runtime. Binary protobuf comes +# from the default "http-proto"; adding "http-json" back would make JSON the exporter's default +# encoding, which is why the builder names its encoding instead of inheriting it. +opentelemetry-otlp = { version = "0.31", features = ["grpc-tonic", "gzip-http", "gzip-tonic", "tls-roots", "trace"] } opentelemetry_sdk = { version = "0.31", features = ["rt-tokio"] } os_path = "0.8.0" page_size = "0.6.0" diff --git a/bin/otel-metrics-test b/bin/otel-metrics-test index 13c6b1a20..e3f25a1a0 100755 --- a/bin/otel-metrics-test +++ b/bin/otel-metrics-test @@ -9,8 +9,21 @@ # Prerequisites: # cargo build --workspace --features production # docker (running) -# uv, maturin (for oxen-python) +# uv, maturin (for oxen-python). The bindings must be built with production features, or the +# oxen-python trace assertion fails while every other one passes. # jq +# +# Environment: +# OXEN_SERVER_PORT port for the oxen-server this starts (default 3000) +# OXEN_METRICS_PORT port for its Prometheus endpoint (default 9090) +# +# SYNC_DIR is not read. This always uses a temp data directory of its own, removed on exit. +# +# Set the ports to run alongside a development server holding the defaults: +# OXEN_SERVER_PORT=3010 OXEN_METRICS_PORT=9091 bin/otel-metrics-test +# +# Jaeger's ports (16686, 4317, 4318, 5778, 9411) are still fixed, so stop any container +# already publishing them first. set -euo pipefail @@ -22,6 +35,19 @@ SERVER="$REPO_ROOT/target/debug/oxen-server" SERVER_PID="" JAEGER_NAME="" +# Overridable so a run can step around a development server already on these ports. A server +# holding 3000 answers the readiness probe below, and the workflow then runs against it with no +# telemetry configured, failing every assertion for a reason unrelated to telemetry. +OXEN_SERVER_PORT="${OXEN_SERVER_PORT:-3000}" +OXEN_METRICS_PORT="${OXEN_METRICS_PORT:-9090}" +SERVER_URL="http://localhost:$OXEN_SERVER_PORT" + +# Overrides any SYNC_DIR in the environment, which a developer's shell commonly exports for their +# own server. The workflow below creates a remote repository that nothing deletes, so inheriting +# that value writes into data this must not touch and leaves the next run to fail on leftovers. +SYNC_DIR="$(mktemp -d)" +export SYNC_DIR + # Track which config files we modified so cleanup can restore/remove them MODIFIED_HOME_CONFIG=false HOME_CONFIG_BACKUP="" @@ -89,7 +115,7 @@ cleanup() { fi fi # Remove only the temp paths this run created. - rm -rf "$REPO_A_DIR" "$REPO_B_PARENT" "$CI_USER_TOML" + rm -rf "$REPO_A_DIR" "$REPO_B_PARENT" "$CI_USER_TOML" "$SYNC_DIR" } trap cleanup EXIT INT TERM @@ -163,7 +189,7 @@ if [ -f "$AUTH_CONFIG" ]; then AUTH_CONFIG_BACKUP=$(mktemp) cp "$AUTH_CONFIG" "$AUTH_CONFIG_BACKUP" fi -"$OXEN" config --auth localhost:3000 "$TOKEN" +"$OXEN" config --auth "localhost:$OXEN_SERVER_PORT" "$TOKEN" MODIFIED_AUTH_CONFIG=true # ── Start oxen-server with OTel export and Prometheus metrics ── @@ -171,9 +197,17 @@ MODIFIED_AUTH_CONFIG=true # log destinations, so every trace asserted below arrives at the stock log level — which for the # CLI and the Python bindings is "off". Setting RUST_LOG=info here would hide a regression that # re-couples the two. -OXEN_OTEL_ENDPOINT=http://localhost:4317 \ -OXEN_METRICS_PORT=9090 \ -"$SERVER" start & +# +# The server exports over OTLP/HTTP while the CLI and the Python bindings below stay on gRPC, so +# one run covers both transports against a real collector. The server is the one on HTTP because +# that is what a deployment uses: it is the binary built with the otel feature, and hosted +# collectors commonly accept OTLP/HTTP with binary protobuf and nothing else. The trace +# assertions below read the server's spans, so a break in HTTP export shows up there as missing +# or unparented spans rather than as an obvious export failure. +OXEN_OTEL_ENDPOINT=http://localhost:4318 \ +OXEN_OTEL_PROTOCOL=http \ +OXEN_METRICS_PORT="$OXEN_METRICS_PORT" \ +"$SERVER" start -p "$OXEN_SERVER_PORT" & SERVER_PID=$! # Wait for server to be ready (bounded: 30 attempts, 1s apart) @@ -184,7 +218,7 @@ for _i in $(seq 1 30); do echo "FAIL: oxen-server exited before becoming ready" exit 1 fi - if curl -sf --max-time 2 http://localhost:3000/api/health > /dev/null 2>&1; then + if curl -sf --max-time 2 "$SERVER_URL/api/health" > /dev/null 2>&1; then echo "oxen-server is ready" server_ready=true break @@ -205,7 +239,7 @@ TRACEPARENT_SPAN_ID="00f067aa0ba902b7" curl -sf --max-time 10 \ -H "Authorization: Bearer $TOKEN" \ -H "traceparent: 00-$TRACEPARENT_TRACE_ID-$TRACEPARENT_SPAN_ID-01" \ - http://localhost:3000/api/version > /dev/null + "$SERVER_URL/api/version" > /dev/null # ── oxen client: Full push / clone / modify / push / pull workflow ── @@ -213,12 +247,12 @@ curl -sf --max-time 10 \ curl -sf --max-time 30 -H "Authorization: Bearer $TOKEN" \ -H "Content-Type: application/json" \ -X POST -d '{"namespace":"ox","name":"otel-metrics-test"}' \ - http://localhost:3000/api/repos + "$SERVER_URL/api/repos" # Repo A: init, commit, push (REPO_A_DIR already created by mktemp -d) cd "$REPO_A_DIR" "$OXEN" init -"$OXEN" config --set-remote origin http://localhost:3000/ox/otel-metrics-test +"$OXEN" config --set-remote origin "$SERVER_URL/ox/otel-metrics-test" echo "hello from CI" > test-file.txt "$OXEN" add test-file.txt "$OXEN" commit -m "initial commit" @@ -227,7 +261,7 @@ echo "hello from CI" > test-file.txt # Clone to Repo B (oxen clone rejects absolute paths, so cd into a parent # tmpdir and clone using the relative "repo-b" name) cd "$REPO_B_PARENT" -"$OXEN" clone http://localhost:3000/ox/otel-metrics-test repo-b +"$OXEN" clone "$SERVER_URL/ox/otel-metrics-test" repo-b # Repo B: modify, commit, push cd "$REPO_B_DIR" @@ -335,9 +369,9 @@ fi # ── Verify Prometheus metrics ── -METRICS_OUTPUT=$(curl -sf --max-time 5 http://localhost:9090/metrics 2>/dev/null || true) +METRICS_OUTPUT=$(curl -sf --max-time 5 "http://localhost:$OXEN_METRICS_PORT/metrics" 2>/dev/null || true) if [ -z "$METRICS_OUTPUT" ]; then - echo "FAIL: Could not fetch /metrics from Prometheus endpoint on port 9090" + echo "FAIL: Could not fetch /metrics from Prometheus endpoint on port $OXEN_METRICS_PORT" did_fail=true else echo "--- Prometheus /metrics sample ---" diff --git a/crates/liboxen/src/util/telemetry.rs b/crates/liboxen/src/util/telemetry.rs index 6b46701d1..21269bd0b 100644 --- a/crates/liboxen/src/util/telemetry.rs +++ b/crates/liboxen/src/util/telemetry.rs @@ -22,7 +22,9 @@ pub enum TelemetryError { #[error("Failed to initialize tracing: {0}")] InitFail(#[from] TryInitError), #[cfg(feature = "otel")] - #[error("Unknown OXEN_OTEL_PROTOCOL value: {0}")] + #[error( + "Unknown OXEN_OTEL_PROTOCOL / OTEL_EXPORTER_OTLP_PROTOCOL value: {0} (expected grpc, http, http/protobuf, or http/json)" + )] UnknownProtocol(String), #[cfg(feature = "otel")] #[error( @@ -167,8 +169,12 @@ mod atexit_flush { /// the server README for configuration details. Also accepts the standard named /// env var `OTEL_EXPORTER_OTLP_ENDPOINT`, but checks `OXEN_OTEL_ENDPOINT` first. /// -/// The `OXEN_OTEL_PROTOCOL` env var is either `"http"` or `"grpc"`: it controls -/// the protocol used for OTLP exports. If not set, defaults to `"grpc"`. +/// The `OXEN_OTEL_PROTOCOL` env var selects the transport OTLP exports use, and +/// accepts `"grpc"`, `"http"`, `"http/protobuf"`, or `"http/json"`, the last +/// three all meaning HTTP. If not set, the standard +/// `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL` and `OTEL_EXPORTER_OTLP_PROTOCOL` are +/// read in that order, and absent all three it defaults to `"grpc"`. Spans are +/// encoded as binary protobuf under either transport. /// /// **Filtering** is per-layer, not global. `RUST_LOG` (falling back to `default`) gates the log /// destinations — stderr, the JSON file, and the caller-supplied layer. Span export is gated @@ -253,32 +259,20 @@ pub fn init_tracing_with_layer( // concrete subscriber type (`S`) changes with each `.with()` call and // `OpenTelemetryLayer` must match the exact inner subscriber. #[cfg(feature = "otel")] - let (m_otel_layer, m_tracer_provider, m_endpoint_p) = match std::env::var("OXEN_OTEL_ENDPOINT") - .or_else(|_| std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT")) - .ok() - { - Some(endpoint) => { - let endpoint = normalize_otel_endpoint(&endpoint)?; + let (m_otel_layer, m_tracer_provider, m_endpoint_p) = match otel_endpoint() { + Some(mut endpoint) => { + endpoint.url = normalize_otel_endpoint(&endpoint.url)?; - let protocol = match std::env::var("OXEN_OTEL_PROTOCOL") - .map(|x| x.to_lowercase()) - .ok() - .as_deref() - { - Some("http") => Protocol::Http, - Some("grpc") | None => Protocol::Grpc, - Some(unknown) => { - return Err(TelemetryError::UnknownProtocol(unknown.to_string())); - } - }; + let protocol = otel_protocol()?; match build_otel_layer(app_name, &protocol, &endpoint) { (Some(layer), Some(provider)) => { atexit_flush::register(provider.clone()); + let url = &endpoint.url; ( Some(layer), Some(provider), - Some(format!("{protocol} -> {endpoint}")), + Some(format!("{protocol} (protobuf) -> {url}")), ) } _ => (None, None, None), @@ -309,13 +303,15 @@ pub fn init_tracing_with_layer( { registry.try_init()?; - if std::env::var("OXEN_OTEL_ENDPOINT").is_ok() { - log::error!("OXEN_OTEL_ENDPOINT is set but otel feature is not enabled! (Ignoring)") - } - - if std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").is_ok() { + if otel_env( + "OXEN_OTEL_ENDPOINT", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "OTEL_EXPORTER_OTLP_ENDPOINT", + ) + .is_some() + { log::error!( - "OTEL_EXPORTER_OTLP_ENDPOINT is set but otel feature is not enabled! (Ignoring)" + "An OTLP endpoint is configured but the otel feature is not enabled! (Ignoring)" ) } } @@ -371,6 +367,51 @@ fn env_names_a_service( .any(|name| !name.trim().is_empty()) } +/// An OTLP endpoint as configured, and whether it names the collector or the traces signal. +#[cfg(feature = "otel")] +struct OtlpEndpoint { + url: String, + /// Whether the OTLP signal path is this crate's to append. A base endpoint names the collector, + /// so `/v1/traces` is appended under HTTP. A signal-specific endpoint already names the signal + /// and is dialed exactly as configured, which is what the OpenTelemetry specification requires + /// of `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`. + is_base: bool, +} + +/// The OTLP endpoint, from the first of the endpoint variables to name one. +/// +/// Resolved here rather than through [`otel_env`] because the tier that matched decides whether the +/// signal path is appended, and that distinction is lost once the value is just a string. +#[cfg(feature = "otel")] +fn otel_endpoint() -> Option { + if let Some(url) = non_empty_env("OXEN_OTEL_ENDPOINT") { + return Some(OtlpEndpoint { url, is_base: true }); + } + if let Some(url) = non_empty_env("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") { + return Some(OtlpEndpoint { + url, + is_base: false, + }); + } + non_empty_env("OTEL_EXPORTER_OTLP_ENDPOINT").map(|url| OtlpEndpoint { url, is_base: true }) +} + +/// The first of three variables to name something: this project's own, then the traces-specific +/// standard variable, then the general standard one. +/// +/// That is the precedence the OpenTelemetry SDK applies to its own variables, but only to settings +/// it reads for itself. The transport is handed to the exporter builder instead, which bypasses +/// that lookup, so a value configured through a standard variable reaches the exporter only by +/// being read here. +/// +/// A blank value names nothing and falls through, so blanking a variable leaves the next one +/// standing rather than shadowing it with an empty string. +fn otel_env(oxen: &str, traces: &str, general: &str) -> Option { + non_empty_env(oxen) + .or_else(|| non_empty_env(traces)) + .or_else(|| non_empty_env(general)) +} + /// The value of `name`, or `None` when it is unset or blank. fn non_empty_env(name: &str) -> Option { std::env::var(name) @@ -477,6 +518,34 @@ impl std::fmt::Display for Protocol { } } +/// The transport OTLP export uses, from the first of the protocol variables to name one. +#[cfg(feature = "otel")] +fn otel_protocol() -> Result { + let configured = otel_env( + "OXEN_OTEL_PROTOCOL", + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", + "OTEL_EXPORTER_OTLP_PROTOCOL", + ); + parse_otel_protocol(configured.as_deref()) +} + +/// Read a configured protocol as a transport, defaulting to gRPC where nothing is configured. +/// +/// The standard variables' `http/protobuf` and `http/json` name an encoding as well as a transport, +/// and both select HTTP. Spans are encoded as binary protobuf either way, which every OTLP/HTTP +/// endpoint accepts. +#[cfg(feature = "otel")] +fn parse_otel_protocol(configured: Option<&str>) -> Result { + match configured + .map(|value| value.trim().to_lowercase()) + .as_deref() + { + None | Some("grpc") => Ok(Protocol::Grpc), + Some("http" | "http/protobuf" | "http/json") => Ok(Protocol::Http), + Some(unknown) => Err(TelemetryError::UnknownProtocol(unknown.to_string())), + } +} + /// Turn a configured OTLP endpoint into an absolute URL the exporter can dial. /// /// `http://` and `https://` URLs are taken as given — TLS is what every hosted collector speaks. A @@ -518,6 +587,17 @@ fn require_host(url: &str, configured: &str) -> Result<(), TelemetryError> { #[cfg(feature = "otel")] const OTLP_HTTP_TRACES_PATH: &str = "/v1/traces"; +/// The URL the OTLP/HTTP exporter posts spans to, which only a base endpoint has the signal path +/// appended to. See [`OtlpEndpoint::is_base`]. +#[cfg(feature = "otel")] +fn http_endpoint_url(endpoint: &OtlpEndpoint) -> String { + if endpoint.is_base { + http_traces_endpoint(&endpoint.url) + } else { + endpoint.url.clone() + } +} + /// The URL OTLP/HTTP posts spans to for a configured base endpoint. /// /// The exporter appends the signal path only to an endpoint it reads from the environment itself; @@ -547,7 +627,7 @@ fn http_traces_endpoint(endpoint: &str) -> String { fn build_otel_layer( app_name: &str, protocol: &Protocol, - endpoint: &str, + endpoint: &OtlpEndpoint, ) -> ( Option>, Option, @@ -567,9 +647,14 @@ where let exporter = match protocol { Protocol::Http => { + // Named rather than left to the exporter's default, which is whichever encoding the + // enabled cargo features imply: any crate in the graph turning on "http-json" would + // otherwise switch every export to JSON, silently and without a compile error. + // `opentelemetry_otlp::Protocol` is qualified to keep it apart from this module's own. match opentelemetry_otlp::SpanExporter::builder() .with_http() - .with_endpoint(http_traces_endpoint(endpoint)) + .with_protocol(opentelemetry_otlp::Protocol::HttpBinary) + .with_endpoint(http_endpoint_url(endpoint)) .build() { Ok(e) => e, @@ -584,7 +669,7 @@ where // against. An `http://` endpoint ignores it. match opentelemetry_otlp::SpanExporter::builder() .with_tonic() - .with_endpoint(endpoint) + .with_endpoint(&endpoint.url) .with_tls_config(ClientTlsConfig::new().with_native_roots()) .build() { @@ -789,10 +874,53 @@ mod tests { #[cfg(feature = "otel")] mod otel_tests { use super::super::{ - TelemetryError, env_names_a_service, http_traces_endpoint, normalize_otel_endpoint, - otel_filter_directives, + OtlpEndpoint, Protocol, TelemetryError, env_names_a_service, http_endpoint_url, + http_traces_endpoint, normalize_otel_endpoint, otel_filter_directives, + parse_otel_protocol, }; + /// Nothing configured is gRPC, the transport a collector reached at a bare `host:port` + /// speaks. + #[test] + fn no_configured_protocol_is_grpc() { + assert_eq!(parse_otel_protocol(None).unwrap(), Protocol::Grpc); + } + + /// The values the standard `OTEL_EXPORTER_OTLP_PROTOCOL` carries name an encoding along + /// with the transport. Both HTTP spellings select HTTP rather than being rejected, so a + /// deployment configured only through the standard variable exports over the transport it + /// asked for instead of silently falling back to gRPC. + #[test] + fn standard_protocol_values_select_a_transport() { + assert_eq!(parse_otel_protocol(Some("grpc")).unwrap(), Protocol::Grpc); + // Case and surrounding whitespace are normalized before the match, so the HTTP + // spellings cover both for every value. + for value in [ + "http", + "http/protobuf", + "http/json", + "HTTP/protobuf", + " http ", + ] { + assert_eq!( + parse_otel_protocol(Some(value)).unwrap(), + Protocol::Http, + "{value} should select HTTP" + ); + } + } + + #[test] + fn rejects_an_unknown_protocol() { + for value in ["https", "http/proto", "tcp"] { + let result = parse_otel_protocol(Some(value)); + assert!( + matches!(result, Err(TelemetryError::UnknownProtocol(_))), + "{value} should be rejected, got {result:?}" + ); + } + } + /// A service the operator named through either variable is kept, and the app's own name is /// used only when neither does. Overriding a configured `service.name` would rename the /// deployment's traces out from under whoever set it. @@ -933,6 +1061,27 @@ mod tests { ); } + /// Only a base endpoint names the collector, so only a base endpoint gets the signal path. + /// A traces endpoint is posted to as configured even when it names no path of its own. + /// Appending one would send spans somewhere the operator never pointed us. + #[test] + fn only_a_base_endpoint_carries_the_signal_path() { + let url = "https://vendor.example/otlp"; + let base = OtlpEndpoint { + url: url.to_string(), + is_base: true, + }; + let traces = OtlpEndpoint { + url: url.to_string(), + is_base: false, + }; + assert_eq!( + http_endpoint_url(&base), + "https://vendor.example/otlp/v1/traces" + ); + assert_eq!(http_endpoint_url(&traces), url); + } + /// Span export must not be silenced by the stock log level: `#[tracing::instrument]` and /// the HTTP root span record at `INFO`, and the server logs at `WARN` by default. #[test] diff --git a/crates/oxen-cli/README.md b/crates/oxen-cli/README.md index 586e9d5e8..65bf691aa 100644 --- a/crates/oxen-cli/README.md +++ b/crates/oxen-cli/README.md @@ -39,29 +39,33 @@ The `oxen` CLI can export tracing spans to any OTLP-compatible collector cargo build -p oxen-cli --features otel ``` -At runtime, set **both** `OXEN_OTEL_ENDPOINT` and `RUST_LOG=info`: +At runtime, set `OXEN_OTEL_ENDPOINT`: ```bash -OXEN_OTEL_ENDPOINT=http://localhost:4317 RUST_LOG=info oxen pull +OXEN_OTEL_ENDPOINT=http://localhost:4317 oxen pull ``` -### Why `RUST_LOG=info` is required +### Filtering: logs and spans are separate -The CLI defaults to `LevelFilter::OFF` when `RUST_LOG` is not set. The -`RUST_LOG` filter is global — it gates what reaches **all** tracing outputs, -including the OpenTelemetry exporter. Since `#[tracing::instrument]` creates -spans at `INFO` level, they are silently dropped before the OTel layer ever -sees them unless the filter is at `INFO` or below. +`RUST_LOG` gates the CLI's log output. Span export is gated separately by +`OXEN_OTEL_FILTER`, which defaults to `info`, the level +`#[tracing::instrument]` records at. So spans export at the stock settings even +though the CLI logs nothing by default, and raising `RUST_LOG` for a debugging +session does not change what is exported. -Setting `OXEN_OTEL_ENDPOINT` alone is not enough — you must also set -`RUST_LOG=info` (or more targeted, e.g. `RUST_LOG=liboxen=info`) for spans -to be exported. +`OXEN_OTEL_FILTER` takes the same directive syntax as `RUST_LOG`, so export can +be narrowed or widened on its own: + +```bash +OXEN_OTEL_FILTER=warn,liboxen=debug OXEN_OTEL_ENDPOINT=http://localhost:4317 oxen pull +``` | Variable | Description | Default | |---|---|---| -| `OXEN_OTEL_ENDPOINT` | Collector endpoint URL. Absent = disabled. | *(none)* | -| `OXEN_OTEL_PROTOCOL` | Transport: `grpc` or `http` | `grpc` | -| `RUST_LOG` | Must include `info` level for spans to be exported | `off` | +| `OXEN_OTEL_ENDPOINT` | Collector base URL, which under HTTP has `/v1/traces` appended to it. Falls back to `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` (a full traces URL, posted to exactly as configured, so include `/v1/traces` in it), then to `OTEL_EXPORTER_OTLP_ENDPOINT`, a collector base URL appended to like the first. Absent from all three = disabled. | *(none)* | +| `OXEN_OTEL_PROTOCOL` | Transport: `grpc`, or `http` / `http/protobuf` / `http/json` for HTTP (binary protobuf either way). Falls back to `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`, then `OTEL_EXPORTER_OTLP_PROTOCOL` | `grpc` | +| `OXEN_OTEL_FILTER` | Which spans and events are exported. Same syntax as `RUST_LOG`, and independent of it. | `info` | +| `RUST_LOG` | Log verbosity on stderr. Does not affect span export. | `off` | ### Quick start with Jaeger @@ -76,7 +80,7 @@ docker run --rm --name jaeger \ cr.jaegertracing.io/jaegertracing/jaeger:2.17.0 # Run a pull with tracing enabled -OXEN_OTEL_ENDPOINT=http://localhost:4317 RUST_LOG=info cargo run --features otel -p oxen-cli pull +OXEN_OTEL_ENDPOINT=http://localhost:4317 cargo run --features otel -p oxen-cli pull # View traces at http://localhost:16686 under service "oxen" ``` diff --git a/crates/oxen-server/README.md b/crates/oxen-server/README.md index f51622dee..85d9aa7a3 100644 --- a/crates/oxen-server/README.md +++ b/crates/oxen-server/README.md @@ -236,8 +236,8 @@ OXEN_OTEL_ENDPOINT=https://otlp.vendor.example:443 oxen-server start | Variable | Description | Default | |---|---|---| -| `OXEN_OTEL_ENDPOINT` | Collector endpoint: an `http://` or `https://` URL, or a bare `host:port` (which gets `http://`). Absent = export disabled. | *(none)* | -| `OXEN_OTEL_PROTOCOL` | Transport: `grpc` or `http`. Under `http` the OTLP signal path `/v1/traces` is appended to the endpoint unless it already names one. | `grpc` | +| `OXEN_OTEL_ENDPOINT` | Collector endpoint: an `http://` or `https://` URL, or a bare `host:port` (which gets `http://`). Absent = export disabled, unless a standard endpoint variable below names one. | *(none)* | +| `OXEN_OTEL_PROTOCOL` | Transport: `grpc`, or `http` / `http/protobuf` / `http/json` for HTTP. Under HTTP the OTLP signal path `/v1/traces` is appended to the endpoint unless it already names one, and the payload is binary protobuf whichever of the three spellings is used. | `grpc` | | `OXEN_OTEL_FILTER` | Which spans and events are exported. Same syntax as `RUST_LOG`, and independent of it. | `info` | An `https://` endpoint is verified against the platform's root certificate @@ -245,8 +245,23 @@ store under both transports, so a collector behind a publicly trusted certificate needs no further configuration. A private CA has to be installed in that store. -The standard `OTEL_EXPORTER_OTLP_ENDPOINT` variable is also respected as a -fallback if `OXEN_OTEL_ENDPOINT` is not set. +The endpoint and the transport each fall back to their standard variables where +the `OXEN_` one is not set, so a vendor's stock configuration snippet works as +given. Each is resolved in the order `OXEN_OTEL_*`, then the traces-specific +standard variable, then the general one: + +| Setting | Resolved in this order | +|---|---| +| Endpoint | `OXEN_OTEL_ENDPOINT`, `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, `OTEL_EXPORTER_OTLP_ENDPOINT` | +| Transport | `OXEN_OTEL_PROTOCOL`, `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`, `OTEL_EXPORTER_OTLP_PROTOCOL` | + +A variable set to a blank value names nothing and falls through to the next, +rather than shadowing it. + +`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` names the traces signal rather than the +collector, so under HTTP it is posted to exactly as configured. Include +`/v1/traces` in it. The other two name the collector, and `/v1/traces` is +appended to them. These standard `OTEL_*` variables are read by the SDK itself: @@ -257,6 +272,8 @@ These standard `OTEL_*` variables are read by the SDK itself: | `OTEL_TRACES_SAMPLER` | `always_on`, `always_off`, `traceidratio`, `parentbased_always_on`, `parentbased_always_off`, `parentbased_traceidratio`. | `parentbased_always_on` | | `OTEL_TRACES_SAMPLER_ARG` | Sampling probability, `0.0`–`1.0`, for the ratio samplers. | `1.0` | | `OTEL_BSP_MAX_QUEUE_SIZE`, `OTEL_BSP_SCHEDULE_DELAY`, `OTEL_BSP_MAX_EXPORT_BATCH_SIZE`, `OTEL_BSP_EXPORT_TIMEOUT` | Batch-processor tuning: queue depth, how often a batch drains, batch size, and how long the processor waits on one export. | `4096`, `2000` ms, `512`, `30000` ms | +| `OTEL_EXPORTER_OTLP_COMPRESSION` | `gzip` to compress export payloads, which is worth roughly 8x on a full batch of spans for a few milliseconds of CPU on the exporter's own thread. Only `gzip` is compiled in; any other value fails the exporter build, which disables export. Unset sends payloads uncompressed. | *(none)* | +| `OTEL_EXPORTER_OTLP_TRACES_COMPRESSION` | The same setting for span exports alone, and takes precedence over `OTEL_EXPORTER_OTLP_COMPRESSION` where both are set. | *(whatever `OTEL_EXPORTER_OTLP_COMPRESSION` resolves to)* | | `OTEL_EXPORTER_OTLP_TIMEOUT` | How long one export request to the collector may take, for every signal. Distinct from `OTEL_BSP_EXPORT_TIMEOUT` above, which bounds the batch processor rather than the request. | `10000` ms | | `OTEL_EXPORTER_OTLP_TRACES_TIMEOUT` | The same bound for span exports alone, and takes precedence over `OTEL_EXPORTER_OTLP_TIMEOUT` where both are set. | *(whatever `OTEL_EXPORTER_OTLP_TIMEOUT` resolves to)* |