From 3dbc6cbf1fbcf69049b3e1fce21f867696669e95 Mon Sep 17 00:00:00 2001 From: Bob Lee Date: Mon, 27 Jul 2026 05:39:07 -0700 Subject: [PATCH 1/4] fix(relay-deploy): let the arm64 published binary actually run, and stop deploying twice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported symptom: the published-binary deploy built its image, started the container, failed the health check, and then ran the entire published-binary attempt a second time before reaching the source build. ## arm64 relay could never load in the runtime image The release matrix builds x86_64 on ubuntu-22.04 but arm64 on ubuntu-24.04-arm. Verified against the actual published 0.2.14 artifacts: x86_64 relay requires up to GLIBC_2.35 aarch64 relay requires GLIBC_2.38 debian:bookworm-slim ships 2.36 So x86_64 worked and arm64 could not start at all, which is why this went unnoticed. Reproduced on an aarch64 daemon with the real binary: /app/bitfun-relay-server: /lib/aarch64-linux-gnu/libc.so.6: version `GLIBC_2.38' not found (required by /app/bitfun-relay-server) Base image is now debian:trixie-slim (glibc 2.41), verified to load and serve /health on the first probe with the real arm64 binary. The image build now greps `ldd` output and fails on "not found", so a future runner bump surfaces in seconds instead of as an opaque health-check timeout. Note `ldd` exits 0 even while reporting an unsatisfied symbol version, so its output is the gate rather than its status; the relay binary itself cannot serve as the probe because it has no --version flag and simply starts serving. ## The failure was invisible `docker logs --tail 40 bitfun-relay 2>/dev/null` discarded the container's stderr — which is where the relay's tracing output, and the loader error above, both go. That is why the log showed a failed health check with no explanation. Keep both streams and also report container running/exit/OOM state, so a crash is distinguishable from a container that is up but not answering. ## The deploy ran twice `bitfun_run_deploy_sh` is reached only after `bitfun_try_release_deploy` failed, but invoked `deploy.sh` with no arguments — and `deploy.sh`'s own first step is that same release-binary path. It re-downloaded, rebuilt and restarted the binary that had just failed. Pass `--build-from-source`, the flag that already existed for exactly this. Tests: 31 pass. Adds coverage for the runtime base and its ldd gate, the preserved diagnostics, and one --build-from-source per docker mode. --- src/apps/relay-server/release-download.sh | 39 +++++++- .../src/remote_ssh/relay_deploy.rs | 91 ++++++++++++++++++- .../src/features/relay-deploy/README.md | 20 ++++ 3 files changed, 144 insertions(+), 6 deletions(-) diff --git a/src/apps/relay-server/release-download.sh b/src/apps/relay-server/release-download.sh index 468aeff3c5..e30cf3d23f 100644 --- a/src/apps/relay-server/release-download.sh +++ b/src/apps/relay-server/release-download.sh @@ -390,8 +390,15 @@ bitfun_try_release_deploy() { mkdir -p "$context.new" cp "$extracted/bitfun-relay-server" "$extracted/relay-admin" "$context.new/" cp -R "$extracted/static" "$context.new/static" + # Base image glibc must be >= what the published binary was linked against. + # The release matrix builds x86_64 on ubuntu-22.04 (glibc 2.35) but arm64 on + # ubuntu-24.04-arm (glibc 2.39), and the arm64 relay needs GLIBC_2.38. On + # bookworm-slim (2.36) it therefore could not load at all: the container + # exited instantly, the loader error went to stderr, and the deploy surfaced + # only as a failed health check followed by a 20-minute source rebuild. + # trixie-slim carries glibc 2.41, which covers both with headroom. cat >"$context.new/Dockerfile" <<'DOCKERFILE' -FROM debian:bookworm-slim +FROM debian:trixie-slim ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates curl \ @@ -401,6 +408,24 @@ COPY bitfun-relay-server relay-admin /app/ COPY static /app/static RUN chmod 755 /app/bitfun-relay-server /app/relay-admin \ && mkdir -p /app/data /app/room-web +# Fail the build, loudly and in seconds, if either binary cannot be loaded here. +# `ldd` runs the real dynamic loader and prints the exact +# `version 'GLIBC_x.yz' not found` line — but it still exits 0, so its *output* +# is the gate, not its status. The relay binary itself is unusable as a probe: +# it has no --version flag and simply starts serving. Without this check a +# future runner bump reappears as an opaque failed health check plus a +# 20-minute source rebuild. +RUN set -eu; \ + for bin in /app/bitfun-relay-server /app/relay-admin; do \ + out="$(ldd "$bin" 2>&1)"; \ + printf '%s\n' "$out"; \ + case "$out" in \ + *"not found"*) \ + echo "ERROR: $bin cannot be loaded on this base image (see above)." >&2; \ + echo " The published binary needs a newer glibc than this base provides." >&2; \ + exit 1 ;; \ + esac; \ + done HEALTHCHECK --interval=15s --timeout=5s --start-period=20s --retries=5 \ CMD curl -fsS "http://127.0.0.1:${RELAY_PORT:-9700}/health" || exit 1 CMD ["/app/bitfun-relay-server"] @@ -493,7 +518,17 @@ DOCKERFILE done echo ">>> Published Relay binary failed its health check; restoring previous container." - bitfun_docker logs --tail 40 bitfun-relay 2>/dev/null || true + # `docker logs` relays the container's stderr on *its own* stderr, so the + # `2>/dev/null` that used to be here discarded exactly the output we need: + # the relay logs through tracing, i.e. to stderr. Keep both streams, and say + # whether the container died or was up but not answering — the two have + # completely different causes. + echo ">>> Container state: $(bitfun_docker inspect \ + -f 'running={{.State.Running}} exit={{.State.ExitCode}} oom={{.State.OOMKilled}} err={{.State.Error}}' \ + bitfun-relay 2>&1 || true)" + echo ">>> Probed http://${probe_host}:${RELAY_PORT:-9700}/health" + echo ">>> Last 40 log lines from bitfun-relay:" + bitfun_docker logs --tail 40 bitfun-relay 2>&1 | sed 's/^/ /' || true bitfun_restore_previous_relay trap - INT TERM return 1 diff --git a/src/crates/services/services-integrations/src/remote_ssh/relay_deploy.rs b/src/crates/services/services-integrations/src/remote_ssh/relay_deploy.rs index 6e23c4a391..c16530bcf7 100644 --- a/src/crates/services/services-integrations/src/remote_ssh/relay_deploy.rs +++ b/src/crates/services/services-integrations/src/remote_ssh/relay_deploy.rs @@ -1168,6 +1168,11 @@ bitfun_run_deploy_sh() { local port="${RELAY_PORT:-9700}" # Prefer already-resolved mirror mode so deploy.sh does not re-probe. local mirror_mode="${BITFUN_MIRROR:-${BITFUN_MIRROR_MODE:-auto}}" + # Always --build-from-source: this function is reached ONLY after + # bitfun_try_release_deploy already failed, and deploy.sh's own first step is + # that same release-binary path. Without the flag it re-downloads, re-builds + # and re-starts the published binary that just failed — the deploy visibly + # runs twice before reaching the source build it was called for. # DOCKER_BUILDKIT is required for Dockerfile cargo registry/git/target mounts. # DOCKER_CONFIG is deliberately NOT forwarded to the sudo branches: root would # write config.json into the SSH user's ~/.bitfun/docker-config and every later @@ -1184,7 +1189,7 @@ bitfun_run_deploy_sh() { BITFUN_CARGO_SPARSE_URL="${BITFUN_CARGO_SPARSE_URL:-}" \ BITFUN_DOCKER_REGISTRY_MIRRORS="${BITFUN_DOCKER_REGISTRY_MIRRORS:-}" \ BITFUN_GITHUB_PROXY="${BITFUN_GITHUB_PROXY:-}" \ - bash "$dir/deploy.sh" + bash "$dir/deploy.sh" --build-from-source else sudo -E env RELAY_PORT="$port" RELAY_CARGO_BUILD_JOBS="${RELAY_CARGO_BUILD_JOBS:-}" \ DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 BUILDKIT_PROGRESS=plain \ @@ -1194,11 +1199,11 @@ bitfun_run_deploy_sh() { BITFUN_CARGO_SPARSE_URL="${BITFUN_CARGO_SPARSE_URL:-}" \ BITFUN_DOCKER_REGISTRY_MIRRORS="${BITFUN_DOCKER_REGISTRY_MIRRORS:-}" \ BITFUN_GITHUB_PROXY="${BITFUN_GITHUB_PROXY:-}" \ - bash "$dir/deploy.sh" + bash "$dir/deploy.sh" --build-from-source fi ;; sg) - sg docker -c "env RELAY_PORT='$port' RELAY_CARGO_BUILD_JOBS='${RELAY_CARGO_BUILD_JOBS:-}' DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 BUILDKIT_PROGRESS=plain DOCKER_CONFIG='${DOCKER_CONFIG:-}' BITFUN_MIRROR='$mirror_mode' BITFUN_USE_CN_MIRROR='${BITFUN_USE_CN_MIRROR:-0}' BITFUN_APT_MIRROR='${BITFUN_APT_MIRROR:-}' BITFUN_CARGO_SPARSE_URL='${BITFUN_CARGO_SPARSE_URL:-}' BITFUN_DOCKER_REGISTRY_MIRRORS='${BITFUN_DOCKER_REGISTRY_MIRRORS:-}' BITFUN_GITHUB_PROXY='${BITFUN_GITHUB_PROXY:-}' bash '$dir/deploy.sh'" + sg docker -c "env RELAY_PORT='$port' RELAY_CARGO_BUILD_JOBS='${RELAY_CARGO_BUILD_JOBS:-}' DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 BUILDKIT_PROGRESS=plain DOCKER_CONFIG='${DOCKER_CONFIG:-}' BITFUN_MIRROR='$mirror_mode' BITFUN_USE_CN_MIRROR='${BITFUN_USE_CN_MIRROR:-0}' BITFUN_APT_MIRROR='${BITFUN_APT_MIRROR:-}' BITFUN_CARGO_SPARSE_URL='${BITFUN_CARGO_SPARSE_URL:-}' BITFUN_DOCKER_REGISTRY_MIRRORS='${BITFUN_DOCKER_REGISTRY_MIRRORS:-}' BITFUN_GITHUB_PROXY='${BITFUN_GITHUB_PROXY:-}' bash '$dir/deploy.sh' --build-from-source" ;; *) env RELAY_PORT="$port" RELAY_CARGO_BUILD_JOBS="${RELAY_CARGO_BUILD_JOBS:-}" \ @@ -1209,7 +1214,7 @@ bitfun_run_deploy_sh() { BITFUN_CARGO_SPARSE_URL="${BITFUN_CARGO_SPARSE_URL:-}" \ BITFUN_DOCKER_REGISTRY_MIRRORS="${BITFUN_DOCKER_REGISTRY_MIRRORS:-}" \ BITFUN_GITHUB_PROXY="${BITFUN_GITHUB_PROXY:-}" \ - bash "$dir/deploy.sh" + bash "$dir/deploy.sh" --build-from-source ;; esac } @@ -2077,6 +2082,84 @@ sh -c "$(bitfun_shell_join printf '%s\n' 'a b' "it's" '{{{{.State.Running}}}}' ' ); } + /// `bitfun_run_deploy_sh` is reached only after `bitfun_try_release_deploy` + /// failed, and `deploy.sh`'s own first step is that same release path. + /// Without `--build-from-source` the whole published-binary attempt — + /// download, image build, container start — visibly ran a second time before + /// the source build it was called for. + #[test] + fn source_build_fallback_does_not_retry_the_release_path() { + let helpers = prepare_helpers_bash(); + let runner = helpers + .split_once("bitfun_run_deploy_sh() {") + .expect("helpers must define bitfun_run_deploy_sh") + .1; + // Check invocation sites, not a substring count: the surrounding prose + // mentions deploy.sh too. + let mut sites = 0; + for form in [r#"bash "$dir/deploy.sh""#, r#"bash '$dir/deploy.sh'"#] { + let mut rest = runner; + while let Some(i) = rest.find(form) { + let after = &rest[i + form.len()..]; + assert!( + after.starts_with(" --build-from-source"), + "a deploy.sh invocation on the fallback path would re-run the \ + release-binary step that already failed: ...{}", + &after[..after.len().min(40)] + ); + sites += 1; + rest = after; + } + } + assert_eq!(sites, 4, "expected one invocation per docker mode"); + } + + /// The published arm64 relay needs GLIBC_2.38; `debian:bookworm-slim` ships + /// 2.36, so the binary could not load at all and the deploy surfaced only as + /// a failed health check plus a 20-minute source rebuild. + #[test] + fn runtime_image_base_can_load_the_published_binary() { + let script = release_binary_deploy_bash(); + assert!( + !script.contains("FROM debian:bookworm-slim"), + "bookworm-slim (glibc 2.36) cannot load the arm64 relay (needs 2.38)" + ); + assert!( + script.contains("FROM debian:trixie-slim"), + "runtime base must provide a glibc at least as new as the release matrix" + ); + // `ldd` exits 0 even when it reports an unsatisfied symbol version, so + // the gate has to inspect its output. + assert!( + script.contains(r#"*"not found"*)"#), + "the runtime image must fail its build on an unloadable binary" + ); + } + + /// `docker logs` relays the container's stderr on its own stderr, and the + /// relay logs through tracing — so `2>/dev/null` hid the one message that + /// explained the failure (`version 'GLIBC_2.38' not found`). + #[test] + fn health_check_failure_keeps_container_diagnostics() { + let script = release_binary_deploy_bash(); + let failure = script + .split_once("failed its health check") + .expect("health failure branch") + .1; + let logs = failure + .split_once("logs --tail 40 bitfun-relay") + .expect("failure branch must dump container logs") + .1; + assert!( + logs.starts_with(" 2>&1"), + "container stderr must be kept, not sent to /dev/null" + ); + assert!( + failure.contains("Container state:"), + "must report whether the container died or was up but not answering" + ); + } + #[test] fn sync_source_uses_mirror_url_env_with_upstream_fallback() { let sync = sync_source_bash(); diff --git a/src/web-ui/src/features/relay-deploy/README.md b/src/web-ui/src/features/relay-deploy/README.md index cdc8e3134c..3e9c5d034f 100644 --- a/src/web-ui/src/features/relay-deploy/README.md +++ b/src/web-ui/src/features/relay-deploy/README.md @@ -127,6 +127,26 @@ Desktop Tauri surface: `src/apps/desktop/src/api/relay_deploy_api.rs` 19. **Losing the runtime image build costs 20 minutes.** Retry it (clean Docker config, then classic builder) before falling back to a source rebuild. +19a. **The runtime base image's glibc must cover the published binary.** The + release matrix builds x86_64 on ubuntu-22.04 (needs GLIBC_2.35) but arm64 on + ubuntu-24.04-arm (needs **GLIBC_2.38**). On `debian:bookworm-slim` (2.36) the + arm64 relay could not load at all — the container exited instantly and the + deploy showed only a failed health check. Base is `debian:trixie-slim` + (glibc 2.41), and the image build greps `ldd` output to fail fast on a + mismatch. `ldd` exits 0 even when it reports an unsatisfied symbol version, + so its *output* is the gate; the relay binary is no use as a probe because it + has no `--version` and just starts serving. + +19b. **The source-build fallback must not redo the release path.** + `bitfun_run_deploy_sh` is reached only after `bitfun_try_release_deploy` + failed, and `deploy.sh` begins with that same release path — so it must be + invoked with `--build-from-source`. Otherwise the published-binary attempt + runs twice, visibly, before the source build. + +19c. **Never send container diagnostics to /dev/null.** `docker logs` relays the + container's stderr on its own stderr and the relay logs through tracing, so + `2>/dev/null` discards exactly the line that explains the failure. + 20. **Prepare-phase death must surface as failure.** The driver claims `.driver.pid` before anything that can fail. Poll keeps reporting `preparing` while that pid is alive — an open sudo prompt is unbounded — but From 2472d515803e3634088fb1c96bb31f87872dc725 Mon Sep 17 00:00:00 2001 From: Bob Lee Date: Mon, 27 Jul 2026 06:06:53 -0700 Subject: [PATCH 2/4] fix(ci): pin one glibc floor for released Linux binaries and assert it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the arm64 fix from the other side. The release matrix built x86_64 on ubuntu-22.04 (glibc 2.35) but arm64 on ubuntu-24.04-arm (2.39), so the arm64 relay and CLI required GLIBC_2.38 — unable to run on Debian 12 / Ubuntu 22.04 arm64 at all. Changing the relay's runtime image fixed the Docker deploy, but not anyone following the README's "Manual Run (without Docker)" path, who never touches Docker. Pin arm64 to ubuntu-22.04-arm so both arches share a 2.35 floor, and assert it with scripts/ci/check-glibc-floor.sh over every published binary (bitfun-relay-server, relay-admin, bitfun, bitfun-cli). Nothing previously declared the supported floor, which is why a runner image difference silently became a user-visible breakage. The checker reads `.gnu.version_r` via `readelf -V` — the authoritative list of versioned symbols a binary needs. Verified against the real published 0.2.14 archives: x86_64 passes at 2.35, arm64 fails at 2.35 reporting 2.38, arm64 passes at 2.38, and a mixed run reports each binary before failing. The relay runtime base deliberately stays on trixie-slim. Desktop pins BITFUN_RELEASE_TAG to its own version, so a v0.2.14 client installs the v0.2.14 archive forever and published archives keep the floor they were built with — the base must cover the highest floor still in the wild (2.38), not the floor CI produces from now on. Both files now say so, so neither gets "optimized" back. --- .github/workflows/linux-binaries.yml | 21 ++++- scripts/ci/check-glibc-floor.sh | 76 +++++++++++++++++++ src/apps/relay-server/release-download.sh | 22 ++++-- .../src/features/relay-deploy/README.md | 21 ++--- 4 files changed, 123 insertions(+), 17 deletions(-) create mode 100755 scripts/ci/check-glibc-floor.sh diff --git a/.github/workflows/linux-binaries.yml b/.github/workflows/linux-binaries.yml index 5fee1c4fe3..cbe85b6195 100644 --- a/.github/workflows/linux-binaries.yml +++ b/.github/workflows/linux-binaries.yml @@ -39,10 +39,15 @@ jobs: fail-fast: false matrix: platform: + # Both arches build on 22.04 (glibc 2.35) to keep one supported floor. + # arm64 used to build on ubuntu-24.04-arm (glibc 2.39), which produced + # a relay needing GLIBC_2.38 — unable to start on Debian 12 / Ubuntu + # 22.04 arm64, or in the deploy runtime image. The floor is asserted + # below rather than left to whatever the runner image happens to ship. - os: ubuntu-22.04 name: linux-x64 target: x86_64-unknown-linux-gnu - - os: ubuntu-24.04-arm + - os: ubuntu-22.04-arm name: linux-arm64 target: aarch64-unknown-linux-gnu @@ -113,6 +118,20 @@ jobs: -p bitfun-relay-server \ --bins + # A binary above this floor cannot start on older distributions, and the + # relay one-click deploy installs it into a Debian runtime image. Catch it + # here, where the fix is a runner label, rather than as a container that + # exits instantly with a loader error. + - name: Verify glibc floor + shell: bash + run: | + rel="target/${{ matrix.platform.target }}/release" + bash scripts/ci/check-glibc-floor.sh 2.35 \ + "$rel/bitfun-relay-server" \ + "$rel/relay-admin" \ + "$rel/bitfun" \ + "$rel/bitfun-cli" + - name: Test CLI installer shell: bash env: diff --git a/scripts/ci/check-glibc-floor.sh b/scripts/ci/check-glibc-floor.sh new file mode 100755 index 0000000000..7f0a66b461 --- /dev/null +++ b/scripts/ci/check-glibc-floor.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# +# Assert that released Linux binaries do not require a newer glibc than we +# promise to support. +# +# Usage: check-glibc-floor.sh [binary...] +# +# Why this exists: the release matrix used to build x86_64 on ubuntu-22.04 +# (glibc 2.35) but arm64 on ubuntu-24.04-arm (2.39). The arm64 relay therefore +# required GLIBC_2.38 and could not start at all in the deploy runtime image +# (debian:bookworm-slim, 2.36) — nor on a Debian 12 / Ubuntu 22.04 arm64 host +# for anyone running the tarball directly. Nothing anywhere declared the +# supported floor, so the mismatch was invisible until a container exited with a +# loader error on stderr that the deploy script discarded. +# +# The floor is a promise to users. Make it explicit and machine-checked, so +# raising it becomes a deliberate decision rather than a side effect of a runner +# image bump. +# +# `.gnu.version_r` is the authoritative list of versioned symbols a binary needs +# from its shared libraries; `readelf -V` prints it. + +set -euo pipefail + +MAX="${1:?usage: check-glibc-floor.sh [binary...]}" +shift +if [ "$#" -eq 0 ]; then + echo "ERROR: no binaries given" >&2 + exit 2 +fi + +# Highest GLIBC_x.y this binary asks for; empty when it needs none. +max_required_glibc() { + readelf -V "$1" 2>/dev/null | + grep -oE 'GLIBC_[0-9]+\.[0-9]+' | + sort -u -V | + tail -n 1 +} + +status=0 +for bin in "$@"; do + if [ ! -f "$bin" ]; then + echo "ERROR: $bin does not exist" >&2 + status=1 + continue + fi + required="$(max_required_glibc "$bin")" + if [ -z "$required" ]; then + echo "ok $(basename "$bin"): no versioned glibc requirement" + continue + fi + required="${required#GLIBC_}" + # sort -V puts the larger version last: if that is not MAX, MAX was exceeded. + highest="$(printf '%s\n%s\n' "$MAX" "$required" | sort -V | tail -n 1)" + if [ "$highest" != "$MAX" ]; then + echo "FAIL $(basename "$bin"): requires glibc $required, above the $MAX floor" >&2 + status=1 + else + echo "ok $(basename "$bin"): requires glibc $required (floor $MAX)" + fi +done + +if [ "$status" -ne 0 ]; then + cat >&2 <= what the published binary was linked against. - # The release matrix builds x86_64 on ubuntu-22.04 (glibc 2.35) but arm64 on - # ubuntu-24.04-arm (glibc 2.39), and the arm64 relay needs GLIBC_2.38. On - # bookworm-slim (2.36) it therefore could not load at all: the container - # exited instantly, the loader error went to stderr, and the deploy surfaced - # only as a failed health check followed by a 20-minute source rebuild. - # trixie-slim carries glibc 2.41, which covers both with headroom. + # Base image glibc must be >= what the *archive being installed* was linked + # against — which is not the same as what CI builds today. + # + # arm64 releases up to and including v0.2.14 were built on ubuntu-24.04-arm and + # require GLIBC_2.38. On bookworm-slim (2.36) they could not load at all: the + # container exited instantly, the loader error went to stderr, and the deploy + # surfaced only as a failed health check followed by a 20-minute rebuild. + # + # The release matrix now pins both arches to ubuntu-22.04 (glibc 2.35, asserted + # by scripts/ci/check-glibc-floor.sh), but that does NOT make bookworm safe + # again: Desktop pins BITFUN_RELEASE_TAG to its own version, so a v0.2.14 + # client installs the v0.2.14 archive forever, and published archives keep the + # floor they were built with. This base must satisfy the highest floor across + # every release a client in the wild might still install. trixie-slim carries + # glibc 2.41 and covers both 2.38 and 2.35. cat >"$context.new/Dockerfile" <<'DOCKERFILE' FROM debian:trixie-slim ENV DEBIAN_FRONTEND=noninteractive diff --git a/src/web-ui/src/features/relay-deploy/README.md b/src/web-ui/src/features/relay-deploy/README.md index 3e9c5d034f..f60c4bebee 100644 --- a/src/web-ui/src/features/relay-deploy/README.md +++ b/src/web-ui/src/features/relay-deploy/README.md @@ -127,15 +127,18 @@ Desktop Tauri surface: `src/apps/desktop/src/api/relay_deploy_api.rs` 19. **Losing the runtime image build costs 20 minutes.** Retry it (clean Docker config, then classic builder) before falling back to a source rebuild. -19a. **The runtime base image's glibc must cover the published binary.** The - release matrix builds x86_64 on ubuntu-22.04 (needs GLIBC_2.35) but arm64 on - ubuntu-24.04-arm (needs **GLIBC_2.38**). On `debian:bookworm-slim` (2.36) the - arm64 relay could not load at all — the container exited instantly and the - deploy showed only a failed health check. Base is `debian:trixie-slim` - (glibc 2.41), and the image build greps `ldd` output to fail fast on a - mismatch. `ldd` exits 0 even when it reports an unsatisfied symbol version, - so its *output* is the gate; the relay binary is no use as a probe because it - has no `--version` and just starts serving. +19a. **The runtime base image's glibc must cover every archive a client might + still install — not just what CI builds today.** arm64 releases through + v0.2.14 were built on ubuntu-24.04-arm and require **GLIBC_2.38**; on + `debian:bookworm-slim` (2.36) the relay could not load at all, and the deploy + showed only a failed health check. The release matrix now pins both arches to + ubuntu-22.04 (glibc 2.35, asserted by `scripts/ci/check-glibc-floor.sh`), but + that does **not** make bookworm safe again: Desktop pins the release tag to + its own version, so a v0.2.14 client installs the 2.38 archive forever. Base + stays `debian:trixie-slim` (2.41). The image build also greps `ldd` output to + fail fast on a mismatch — `ldd` exits 0 even while reporting an unsatisfied + symbol version, so its *output* is the gate, and the relay binary is no use + as a probe because it has no `--version` and just starts serving. 19b. **The source-build fallback must not redo the release path.** `bitfun_run_deploy_sh` is reached only after `bitfun_try_release_deploy` From 01cd5f62bf496d5f208ecbe1d16206876cc718a0 Mon Sep 17 00:00:00 2001 From: Bob Lee Date: Mon, 27 Jul 2026 06:23:05 -0700 Subject: [PATCH 3/4] fix(relay-deploy): make the host-side CR strip independent of a raw CR byte MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `stage_scripts_command` used `sed 's/$//'` with a literal CR in the command string. That CR then travelled through `to_unix_script` on its way to the host — the CR remover passing through the CR remover — and any text-mode hop that rewrote line endings would silently degrade the strip to a no-op. Found while testing on a real arm64 host: a Python text-mode read in the test harness turned the CR into LF, and the command reached the host as an unterminated `sed` expression. `tr -d '\r'` expands the escape itself, so the command is pure ASCII and cannot be damaged in transit. Removing every CR rather than only trailing ones is safe for generated bash, which never contains an intentional CR (`embedded_scripts_are_lf_only` enforces that). The rewrite now also removes its scratch file when it fails, instead of leaving a partial `.lf` behind. Verified end to end on Ubuntu 24.04 arm64: scripts uploaded with all 3879 lines CRLF-corrupted came out with 0 CRs, mode 700, no scratch file, and both parsed. --- .../src/remote_ssh/relay_deploy.rs | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/crates/services/services-integrations/src/remote_ssh/relay_deploy.rs b/src/crates/services/services-integrations/src/remote_ssh/relay_deploy.rs index c16530bcf7..75d4e8dba6 100644 --- a/src/crates/services/services-integrations/src/remote_ssh/relay_deploy.rs +++ b/src/crates/services/services-integrations/src/remote_ssh/relay_deploy.rs @@ -511,14 +511,25 @@ pub async fn start_task( Ok(RelayTaskStart { script_path }) } -/// Strip trailing CR from a file already on the relay host, in place. +/// Strip CR from a file already on the relay host, in place. /// -/// POSIX `sed` (no `-i`, whose syntax differs between GNU and BSD userlands). -/// The rewrite drops the file's mode, so callers must `chmod` afterwards. +/// Deliberately `tr -d '\r'` and not `sed 's/$//'`: `tr` expands the `\r` +/// escape itself, so the command contains no raw CR byte. A raw CR would be +/// carried through `to_unix_script` on its way out — the CR remover travelling +/// through the CR remover — and any text-mode hop that rewrites line endings +/// would silently turn this into a no-op. Removing every CR rather than only +/// trailing ones is safe here because the scripts are generated bash that never +/// contains an intentional CR (`embedded_scripts_are_lf_only` enforces that). +/// +/// `sed -i` is avoided too: its syntax differs between GNU and BSD userlands. +/// The rewrite replaces the file, so callers must `chmod` afterwards, and the +/// scratch file is cleaned up even when the rewrite fails. fn strip_cr_command(path: &str) -> String { let src = shell_quote_posix(path); let tmp = shell_quote_posix(&format!("{path}.lf")); - format!("sed 's/\r$//' {src} > {tmp} && mv {tmp} {src}") + format!( + "{{ tr -d '\\r' < {src} > {tmp} && mv {tmp} {src}; }} || {{ rm -f {tmp}; false; }}" + ) } /// Prepare uploaded scripts for the PTY: normalize line endings, make them @@ -1931,6 +1942,12 @@ mod tests { } // The rewrite must not leave its scratch file behind. assert!(!std::path::Path::new(&format!("{script_path}.lf")).exists()); + // No raw CR in the command itself: it would be eaten by to_unix_script + // or by any text-mode hop, silently disabling the strip. + assert!( + !command.contains('\r'), + "the CR strip must not depend on a raw CR surviving transport" + ); assert!(!std::path::Path::new(&pid_path).exists(), "stale pid cleared"); assert!( !std::path::Path::new(&driver_pid_path).exists(), From a6a118f60e29aa0d34ce2e7c4e6dbb90a2840e55 Mon Sep 17 00:00:00 2001 From: Bob Lee Date: Mon, 27 Jul 2026 18:21:17 -0700 Subject: [PATCH 4/4] fix(ci): do not let the glibc check pass silently MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the new checker found two ways it could report success without actually checking anything. `grep` exits non-zero when a binary has no versioned glibc symbols at all, and under `set -euo pipefail` that aborted the whole run at the command substitution — before the branch that handles the empty case. Verified: the pre-fix script exits 1 with no output on a static binary; it now reports "no versioned glibc requirement" and continues. Worse, without `readelf` every binary looked requirement-free, so the check passed everything. Verified in debian:trixie-slim, which ships no binutils: the script now refuses with exit 2 instead of green-lighting the release. --- scripts/ci/check-glibc-floor.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/ci/check-glibc-floor.sh b/scripts/ci/check-glibc-floor.sh index 7f0a66b461..63eb3a5d25 100755 --- a/scripts/ci/check-glibc-floor.sh +++ b/scripts/ci/check-glibc-floor.sh @@ -29,12 +29,22 @@ if [ "$#" -eq 0 ]; then exit 2 fi +# Without readelf every binary would look requirement-free and the check would +# pass everything — a silent false negative is worse than no check at all. +if ! command -v readelf >/dev/null 2>&1; then + echo "ERROR: readelf not found (install binutils); refusing to skip the check" >&2 + exit 2 +fi + # Highest GLIBC_x.y this binary asks for; empty when it needs none. +# The trailing `|| true` matters: `grep` exits non-zero when a binary has no +# versioned glibc symbols at all (a static build), and under `set -e` that +# would abort the whole run at the command substitution below. max_required_glibc() { readelf -V "$1" 2>/dev/null | grep -oE 'GLIBC_[0-9]+\.[0-9]+' | sort -u -V | - tail -n 1 + tail -n 1 || true } status=0