diff --git a/.github/workflows/linux-binaries.yml b/.github/workflows/linux-binaries.yml index 5fee1c4fe3..cbe85b6195 100644 --- a/.github/workflows/linux-binaries.yml +++ b/.github/workflows/linux-binaries.yml @@ -39,10 +39,15 @@ jobs: fail-fast: false matrix: platform: + # Both arches build on 22.04 (glibc 2.35) to keep one supported floor. + # arm64 used to build on ubuntu-24.04-arm (glibc 2.39), which produced + # a relay needing GLIBC_2.38 — unable to start on Debian 12 / Ubuntu + # 22.04 arm64, or in the deploy runtime image. The floor is asserted + # below rather than left to whatever the runner image happens to ship. - os: ubuntu-22.04 name: linux-x64 target: x86_64-unknown-linux-gnu - - os: ubuntu-24.04-arm + - os: ubuntu-22.04-arm name: linux-arm64 target: aarch64-unknown-linux-gnu @@ -113,6 +118,20 @@ jobs: -p bitfun-relay-server \ --bins + # A binary above this floor cannot start on older distributions, and the + # relay one-click deploy installs it into a Debian runtime image. Catch it + # here, where the fix is a runner label, rather than as a container that + # exits instantly with a loader error. + - name: Verify glibc floor + shell: bash + run: | + rel="target/${{ matrix.platform.target }}/release" + bash scripts/ci/check-glibc-floor.sh 2.35 \ + "$rel/bitfun-relay-server" \ + "$rel/relay-admin" \ + "$rel/bitfun" \ + "$rel/bitfun-cli" + - name: Test CLI installer shell: bash env: diff --git a/scripts/ci/check-glibc-floor.sh b/scripts/ci/check-glibc-floor.sh new file mode 100755 index 0000000000..63eb3a5d25 --- /dev/null +++ b/scripts/ci/check-glibc-floor.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# +# Assert that released Linux binaries do not require a newer glibc than we +# promise to support. +# +# Usage: check-glibc-floor.sh [binary...] +# +# Why this exists: the release matrix used to build x86_64 on ubuntu-22.04 +# (glibc 2.35) but arm64 on ubuntu-24.04-arm (2.39). The arm64 relay therefore +# required GLIBC_2.38 and could not start at all in the deploy runtime image +# (debian:bookworm-slim, 2.36) — nor on a Debian 12 / Ubuntu 22.04 arm64 host +# for anyone running the tarball directly. Nothing anywhere declared the +# supported floor, so the mismatch was invisible until a container exited with a +# loader error on stderr that the deploy script discarded. +# +# The floor is a promise to users. Make it explicit and machine-checked, so +# raising it becomes a deliberate decision rather than a side effect of a runner +# image bump. +# +# `.gnu.version_r` is the authoritative list of versioned symbols a binary needs +# from its shared libraries; `readelf -V` prints it. + +set -euo pipefail + +MAX="${1:?usage: check-glibc-floor.sh [binary...]}" +shift +if [ "$#" -eq 0 ]; then + echo "ERROR: no binaries given" >&2 + exit 2 +fi + +# Without readelf every binary would look requirement-free and the check would +# pass everything — a silent false negative is worse than no check at all. +if ! command -v readelf >/dev/null 2>&1; then + echo "ERROR: readelf not found (install binutils); refusing to skip the check" >&2 + exit 2 +fi + +# Highest GLIBC_x.y this binary asks for; empty when it needs none. +# The trailing `|| true` matters: `grep` exits non-zero when a binary has no +# versioned glibc symbols at all (a static build), and under `set -e` that +# would abort the whole run at the command substitution below. +max_required_glibc() { + readelf -V "$1" 2>/dev/null | + grep -oE 'GLIBC_[0-9]+\.[0-9]+' | + sort -u -V | + tail -n 1 || true +} + +status=0 +for bin in "$@"; do + if [ ! -f "$bin" ]; then + echo "ERROR: $bin does not exist" >&2 + status=1 + continue + fi + required="$(max_required_glibc "$bin")" + if [ -z "$required" ]; then + echo "ok $(basename "$bin"): no versioned glibc requirement" + continue + fi + required="${required#GLIBC_}" + # sort -V puts the larger version last: if that is not MAX, MAX was exceeded. + highest="$(printf '%s\n%s\n' "$MAX" "$required" | sort -V | tail -n 1)" + if [ "$highest" != "$MAX" ]; then + echo "FAIL $(basename "$bin"): requires glibc $required, above the $MAX floor" >&2 + status=1 + else + echo "ok $(basename "$bin"): requires glibc $required (floor $MAX)" + fi +done + +if [ "$status" -ne 0 ]; then + cat >&2 <= what the *archive being installed* was linked + # against — which is not the same as what CI builds today. + # + # arm64 releases up to and including v0.2.14 were built on ubuntu-24.04-arm and + # require GLIBC_2.38. On bookworm-slim (2.36) they could not load at all: the + # container exited instantly, the loader error went to stderr, and the deploy + # surfaced only as a failed health check followed by a 20-minute rebuild. + # + # The release matrix now pins both arches to ubuntu-22.04 (glibc 2.35, asserted + # by scripts/ci/check-glibc-floor.sh), but that does NOT make bookworm safe + # again: Desktop pins BITFUN_RELEASE_TAG to its own version, so a v0.2.14 + # client installs the v0.2.14 archive forever, and published archives keep the + # floor they were built with. This base must satisfy the highest floor across + # every release a client in the wild might still install. trixie-slim carries + # glibc 2.41 and covers both 2.38 and 2.35. cat >"$context.new/Dockerfile" <<'DOCKERFILE' -FROM debian:bookworm-slim +FROM debian:trixie-slim ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates curl \ @@ -401,6 +416,24 @@ COPY bitfun-relay-server relay-admin /app/ COPY static /app/static RUN chmod 755 /app/bitfun-relay-server /app/relay-admin \ && mkdir -p /app/data /app/room-web +# Fail the build, loudly and in seconds, if either binary cannot be loaded here. +# `ldd` runs the real dynamic loader and prints the exact +# `version 'GLIBC_x.yz' not found` line — but it still exits 0, so its *output* +# is the gate, not its status. The relay binary itself is unusable as a probe: +# it has no --version flag and simply starts serving. Without this check a +# future runner bump reappears as an opaque failed health check plus a +# 20-minute source rebuild. +RUN set -eu; \ + for bin in /app/bitfun-relay-server /app/relay-admin; do \ + out="$(ldd "$bin" 2>&1)"; \ + printf '%s\n' "$out"; \ + case "$out" in \ + *"not found"*) \ + echo "ERROR: $bin cannot be loaded on this base image (see above)." >&2; \ + echo " The published binary needs a newer glibc than this base provides." >&2; \ + exit 1 ;; \ + esac; \ + done HEALTHCHECK --interval=15s --timeout=5s --start-period=20s --retries=5 \ CMD curl -fsS "http://127.0.0.1:${RELAY_PORT:-9700}/health" || exit 1 CMD ["/app/bitfun-relay-server"] @@ -493,7 +526,17 @@ DOCKERFILE done echo ">>> Published Relay binary failed its health check; restoring previous container." - bitfun_docker logs --tail 40 bitfun-relay 2>/dev/null || true + # `docker logs` relays the container's stderr on *its own* stderr, so the + # `2>/dev/null` that used to be here discarded exactly the output we need: + # the relay logs through tracing, i.e. to stderr. Keep both streams, and say + # whether the container died or was up but not answering — the two have + # completely different causes. + echo ">>> Container state: $(bitfun_docker inspect \ + -f 'running={{.State.Running}} exit={{.State.ExitCode}} oom={{.State.OOMKilled}} err={{.State.Error}}' \ + bitfun-relay 2>&1 || true)" + echo ">>> Probed http://${probe_host}:${RELAY_PORT:-9700}/health" + echo ">>> Last 40 log lines from bitfun-relay:" + bitfun_docker logs --tail 40 bitfun-relay 2>&1 | sed 's/^/ /' || true bitfun_restore_previous_relay trap - INT TERM return 1 diff --git a/src/crates/services/services-integrations/src/remote_ssh/relay_deploy.rs b/src/crates/services/services-integrations/src/remote_ssh/relay_deploy.rs index 6e23c4a391..75d4e8dba6 100644 --- a/src/crates/services/services-integrations/src/remote_ssh/relay_deploy.rs +++ b/src/crates/services/services-integrations/src/remote_ssh/relay_deploy.rs @@ -511,14 +511,25 @@ pub async fn start_task( Ok(RelayTaskStart { script_path }) } -/// Strip trailing CR from a file already on the relay host, in place. +/// Strip CR from a file already on the relay host, in place. /// -/// POSIX `sed` (no `-i`, whose syntax differs between GNU and BSD userlands). -/// The rewrite drops the file's mode, so callers must `chmod` afterwards. +/// Deliberately `tr -d '\r'` and not `sed 's/$//'`: `tr` expands the `\r` +/// escape itself, so the command contains no raw CR byte. A raw CR would be +/// carried through `to_unix_script` on its way out — the CR remover travelling +/// through the CR remover — and any text-mode hop that rewrites line endings +/// would silently turn this into a no-op. Removing every CR rather than only +/// trailing ones is safe here because the scripts are generated bash that never +/// contains an intentional CR (`embedded_scripts_are_lf_only` enforces that). +/// +/// `sed -i` is avoided too: its syntax differs between GNU and BSD userlands. +/// The rewrite replaces the file, so callers must `chmod` afterwards, and the +/// scratch file is cleaned up even when the rewrite fails. fn strip_cr_command(path: &str) -> String { let src = shell_quote_posix(path); let tmp = shell_quote_posix(&format!("{path}.lf")); - format!("sed 's/\r$//' {src} > {tmp} && mv {tmp} {src}") + format!( + "{{ tr -d '\\r' < {src} > {tmp} && mv {tmp} {src}; }} || {{ rm -f {tmp}; false; }}" + ) } /// Prepare uploaded scripts for the PTY: normalize line endings, make them @@ -1168,6 +1179,11 @@ bitfun_run_deploy_sh() { local port="${RELAY_PORT:-9700}" # Prefer already-resolved mirror mode so deploy.sh does not re-probe. local mirror_mode="${BITFUN_MIRROR:-${BITFUN_MIRROR_MODE:-auto}}" + # Always --build-from-source: this function is reached ONLY after + # bitfun_try_release_deploy already failed, and deploy.sh's own first step is + # that same release-binary path. Without the flag it re-downloads, re-builds + # and re-starts the published binary that just failed — the deploy visibly + # runs twice before reaching the source build it was called for. # DOCKER_BUILDKIT is required for Dockerfile cargo registry/git/target mounts. # DOCKER_CONFIG is deliberately NOT forwarded to the sudo branches: root would # write config.json into the SSH user's ~/.bitfun/docker-config and every later @@ -1184,7 +1200,7 @@ bitfun_run_deploy_sh() { BITFUN_CARGO_SPARSE_URL="${BITFUN_CARGO_SPARSE_URL:-}" \ BITFUN_DOCKER_REGISTRY_MIRRORS="${BITFUN_DOCKER_REGISTRY_MIRRORS:-}" \ BITFUN_GITHUB_PROXY="${BITFUN_GITHUB_PROXY:-}" \ - bash "$dir/deploy.sh" + bash "$dir/deploy.sh" --build-from-source else sudo -E env RELAY_PORT="$port" RELAY_CARGO_BUILD_JOBS="${RELAY_CARGO_BUILD_JOBS:-}" \ DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 BUILDKIT_PROGRESS=plain \ @@ -1194,11 +1210,11 @@ bitfun_run_deploy_sh() { BITFUN_CARGO_SPARSE_URL="${BITFUN_CARGO_SPARSE_URL:-}" \ BITFUN_DOCKER_REGISTRY_MIRRORS="${BITFUN_DOCKER_REGISTRY_MIRRORS:-}" \ BITFUN_GITHUB_PROXY="${BITFUN_GITHUB_PROXY:-}" \ - bash "$dir/deploy.sh" + bash "$dir/deploy.sh" --build-from-source fi ;; sg) - sg docker -c "env RELAY_PORT='$port' RELAY_CARGO_BUILD_JOBS='${RELAY_CARGO_BUILD_JOBS:-}' DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 BUILDKIT_PROGRESS=plain DOCKER_CONFIG='${DOCKER_CONFIG:-}' BITFUN_MIRROR='$mirror_mode' BITFUN_USE_CN_MIRROR='${BITFUN_USE_CN_MIRROR:-0}' BITFUN_APT_MIRROR='${BITFUN_APT_MIRROR:-}' BITFUN_CARGO_SPARSE_URL='${BITFUN_CARGO_SPARSE_URL:-}' BITFUN_DOCKER_REGISTRY_MIRRORS='${BITFUN_DOCKER_REGISTRY_MIRRORS:-}' BITFUN_GITHUB_PROXY='${BITFUN_GITHUB_PROXY:-}' bash '$dir/deploy.sh'" + sg docker -c "env RELAY_PORT='$port' RELAY_CARGO_BUILD_JOBS='${RELAY_CARGO_BUILD_JOBS:-}' DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 BUILDKIT_PROGRESS=plain DOCKER_CONFIG='${DOCKER_CONFIG:-}' BITFUN_MIRROR='$mirror_mode' BITFUN_USE_CN_MIRROR='${BITFUN_USE_CN_MIRROR:-0}' BITFUN_APT_MIRROR='${BITFUN_APT_MIRROR:-}' BITFUN_CARGO_SPARSE_URL='${BITFUN_CARGO_SPARSE_URL:-}' BITFUN_DOCKER_REGISTRY_MIRRORS='${BITFUN_DOCKER_REGISTRY_MIRRORS:-}' BITFUN_GITHUB_PROXY='${BITFUN_GITHUB_PROXY:-}' bash '$dir/deploy.sh' --build-from-source" ;; *) env RELAY_PORT="$port" RELAY_CARGO_BUILD_JOBS="${RELAY_CARGO_BUILD_JOBS:-}" \ @@ -1209,7 +1225,7 @@ bitfun_run_deploy_sh() { BITFUN_CARGO_SPARSE_URL="${BITFUN_CARGO_SPARSE_URL:-}" \ BITFUN_DOCKER_REGISTRY_MIRRORS="${BITFUN_DOCKER_REGISTRY_MIRRORS:-}" \ BITFUN_GITHUB_PROXY="${BITFUN_GITHUB_PROXY:-}" \ - bash "$dir/deploy.sh" + bash "$dir/deploy.sh" --build-from-source ;; esac } @@ -1926,6 +1942,12 @@ mod tests { } // The rewrite must not leave its scratch file behind. assert!(!std::path::Path::new(&format!("{script_path}.lf")).exists()); + // No raw CR in the command itself: it would be eaten by to_unix_script + // or by any text-mode hop, silently disabling the strip. + assert!( + !command.contains('\r'), + "the CR strip must not depend on a raw CR surviving transport" + ); assert!(!std::path::Path::new(&pid_path).exists(), "stale pid cleared"); assert!( !std::path::Path::new(&driver_pid_path).exists(), @@ -2077,6 +2099,84 @@ sh -c "$(bitfun_shell_join printf '%s\n' 'a b' "it's" '{{{{.State.Running}}}}' ' ); } + /// `bitfun_run_deploy_sh` is reached only after `bitfun_try_release_deploy` + /// failed, and `deploy.sh`'s own first step is that same release path. + /// Without `--build-from-source` the whole published-binary attempt — + /// download, image build, container start — visibly ran a second time before + /// the source build it was called for. + #[test] + fn source_build_fallback_does_not_retry_the_release_path() { + let helpers = prepare_helpers_bash(); + let runner = helpers + .split_once("bitfun_run_deploy_sh() {") + .expect("helpers must define bitfun_run_deploy_sh") + .1; + // Check invocation sites, not a substring count: the surrounding prose + // mentions deploy.sh too. + let mut sites = 0; + for form in [r#"bash "$dir/deploy.sh""#, r#"bash '$dir/deploy.sh'"#] { + let mut rest = runner; + while let Some(i) = rest.find(form) { + let after = &rest[i + form.len()..]; + assert!( + after.starts_with(" --build-from-source"), + "a deploy.sh invocation on the fallback path would re-run the \ + release-binary step that already failed: ...{}", + &after[..after.len().min(40)] + ); + sites += 1; + rest = after; + } + } + assert_eq!(sites, 4, "expected one invocation per docker mode"); + } + + /// The published arm64 relay needs GLIBC_2.38; `debian:bookworm-slim` ships + /// 2.36, so the binary could not load at all and the deploy surfaced only as + /// a failed health check plus a 20-minute source rebuild. + #[test] + fn runtime_image_base_can_load_the_published_binary() { + let script = release_binary_deploy_bash(); + assert!( + !script.contains("FROM debian:bookworm-slim"), + "bookworm-slim (glibc 2.36) cannot load the arm64 relay (needs 2.38)" + ); + assert!( + script.contains("FROM debian:trixie-slim"), + "runtime base must provide a glibc at least as new as the release matrix" + ); + // `ldd` exits 0 even when it reports an unsatisfied symbol version, so + // the gate has to inspect its output. + assert!( + script.contains(r#"*"not found"*)"#), + "the runtime image must fail its build on an unloadable binary" + ); + } + + /// `docker logs` relays the container's stderr on its own stderr, and the + /// relay logs through tracing — so `2>/dev/null` hid the one message that + /// explained the failure (`version 'GLIBC_2.38' not found`). + #[test] + fn health_check_failure_keeps_container_diagnostics() { + let script = release_binary_deploy_bash(); + let failure = script + .split_once("failed its health check") + .expect("health failure branch") + .1; + let logs = failure + .split_once("logs --tail 40 bitfun-relay") + .expect("failure branch must dump container logs") + .1; + assert!( + logs.starts_with(" 2>&1"), + "container stderr must be kept, not sent to /dev/null" + ); + assert!( + failure.contains("Container state:"), + "must report whether the container died or was up but not answering" + ); + } + #[test] fn sync_source_uses_mirror_url_env_with_upstream_fallback() { let sync = sync_source_bash(); diff --git a/src/web-ui/src/features/relay-deploy/README.md b/src/web-ui/src/features/relay-deploy/README.md index cdc8e3134c..f60c4bebee 100644 --- a/src/web-ui/src/features/relay-deploy/README.md +++ b/src/web-ui/src/features/relay-deploy/README.md @@ -127,6 +127,29 @@ Desktop Tauri surface: `src/apps/desktop/src/api/relay_deploy_api.rs` 19. **Losing the runtime image build costs 20 minutes.** Retry it (clean Docker config, then classic builder) before falling back to a source rebuild. +19a. **The runtime base image's glibc must cover every archive a client might + still install — not just what CI builds today.** arm64 releases through + v0.2.14 were built on ubuntu-24.04-arm and require **GLIBC_2.38**; on + `debian:bookworm-slim` (2.36) the relay could not load at all, and the deploy + showed only a failed health check. The release matrix now pins both arches to + ubuntu-22.04 (glibc 2.35, asserted by `scripts/ci/check-glibc-floor.sh`), but + that does **not** make bookworm safe again: Desktop pins the release tag to + its own version, so a v0.2.14 client installs the 2.38 archive forever. Base + stays `debian:trixie-slim` (2.41). The image build also greps `ldd` output to + fail fast on a mismatch — `ldd` exits 0 even while reporting an unsatisfied + symbol version, so its *output* is the gate, and the relay binary is no use + as a probe because it has no `--version` and just starts serving. + +19b. **The source-build fallback must not redo the release path.** + `bitfun_run_deploy_sh` is reached only after `bitfun_try_release_deploy` + failed, and `deploy.sh` begins with that same release path — so it must be + invoked with `--build-from-source`. Otherwise the published-binary attempt + runs twice, visibly, before the source build. + +19c. **Never send container diagnostics to /dev/null.** `docker logs` relays the + container's stderr on its own stderr and the relay logs through tracing, so + `2>/dev/null` discards exactly the line that explains the failure. + 20. **Prepare-phase death must surface as failure.** The driver claims `.driver.pid` before anything that can fail. Poll keeps reporting `preparing` while that pid is alive — an open sudo prompt is unbounded — but