From 0cd84bfb0d3643b81e277381768fafe3c671f240 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 3 Jul 2026 21:57:45 +0000 Subject: [PATCH 1/9] Add gleam_host convenience repository Resolves the TODO in gleam_register_toolchains: creates a "_host" repository (gleam/private/host_repo.bzl) that detects the host OS/arch and re-exposes the matching per-platform toolchain repo, so `gleam` can be run directly without knowing the host's platform string: bazel run @gleam_host//:gleam -- format Also fixes a visibility gap in _gleam_repo_impl's generated BUILD.bazel (no default_visibility, unlike hermetic_erlang_repository.bzl) that would have made the per-platform repos' raw gleam/gleam.exe file targets unreachable from gleam_host's alias. --- .github/workflows/ci.yaml | 9 ++ MODULE.bazel | 2 +- README.md | 4 + gleam/extensions.bzl | 10 +++ gleam/private/BUILD.bazel | 7 ++ gleam/private/host_repo.bzl | 94 +++++++++++++++++++ gleam/repositories.bzl | 17 +++- test/unit/host_repo/BUILD.bazel | 3 + test/unit/host_repo/host_repo_test.bzl | 119 +++++++++++++++++++++++++ 9 files changed, 261 insertions(+), 4 deletions(-) create mode 100644 gleam/private/host_repo.bzl create mode 100644 test/unit/host_repo/BUILD.bazel create mode 100644 test/unit/host_repo/host_repo_test.bzl diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6007e32..e22d64f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -66,6 +66,15 @@ jobs: run: | echo "Running the gleam Gazelle language extension's Go unit tests" bazel test --test_output=errors //gleam_gazelle/... + - name: Verify @gleam_host//:gleam is directly runnable + run: | + # gleam_host_repo (gleam/private/host_repo.bzl) detects the host platform without + # needing to know its OS/arch string; prove the resulting binary is real and matches + # the pinned toolchain version, not just that the target resolves. + echo "Running bazel run @gleam_host//:gleam -- --version" + output="$(bazel run @gleam_host//:gleam -- --version)" + echo "$output" + echo "$output" | grep -q "gleam 1.17.0" - name: Verify gazelle generates a BUILD file for a fresh Gleam package run: | echo "Running bazel run //:gazelle against gleam_gazelle/testdata/sample_package (no BUILD.bazel checked in)" diff --git a/MODULE.bazel b/MODULE.bazel index d545e51..2c751a5 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -20,7 +20,7 @@ go_sdk.download(version = "1.24.12") gleam = use_extension("//gleam:extensions.bzl", "gleam") gleam.toolchain(version = "1.17.0") -use_repo(gleam, "gleam_toolchains", "local_config_erlang") +use_repo(gleam, "gleam_host", "gleam_toolchains", "local_config_erlang") register_toolchains("@gleam_toolchains//:all") diff --git a/README.md b/README.md index 2988f63..bcc5622 100644 --- a/README.md +++ b/README.md @@ -174,3 +174,7 @@ extension turns any `gleam.toml` + `src/**/*.gleam` directory into a `gleam_pack Wire it into your own `gazelle_binary`'s `languages` list alongside `@rules_gleam//gleam_gazelle` and run `bazel run //:gazelle` -- see [examples/gazelle_smoke](examples/gazelle_smoke) for a worked example of wiring this into a downstream project's own `MODULE.bazel`/`BUILD.bazel`. + +To run the `gleam` CLI itself (e.g. `gleam format`) without knowing your machine's OS/arch +string, add `"gleam_host"` to the extension's `use_repo(...)` call and run +`bazel run @gleam_host//:gleam -- format`. diff --git a/gleam/extensions.bzl b/gleam/extensions.bzl index e267030..eabe013 100644 --- a/gleam/extensions.bzl +++ b/gleam/extensions.bzl @@ -39,6 +39,16 @@ project's manifest.toml lockfile: ```starlark gleam.hex_manifest(manifest = "//path/to:manifest.toml") ``` + +A `@_host` repository (e.g. `@gleam_host` for the default toolchain name) is always +created alongside the per-platform ones, exposing a directly `bazel run`-able `gleam` CLI for +whatever platform Bazel happens to be running on, without needing to know its OS/arch string: +```starlark +use_repo(gleam, "gleam_host", ...) +``` +```shell +bazel run @gleam_host//:gleam -- format +``` """ load("//erlang/private:hermetic_erlang_repository.bzl", "hermetic_erlang_repository") # buildifier: disable=bzl-visibility diff --git a/gleam/private/BUILD.bazel b/gleam/private/BUILD.bazel index 1d5bb15..6852e26 100644 --- a/gleam/private/BUILD.bazel +++ b/gleam/private/BUILD.bazel @@ -64,6 +64,13 @@ bzl_library( visibility = ["//gleam:__subpackages__"], ) +bzl_library( + name = "host_repo", + srcs = ["host_repo.bzl"], + visibility = ["//gleam:__subpackages__"], + deps = [":toolchains_repo"], +) + bzl_library( name = "versions", srcs = ["versions.bzl"], diff --git a/gleam/private/host_repo.bzl b/gleam/private/host_repo.bzl new file mode 100644 index 0000000..bff74df --- /dev/null +++ b/gleam/private/host_repo.bzl @@ -0,0 +1,94 @@ +"""Repository rule that exposes a runnable `gleam` binary for the host platform. + +`gleam_register_toolchains` (gleam/repositories.bzl) creates one repository per known target +platform (e.g. "gleam1_17_x86_64-unknown-linux-gnu"), plus a `_toolchains` repo wiring them all +into Bazel's toolchain resolution -- neither is meant to be depended on directly, since which one +matches a given machine depends on where Bazel happens to be running. This repository rule picks +the one matching the *host* platform (where `bazel` itself runs, not necessarily a cross-compiled +build's target platform) and re-exposes it under a single, unconditional name, so it can be run +directly without knowing the host's OS/arch string: `bazel run @gleam_host//:gleam -- format`. +""" + +load("//gleam/private:toolchains_repo.bzl", "PLATFORMS") + +# repository_ctx.os.arch normally matches PLATFORMS' own "aarch64"/"x86_64" naming, but some +# Bazel/OS combinations report "arm64"/"amd64" instead (the same alternate spelling +# erlang/private/hermetic_erlang_repository.bzl normalizes) -- accept both. +_ARCH = { + "aarch64": "aarch64", + "arm64": "arm64", + "x86_64": "x86_64", + "amd64": "x86_64", +} + +def host_platform(repository_ctx): + """Maps a repository_ctx's os.name/os.arch to a PLATFORMS key. + + Pulled out of _gleam_host_repo_impl so it can be unit-tested (test/unit/host_repo) against + plain fake structs, without needing a real repository_ctx. + + Args: + repository_ctx: a repository_ctx, or any struct exposing the same `.os.name`/`.os.arch` + fields (e.g. a fake used by tests). + + Returns: + A key of PLATFORMS (gleam/private/toolchains_repo.bzl), e.g. "x86_64-unknown-linux-gnu". + """ + os_name = repository_ctx.os.name.lower() + + if "windows" in os_name: + return "x86_64-pc-windows-msvc" + + arch = _ARCH.get(repository_ctx.os.arch) + if not arch: + fail("gleam_host: unsupported CPU architecture '{}'".format(repository_ctx.os.arch)) + + if "linux" in os_name: + return "{}-unknown-linux-gnu".format(arch) + if "mac os" in os_name: + return "{}-apple-darwin".format(arch) + fail("gleam_host: unsupported host OS '{}'".format(repository_ctx.os.name)) + +def _gleam_host_repo_impl(repository_ctx): + platform = host_platform(repository_ctx) + if platform not in PLATFORMS: + fail(( + "gleam_host: resolved host platform '{}' has no matching gleam toolchain " + + "repository (known platforms: {})" + ).format(platform, ", ".join(PLATFORMS.keys()))) + + user_repository_name = repository_ctx.attr.user_repository_name + tool_file = "gleam.exe" if platform == "x86_64-pc-windows-msvc" else "gleam" + + build_content = """# Generated by gleam/private:host_repo.bzl +load("@bazel_skylib//rules:native_binary.bzl", "native_binary") + +package(default_visibility = ["//visibility:public"]) + +# The same toolchain instance @{name}_{platform}//:gleam_toolchain resolves to for this host -- +# useful for rules that want the ToolchainInfo directly rather than running the CLI. +alias( + name = "gleam_toolchain", + actual = "@{name}_{platform}//:gleam_toolchain", +) + +# A directly `bazel run`-able gleam CLI for the host platform, e.g.: +# bazel run @{name}_host//:gleam -- format +native_binary( + name = "gleam", + src = "@{name}_{platform}//:{tool_file}", +) +""".format(name = user_repository_name, platform = platform, tool_file = tool_file) + + repository_ctx.file("BUILD.bazel", build_content) + +gleam_host_repo = repository_rule( + implementation = _gleam_host_repo_impl, + attrs = { + "user_repository_name": attr.string( + doc = "Base name shared with the per-platform repositories created by gleam_register_toolchains, e.g. \"gleam1_17\".", + mandatory = True, + ), + }, + doc = "Detects the host platform and re-exposes that platform's gleam toolchain repository under a single, unconditional name.", +) diff --git a/gleam/repositories.bzl b/gleam/repositories.bzl index f079ac6..86b792b 100644 --- a/gleam/repositories.bzl +++ b/gleam/repositories.bzl @@ -4,6 +4,7 @@ Dependencies on other modules (bazel_skylib, platforms, etc.) are declared direc MODULE.bazel via bazel_dep -- bzlmod is the only supported dependency mode. """ +load("//gleam/private:host_repo.bzl", "gleam_host_repo") load("//gleam/private:toolchains_repo.bzl", "PLATFORMS", "toolchains_repo") load("//gleam/private:versions.bzl", "TOOL_VERSIONS") @@ -36,6 +37,8 @@ def _gleam_repo_impl(repository_ctx): build_content = """# Generated by gleam/repositories.bzl load("@muchq_rules_gleam//gleam:toolchain.bzl", "gleam_toolchain") +package(default_visibility = ["//visibility:public"]) + gleam_toolchain( name = "gleam_toolchain", target_tool = select({ @@ -59,9 +62,12 @@ gleam_repositories = repository_rule( def gleam_register_toolchains(name, **kwargs): """Convenience macro used by the bzlmod extension (gleam/extensions.bzl) for typical setup. - - create a repository for each built-in platform like "gleam_linux_amd64" - - TODO: create a convenience repository for the host platform like "gleam_host" - - create a repository exposing toolchains for each platform like "gleam_platforms" + - create a repository for each built-in platform, like "gleam1_14_x86_64-unknown-linux-gnu" + - create a repository exposing toolchains for each platform, "gleam1_14_toolchains" + - create a convenience repository for the host platform, "gleam1_14_host" -- exposes a + directly `bazel run`-able gleam CLI (`@gleam1_14_host//:gleam`) without needing to know + the host's OS/arch string, plus a `:gleam_toolchain` alias to the same underlying + toolchain instance registered for that platform Users can avoid this macro and do these steps themselves, if they want more control. Does not itself call native.register_toolchains -- the bzlmod extension's caller does that @@ -83,3 +89,8 @@ def gleam_register_toolchains(name, **kwargs): name = name + "_toolchains", user_repository_name = name, ) + + gleam_host_repo( + name = name + "_host", + user_repository_name = name, + ) diff --git a/test/unit/host_repo/BUILD.bazel b/test/unit/host_repo/BUILD.bazel new file mode 100644 index 0000000..15104a5 --- /dev/null +++ b/test/unit/host_repo/BUILD.bazel @@ -0,0 +1,3 @@ +load(":host_repo_test.bzl", "host_repo_test_suite") + +host_repo_test_suite(name = "host_repo_test_suite") diff --git a/test/unit/host_repo/host_repo_test.bzl b/test/unit/host_repo/host_repo_test.bzl new file mode 100644 index 0000000..a7ad924 --- /dev/null +++ b/test/unit/host_repo/host_repo_test.bzl @@ -0,0 +1,119 @@ +"""Unit tests for gleam/private/host_repo.bzl's host_platform. + +host_platform only ever reads `.os.name`/`.os.arch` off whatever it's given, so a plain struct +stands in for a real repository_ctx without needing any Bazel machinery -- same trick +erlang_otp_tree_test.bzl uses for File objects via `_fake_file`. +""" + +load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts", "unittest") +load("//gleam/private:host_repo.bzl", "host_platform") # buildifier: disable=bzl-visibility + +def _fake_repository_ctx(os_name, os_arch): + return struct(os = struct(name = os_name, arch = os_arch)) + +def _resolves_linux_x86_64_test_impl(ctx): + env = unittest.begin(ctx) + ctx_fake = _fake_repository_ctx("linux", "x86_64") + asserts.equals(env, "x86_64-unknown-linux-gnu", host_platform(ctx_fake)) + return unittest.end(env) + +resolves_linux_x86_64_test = unittest.make(_resolves_linux_x86_64_test_impl) + +def _resolves_linux_aarch64_test_impl(ctx): + env = unittest.begin(ctx) + ctx_fake = _fake_repository_ctx("linux", "aarch64") + asserts.equals(env, "aarch64-unknown-linux-gnu", host_platform(ctx_fake)) + return unittest.end(env) + +resolves_linux_aarch64_test = unittest.make(_resolves_linux_aarch64_test_impl) + +def _resolves_macos_arm64_alternate_spelling_test_impl(ctx): + env = unittest.begin(ctx) + + # Real macOS Bazel releases have been observed reporting "arm64" (not "aarch64") for + # repository_ctx.os.arch -- the same alternate spelling + # erlang/private/hermetic_erlang_repository.bzl normalizes; make sure host_platform accepts + # it too rather than failing on an unrecognized arch string. + ctx_fake = _fake_repository_ctx("mac os x", "arm64") + asserts.equals(env, "aarch64-apple-darwin", host_platform(ctx_fake)) + return unittest.end(env) + +resolves_macos_arm64_alternate_spelling_test = unittest.make(_resolves_macos_arm64_alternate_spelling_test_impl) + +def _resolves_macos_x86_64_test_impl(ctx): + env = unittest.begin(ctx) + ctx_fake = _fake_repository_ctx("mac os x", "x86_64") + asserts.equals(env, "x86_64-apple-darwin", host_platform(ctx_fake)) + return unittest.end(env) + +resolves_macos_x86_64_test = unittest.make(_resolves_macos_x86_64_test_impl) + +def _resolves_windows_regardless_of_arch_test_impl(ctx): + env = unittest.begin(ctx) + + # Only one Windows platform is published (x86_64-pc-windows-msvc); windows should resolve to + # it without even consulting os.arch. + ctx_fake = _fake_repository_ctx("windows 10", "x86_64") + asserts.equals(env, "x86_64-pc-windows-msvc", host_platform(ctx_fake)) + return unittest.end(env) + +resolves_windows_regardless_of_arch_test = unittest.make(_resolves_windows_regardless_of_arch_test_impl) + +def _fails_on_unsupported_arch_test_impl(ctx): + env = analysistest.begin(ctx) + asserts.expect_failure(env, "unsupported CPU architecture") + return analysistest.end(env) + +def _unsupported_arch_lookup_impl(_ctx): + host_platform(_fake_repository_ctx("linux", "riscv64")) + return [DefaultInfo()] + +_unsupported_arch_lookup_rule = rule(implementation = _unsupported_arch_lookup_impl) + +_fails_on_unsupported_arch_test = analysistest.make(_fails_on_unsupported_arch_test_impl, expect_failure = True) + +def _fails_on_unsupported_os_test_impl(ctx): + env = analysistest.begin(ctx) + asserts.expect_failure(env, "unsupported host OS") + return analysistest.end(env) + +def _unsupported_os_lookup_impl(_ctx): + host_platform(_fake_repository_ctx("plan9", "x86_64")) + return [DefaultInfo()] + +_unsupported_os_lookup_rule = rule(implementation = _unsupported_os_lookup_impl) + +_fails_on_unsupported_os_test = analysistest.make(_fails_on_unsupported_os_test_impl, expect_failure = True) + +def host_repo_test_suite(name): + """Registers all host_repo unit tests as a single test_suite target. + + Args: + name: name of the generated test_suite. + """ + _unsupported_arch_lookup_rule(name = "unsupported_arch_lookup", tags = ["manual"]) + _fails_on_unsupported_arch_test( + name = "fails_on_unsupported_arch_test", + target_under_test = ":unsupported_arch_lookup", + ) + _unsupported_os_lookup_rule(name = "unsupported_os_lookup", tags = ["manual"]) + _fails_on_unsupported_os_test( + name = "fails_on_unsupported_os_test", + target_under_test = ":unsupported_os_lookup", + ) + unittest.suite( + "host_repo_pure_tests", + resolves_linux_x86_64_test, + resolves_linux_aarch64_test, + resolves_macos_arm64_alternate_spelling_test, + resolves_macos_x86_64_test, + resolves_windows_regardless_of_arch_test, + ) + native.test_suite( + name = name, + tests = [ + ":host_repo_pure_tests", + ":fails_on_unsupported_arch_test", + ":fails_on_unsupported_os_test", + ], + ) From cad8703fe9c4435d9f5a3b79c404c73e113a34ea Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 3 Jul 2026 22:01:14 +0000 Subject: [PATCH 2/9] Fix arm64 arch normalization in gleam_host's host_platform repository_ctx.os.arch on real Apple Silicon macOS reports "arm64" (confirmed by CI on this PR), but _ARCH mapped it to itself instead of normalizing to PLATFORMS' "aarch64" spelling, so host_platform returned the non-existent "arm64-apple-darwin" instead of "aarch64-apple-darwin". --- gleam/private/host_repo.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gleam/private/host_repo.bzl b/gleam/private/host_repo.bzl index bff74df..28883aa 100644 --- a/gleam/private/host_repo.bzl +++ b/gleam/private/host_repo.bzl @@ -16,7 +16,7 @@ load("//gleam/private:toolchains_repo.bzl", "PLATFORMS") # erlang/private/hermetic_erlang_repository.bzl normalizes) -- accept both. _ARCH = { "aarch64": "aarch64", - "arm64": "arm64", + "arm64": "aarch64", "x86_64": "x86_64", "amd64": "x86_64", } From 71b44a996dd3c63bca51eb7a9de6ba5a769f002d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 3 Jul 2026 22:09:59 +0000 Subject: [PATCH 3/9] Complete Phase 4: pin hermetic OTP checksums, determinism check, ARCHITECTURE.md - Pin real, CI-observed sha256 checksums for the hermetic Erlang/OTP toolchain: the built-in default (OTP-28.1, linux_amd64 + macos_arm64) and examples/hermetic_erlang's explicit pin (OTP-29.0.2, same two platforms -- CI's macos-latest runner is Apple Silicon). Platforms CI doesn't exercise (linux_arm64, macos_amd64) remain unpinned until observed on a real run. - Add a CI step building examples/smoke's //:my_app twice (with a `bazel clean` in between to force real re-execution, not a cache hit) and diffing the two escripts byte-for-byte, to catch nondeterminism in gleam_binary's output. - Add ARCHITECTURE.md documenting the repo's layout and its four-layer test taxonomy (Starlark unit tests, gleam_gazelle's Go tests/golden files, example projects, CI-only verification steps), linked from README.md and CONTRIBUTING.md. --- .github/workflows/ci.yaml | 21 ++++++ ARCHITECTURE.md | 93 +++++++++++++++++++++++++++ CONTRIBUTING.md | 3 + README.md | 3 +- examples/hermetic_erlang/MODULE.bazel | 9 ++- gleam/extensions.bzl | 12 ++-- 6 files changed, 134 insertions(+), 7 deletions(-) create mode 100644 ARCHITECTURE.md diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e22d64f..11d0bc5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -100,6 +100,27 @@ jobs: run: | echo "Running tests in examples/smoke directory" bazel test --test_output=errors //... + - name: Verify examples/smoke's //:my_app build is deterministic + working-directory: examples/smoke + run: | + # `bazel clean` (not --expunge) drops the output tree so the actions genuinely + # re-execute, while keeping already-fetched external repos (Erlang/OTP, the gleam + # toolchain, Hex packages) so this doesn't re-download anything. Two builds of the + # same target producing byte-different escripts would mean gleam_binary embeds + # something nondeterministic (a timestamp, a build-machine path, map/set iteration + # order, ...), which would silently break remote caching and bit-for-bit reproducible + # builds for every downstream user, not just this example. + bazel build //:my_app + cp -L bazel-bin/my_app /tmp/my_app_build_1 + bazel clean + bazel build //:my_app + cp -L bazel-bin/my_app /tmp/my_app_build_2 + if ! cmp -s /tmp/my_app_build_1 /tmp/my_app_build_2; then + echo "FAIL: two independent builds of //:my_app produced different output bytes" >&2 + cmp /tmp/my_app_build_1 /tmp/my_app_build_2 || true + exit 1 + fi + echo "PASS: //:my_app build is deterministic byte-for-byte across a clean rebuild" - name: Run Bazel Tests - examples/nested_smoke Directory working-directory: examples/nested_smoke run: | diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..9825024 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,93 @@ +# Architecture + +An overview of how this repo is laid out and how its test suite is organized. See +[README.md](README.md) for usage and [CONTRIBUTING.md](CONTRIBUTING.md) for the contributor +workflow. + +## Layout + +- **`gleam/`** — the public rules (`gleam_library`, `gleam_binary`, `gleam_test`, + `gleam_format_test`, `gleam_release`, `gleam_standalone_release`, `gleam_package`, all + re-exported from `gleam/defs.bzl`), the toolchain type and rule (`gleam/toolchain.bzl`), and + the bzlmod module extension (`gleam/extensions.bzl`) that downstream `MODULE.bazel` files use + to register a Gleam toolchain and Hex packages. `gleam/private/` holds implementation details + not meant to be depended on directly: toolchain/Hex-package repository rules (including + `gleam_host`, which re-exposes the host platform's toolchain repo under one unconditional + name), version/manifest.toml parsers, and similar. +- **`erlang/`** — the Erlang/OTP toolchain type, split the same way: a hermetic repository rule + that downloads a prebuilt OTP release (`erlang/private/hermetic_erlang_repository.bzl`, + the default) and a `local_erlang_repository.bzl` alternative that discovers Erlang/OTP on the + host's `PATH` instead (opt-in via `gleam.local_erlang_toolchain()`). +- **`gleam_gazelle/`** — a [Gazelle](https://github.com/bazelbuild/bazel-gazelle) language + extension (Go) that generates `gleam_package(...)` targets from `gleam.toml` + + `src/**/*.gleam` directories, so BUILD files don't need to be hand-written. +- **`examples/`** — runnable, CI-exercised projects, each demonstrating one thing end-to-end + (see [Test taxonomy](#test-taxonomy) below, item 3). +- **`docs/`** — `rules.md`, generated from `gleam/defs.bzl`'s docstrings via + [Stardoc](https://github.com/bazelbuild/stardoc) (`bazel run //docs:update` regenerates it; + `bazel test //docs:rules_test` checks it's up to date). +- **`test/unit/`** — Starlark unit tests for private helpers (see below). + +## Test taxonomy + +Four layers, each catching a different class of regression: + +1. **Starlark unit tests** (`test/unit/*`, bazel-skylib's `unittest`/`analysistest`) — test pure + Starlark functions extracted from repository rules and macros (version parsing, manifest.toml + parsing, the otp-tree/erl-invocation resolution `gleam_release` depends on, `gleam_host`'s + host-platform detection) directly, without needing a full build or a real toolchain. Also + used to prove a `fail()` actually fires for a bad input, via + `analysistest.make(expect_failure = True)` plus `asserts.expect_failure(env, msg)`. These are + the cheapest and fastest layer, and the first to write when adding logic to a repository rule + or module extension — the majority of these functions were literally extracted from an + existing rule implementation specifically to make them unit-testable in isolation. +2. **`gleam_gazelle`'s own Go unit tests + golden files** (`gleam_gazelle/*_test.go`, + `gleam_gazelle/testdata/golden/*`) — since the Gazelle extension is Go, not Starlark, it's + tested the same way `bazel-gazelle`'s own language extensions are: each fixture directory + under `testdata/golden/` pairs a `gleam.toml` + sources with a checked-in `BUILD.want`, and + `go test ./gleam_gazelle/... -run TestGolden -update` regenerates them after an intentional + output change. This calls `GenerateRules` directly and does not include the `load(...)` + statement Gazelle's own resolver inserts — that's covered separately by a real compiled + `gazelle_binary` run against `gleam_gazelle/testdata/sample_package` in CI (see below). +3. **Example projects** (`examples/*`, exercised by `bazel test //...`/`bazel build //...` in + CI on both Linux and macOS) — end-to-end proof that the public rules actually work together + under real Bazel, real toolchains, and (for the Hex-dependent examples) real downloaded + packages. Each example is scoped to demonstrate one thing: + - **[smoke](examples/smoke)** — the minimal `gleam_package` example from the README's Quick + start. + - **[gazelle_smoke](examples/gazelle_smoke)** — wiring `gleam_gazelle` into a downstream + project's own `gazelle_binary`. + - **[nested_smoke](examples/nested_smoke)** — a package nested below the repo root, and the + one example that opts out of the hermetic Erlang toolchain via + `gleam.local_erlang_toolchain()` (so it needs a system Erlang on `PATH`; see the "Set up + Erlang" CI steps). + - **[web_service](examples/web_service)** — a runnable HTTP service, a `manifest.toml`-driven + bulk Hex dependency declaration (`gleam.hex_manifest(...)`), and (Linux only) a Docker-based + proof that `gleam_release`'s bundled OTP tree makes the result portable to a machine with no + Erlang installed at all. + - **[standalone_cli](examples/standalone_cli)** — same portability proof as `web_service`, + for `gleam_standalone_release` (which requires the hermetic toolchain outright). + - **[hermetic_erlang](examples/hermetic_erlang)** — pins a specific, non-default + `otp_version` via `gleam.erlang_toolchain(...)`, proving a second OTP release works + end-to-end (not just the built-in default). + - **[expect_fail](examples/expect_fail)** — deliberately broken fixtures (a failing test, a + misformatted file); CI asserts `bazel test` actually fails on them, proving `gleam_test`/ + `gleam_format_test` detect real failures rather than always passing. + - **[hello_world](examples/hello_world)** — the smallest possible `gleam_binary` (escript), + including a check that the built escript actually runs. +4. **CI-only verification steps** (`.github/workflows/ci.yaml`, no corresponding + `bazel test` target) — checks that don't fit cleanly into a hermetic Bazel test, either + because they need a genuinely fresh environment (the Docker-based portability checks above) + or because they exercise a real compiled binary rather than in-process Go code (the + `gazelle_binary` vs. `testdata/sample_package` diff, `bazel run @gleam_host//:gleam -- +--version`). + +When adding a rule or macro with any non-trivial branching logic, prefer extracting the logic +into a plain function and covering it with layer 1 first — it's the cheapest to write and the +fastest to run. The macOS hermetic Erlang arch-naming bug and `examples/hermetic_erlang`'s +never-actually-published OTP pin were both only caught after a real build broke, because the +logic at fault lived inside a repository rule's download/extract path rather than a +separately-testable function; `gleam_host`'s own `arm64`→`aarch64` arch-normalization mistake, +by contrast, was caught immediately by its layer-1 unit test the first time it ran under real +Bazel on a real macOS runner (`repository_ctx.os.arch` reports `"arm64"` there, not `"aarch64"`) +-- exactly the kind of bug this layer exists to catch cheaply. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 079bb2a..6aa6761 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,8 @@ # How to Contribute +See [ARCHITECTURE.md](ARCHITECTURE.md) for how the repo is laid out and how its test suite is +organized before diving in. + ## Using devcontainers If you are using [devcontainers](https://code.visualstudio.com/docs/devcontainers/containers) diff --git a/README.md b/README.md index bcc5622..35dae7e 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # Bazel rules for Gleam Bazel rules for building, testing, and packaging [Gleam](https://gleam.run) projects on the -Erlang/OTP target. +Erlang/OTP target. See [ARCHITECTURE.md](ARCHITECTURE.md) for how the repo is laid out and how +its test suite is organized. ## Status diff --git a/examples/hermetic_erlang/MODULE.bazel b/examples/hermetic_erlang/MODULE.bazel index 54eef0c..5e1bd9e 100644 --- a/examples/hermetic_erlang/MODULE.bazel +++ b/examples/hermetic_erlang/MODULE.bazel @@ -17,10 +17,15 @@ gleam.toolchain(version = "1.17.0") # one, so CI actually exercises a second OTP release end-to-end and can catch BEAM-compat # regressions across versions (gleam itself requires OTP 27.0+, so this can't go below that). # Confirmed available on both builds.hex.pm (Linux) and erlef/otp_builds (macOS) before -# picking it. sha256 is not yet pinned -- the first CI run against this version will download -# unverified and print the actual checksum to pin here. +# picking it. Both checksums below are what CI actually observed downloading this release +# unverified (CI's macos-latest runner is Apple Silicon, hence macos_arm64 rather than +# macos_amd64). gleam.erlang_toolchain( otp_version = "29.0.2", + sha256 = { + "linux_amd64": "2e9d154a52de72653a1a3c1269b10df3b4ff5e058753e167bc92602681038e9a", + "macos_arm64": "ec6e09dee496adf646b0a22eef26b769e7f8032c83dd10292c58d50fb38e2210", + }, ) gleam.hex_package( name = "gleam_stdlib", diff --git a/gleam/extensions.bzl b/gleam/extensions.bzl index eabe013..5108c56 100644 --- a/gleam/extensions.bzl +++ b/gleam/extensions.bzl @@ -60,12 +60,16 @@ load(":repositories.bzl", "gleam_register_toolchains") _DEFAULT_NAME = "gleam" # Used when Erlang is hermetic by default (no gleam.erlang_toolchain(...) call) -- see -# erlang/private/hermetic_erlang_repository.bzl. No checksum is pinned yet for this version -- -# the first CI run downloads unverified and prints the observed checksum, same as an explicit -# gleam.erlang_toolchain(...) call with no matching sha256 entry. +# erlang/private/hermetic_erlang_repository.bzl. Checksums below are the actual digests CI +# observed downloading OTP-28.1 unverified (printed by hermetic_erlang_repository.bzl's own +# NOTE: log line); platforms CI doesn't run on (linux_arm64, macos_amd64) are left unpinned +# until a run on that platform observes them, same as any other platform's first use. _DEFAULT_HERMETIC_OTP_VERSION = "28.1" _DEFAULT_HERMETIC_OS_VERSION = "ubuntu-22.04" -_DEFAULT_HERMETIC_SHA256 = {} +_DEFAULT_HERMETIC_SHA256 = { + "linux_amd64": "60c1083df707642f20831c762a68db191984314fef1d4d80b05bb8caf70b70bf", + "macos_arm64": "831fcfb46929c752abf0a921409a46b306e7019061653323fab7f0958ad4db25", +} gleam_toolchain = tag_class(attrs = { "name": attr.string(doc = """\ From 120a4a700368c18e3638a54a7ee12f25c68361d5 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 3 Jul 2026 22:16:06 +0000 Subject: [PATCH 4/9] Fix nondeterministic escript output in escript_builder.erl The new determinism CI check (bazel build //:my_app twice with a `bazel clean` in between) caught a real bug: escript:create's zip archive stamps each entry with the current wall-clock time when no FileInfo is given, not anything about the input files, so two builds run even a few seconds apart produced byte-different escripts. Reproduced and verified standalone with erl: the old code differs across a 3s gap (char 164, line 4 -- matching the exact CI failure), the fixed code doesn't. Fixed by passing an explicit, fixed file_info() (atime/mtime/ctime pinned to a constant) for every archive entry, and sorting file:list_dir/1's output (also unordered, another potential source of nondeterminism) before building the archive. --- gleam/private/escript_builder.erl | 34 +++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/gleam/private/escript_builder.erl b/gleam/private/escript_builder.erl index 963c09f..25d7c4a 100644 --- a/gleam/private/escript_builder.erl +++ b/gleam/private/escript_builder.erl @@ -1,6 +1,8 @@ -module(escript_builder). -export([main/1]). +-include_lib("kernel/include/file.hrl"). + main([OutFile, PackageName, EntryModule, EntryFunction | Files]) -> % Read files ArchiveFiles = lists:flatmap(fun(Path) -> @@ -32,11 +34,15 @@ main([OutFile, PackageName, EntryModule, EntryFunction | Files]) -> process_path(Path) -> case filelib:is_dir(Path) of true -> + % file:list_dir/1 does not guarantee any particular order (it reflects whatever + % order the underlying filesystem happens to return directory entries in, which can + % differ between a from-scratch build and a rebuild of the same sources) -- sort so + % the resulting escript's zip archive has a deterministic entry order. {ok, Files} = file:list_dir(Path), lists:flatmap(fun(File) -> Full = filename:join(Path, File), process_path(Full) - end, Files); + end, lists:sort(Files)); false -> BaseName = filename:basename(Path), IsBeamOrApp = filename:extension(Path) == ".beam" orelse filename:extension(Path) == ".app", @@ -44,7 +50,8 @@ process_path(Path) -> NotInternal = string:str(BaseName, "@@") == 0, if IsBeamOrApp andalso NotInternal -> - [{BaseName, read_file(Path)}]; + Bin = read_file(Path), + [{BaseName, Bin, archive_file_info(Bin)}]; true -> [] end @@ -53,3 +60,26 @@ process_path(Path) -> read_file(Path) -> {ok, Bin} = file:read_file(Path), Bin. + +% escript:create/2's {archive, Files, _} accepts {Name, Bin} or {Name, Bin, FileInfo}; without +% an explicit FileInfo, the zip archive it builds stamps each entry with the current wall-clock +% time (when this escript_builder run happens to execute), not anything about the input files -- +% two otherwise-identical builds run even a couple of seconds apart get a byte-different escript +% as a result. Pin every field to a fixed value instead. +archive_file_info(Bin) -> + FixedTime = {{1980, 1, 1}, {0, 0, 0}}, + #file_info{ + size = byte_size(Bin), + type = regular, + access = read_write, + atime = FixedTime, + mtime = FixedTime, + ctime = FixedTime, + mode = 8#100644, + links = 1, + major_device = 0, + minor_device = 0, + inode = 0, + uid = 0, + gid = 0 + }. From 41558be3b0f00bc01e87cc81bdafb72c512b858c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 3 Jul 2026 22:20:39 +0000 Subject: [PATCH 5/9] Add temporary diagnostic to determinism check The escript_builder.erl fix (previous commit) addressed one real source of nondeterminism (zip entry timestamps) but the determinism check is still failing in CI, now at a different byte offset -- something else still differs. Add a temporary diagnostic that extracts both escripts' archives and reports which named entry's bytes actually differ, so the next fix is based on real evidence instead of another guess. --- .github/workflows/ci.yaml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 11d0bc5..1c09978 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -118,6 +118,33 @@ jobs: if ! cmp -s /tmp/my_app_build_1 /tmp/my_app_build_2; then echo "FAIL: two independent builds of //:my_app produced different output bytes" >&2 cmp /tmp/my_app_build_1 /tmp/my_app_build_2 || true + echo "--- diagnostic: diffing escript archive entries ---" >&2 + erl -noshell -eval ' + {ok, S1} = file:read_file("/tmp/my_app_build_1"), + {ok, S2} = file:read_file("/tmp/my_app_build_2"), + {ok, E1} = escript:extract("/tmp/my_app_build_1", []), + {ok, E2} = escript:extract("/tmp/my_app_build_2", []), + {archive, A1} = lists:keyfind(archive, 1, E1), + {archive, A2} = lists:keyfind(archive, 1, E2), + {ok, Z1} = zip:extract(A1, [memory]), + {ok, Z2} = zip:extract(A2, [memory]), + M1 = maps:from_list(Z1), + M2 = maps:from_list(Z2), + Names1 = lists:sort(maps:keys(M1)), + Names2 = lists:sort(maps:keys(M2)), + io:format("names only in build_1: ~p~n", [Names1 -- Names2]), + io:format("names only in build_2: ~p~n", [Names2 -- Names1]), + [ begin + B1 = maps:get(N, M1), + B2 = maps:get(N, M2), + case B1 =:= B2 of + true -> ok; + false -> + io:format("DIFFERS: ~s (size ~p vs ~p)~n", [N, byte_size(B1), byte_size(B2)]) + end + end || N <- Names1, maps:is_key(N, M2) ], + halt(). + ' >&2 || true exit 1 fi echo "PASS: //:my_app build is deterministic byte-for-byte across a clean rebuild" From 6481b639d751e7729f025ced5ccc4bff8638b81e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 3 Jul 2026 22:26:05 +0000 Subject: [PATCH 6/9] Strip erlc's embedded absolute paths from .beam files for determinism The diagnostic added in the previous commit showed every single .beam file in the escript archive differed between two builds -- including trivially small ones like my_app.beam. Root-caused locally: erlc embeds the absolute source path (CInf chunk) and compiler cwd (Dbgi chunk) into every .beam file it produces. Gleam's own compiler invokes erlc internally as part of `gleam build`, and Bazel runs each action in a fresh sandbox directory, so that path differs between two otherwise-identical builds of the same target -- reproduced directly with `erlc` compiling identical source from two different directories and diffing the resulting .beam files. Fixed by running beam_lib:strip/1 (which removes these chunks, along with other debug-only metadata not needed at runtime) on every .beam file's bytes before archiving it. Verified end-to-end: compiling the same source in two different directories and building an escript from each now produces byte-identical output; without the fix, they differ. Also removes the temporary archive-diffing diagnostic from ci.yaml added while investigating. --- .github/workflows/ci.yaml | 27 --------------------------- gleam/private/escript_builder.erl | 25 +++++++++++++++++++++---- 2 files changed, 21 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1c09978..11d0bc5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -118,33 +118,6 @@ jobs: if ! cmp -s /tmp/my_app_build_1 /tmp/my_app_build_2; then echo "FAIL: two independent builds of //:my_app produced different output bytes" >&2 cmp /tmp/my_app_build_1 /tmp/my_app_build_2 || true - echo "--- diagnostic: diffing escript archive entries ---" >&2 - erl -noshell -eval ' - {ok, S1} = file:read_file("/tmp/my_app_build_1"), - {ok, S2} = file:read_file("/tmp/my_app_build_2"), - {ok, E1} = escript:extract("/tmp/my_app_build_1", []), - {ok, E2} = escript:extract("/tmp/my_app_build_2", []), - {archive, A1} = lists:keyfind(archive, 1, E1), - {archive, A2} = lists:keyfind(archive, 1, E2), - {ok, Z1} = zip:extract(A1, [memory]), - {ok, Z2} = zip:extract(A2, [memory]), - M1 = maps:from_list(Z1), - M2 = maps:from_list(Z2), - Names1 = lists:sort(maps:keys(M1)), - Names2 = lists:sort(maps:keys(M2)), - io:format("names only in build_1: ~p~n", [Names1 -- Names2]), - io:format("names only in build_2: ~p~n", [Names2 -- Names1]), - [ begin - B1 = maps:get(N, M1), - B2 = maps:get(N, M2), - case B1 =:= B2 of - true -> ok; - false -> - io:format("DIFFERS: ~s (size ~p vs ~p)~n", [N, byte_size(B1), byte_size(B2)]) - end - end || N <- Names1, maps:is_key(N, M2) ], - halt(). - ' >&2 || true exit 1 fi echo "PASS: //:my_app build is deterministic byte-for-byte across a clean rebuild" diff --git a/gleam/private/escript_builder.erl b/gleam/private/escript_builder.erl index 25d7c4a..286f5da 100644 --- a/gleam/private/escript_builder.erl +++ b/gleam/private/escript_builder.erl @@ -45,21 +45,38 @@ process_path(Path) -> end, lists:sort(Files)); false -> BaseName = filename:basename(Path), - IsBeamOrApp = filename:extension(Path) == ".beam" orelse filename:extension(Path) == ".app", + IsBeam = filename:extension(Path) == ".beam", + IsBeamOrApp = IsBeam orelse filename:extension(Path) == ".app", % Check if file contains "@@" NotInternal = string:str(BaseName, "@@") == 0, if IsBeamOrApp andalso NotInternal -> - Bin = read_file(Path), + Bin = read_file(Path, IsBeam), [{BaseName, Bin, archive_file_info(Bin)}]; true -> [] end end. -read_file(Path) -> +read_file(Path, IsBeam) -> {ok, Bin} = file:read_file(Path), - Bin. + case IsBeam of + true -> strip_beam(Bin); + false -> Bin + end. + +% erlc embeds the absolute source path (and, in the debug_info chunk, the compiler's cwd) into +% every .beam file it produces -- Gleam's own compiler invokes erlc internally as part of `gleam +% build`, so this isn't something rules_gleam controls directly. Since Bazel runs each action in +% a fresh sandbox directory, that path differs between two otherwise-identical builds, making +% every compiled module byte-different for a reason that has nothing to do with its actual +% content. beam_lib:strip/1 removes the chunks that carry this (and other debug-only metadata +% not needed at runtime), leaving only what's needed to load and run the module. +strip_beam(Bin) -> + case beam_lib:strip(Bin) of + {ok, {_Module, Stripped}} -> Stripped; + _ -> Bin + end. % escript:create/2's {archive, Files, _} accepts {Name, Bin} or {Name, Bin, FileInfo}; without % an explicit FileInfo, the zip archive it builds stamps each entry with the current wall-clock From 33fab7debbf3f30aee79e80f34b352b398236b09 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 3 Jul 2026 22:30:46 +0000 Subject: [PATCH 7/9] Add byte-offset diagnostic to determinism check The beam_lib:strip fix (previous commit) closed the gap significantly -- the failure now happens much further into the escript (byte 36436 vs the original byte 167) -- but something still differs. Refine the diagnostic to report the first differing byte offset and a snippet of both entries' bytes for whichever named archive entry still differs, since the previous version only reported sizes. --- .github/workflows/ci.yaml | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 11d0bc5..3dcda1d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -118,6 +118,45 @@ jobs: if ! cmp -s /tmp/my_app_build_1 /tmp/my_app_build_2; then echo "FAIL: two independent builds of //:my_app produced different output bytes" >&2 cmp /tmp/my_app_build_1 /tmp/my_app_build_2 || true + echo "--- diagnostic: diffing escript archive entries ---" >&2 + erl -noshell -eval ' + {ok, E1} = escript:extract("/tmp/my_app_build_1", []), + {ok, E2} = escript:extract("/tmp/my_app_build_2", []), + {archive, A1} = lists:keyfind(archive, 1, E1), + {archive, A2} = lists:keyfind(archive, 1, E2), + {ok, Z1} = zip:extract(A1, [memory]), + {ok, Z2} = zip:extract(A2, [memory]), + M1 = maps:from_list(Z1), + M2 = maps:from_list(Z2), + Names1 = lists:sort(maps:keys(M1)), + Names2 = lists:sort(maps:keys(M2)), + io:format("names only in build_1: ~p~n", [Names1 -- Names2]), + io:format("names only in build_2: ~p~n", [Names2 -- Names1]), + [ begin + B1 = maps:get(N, M1), + B2 = maps:get(N, M2), + case B1 =:= B2 of + true -> ok; + false -> + FirstDiff = (fun F(I) -> + case {I > byte_size(B1), I > byte_size(B2)} of + {true, _} -> {shorter1, I}; + {_, true} -> {shorter2, I}; + _ -> + case binary:at(B1, I) =:= binary:at(B2, I) of + true -> F(I + 1); + false -> I + end + end + end)(0), + Snip1 = binary:part(B1, FirstDiff, min(60, byte_size(B1) - FirstDiff)), + Snip2 = binary:part(B2, FirstDiff, min(60, byte_size(B2) - FirstDiff)), + io:format("DIFFERS: ~s (size ~p vs ~p, first diff at ~p)~n ~p~n ~p~n", + [N, byte_size(B1), byte_size(B2), FirstDiff, Snip1, Snip2]) + end + end || N <- Names1, maps:is_key(N, M2) ], + halt(). + ' >&2 || true exit 1 fi echo "PASS: //:my_app build is deterministic byte-for-byte across a clean rebuild" From 2a62693ec3f665dc8c5a3670123f66ef8049c581 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 3 Jul 2026 22:38:29 +0000 Subject: [PATCH 8/9] Fix nondeterminism at the source: compile with erlc's +deterministic The refined diagnostic (previous commit) pinpointed the exact remaining culprit: gleam_stdlib.beam's "Line" chunk, which beam_lib:strip does not remove. Root-caused locally: Gleam's own codegen emits `-file("/src/foo.gleam", N).` attributes, and erlc encodes that path into the Line chunk's filename table regardless of stripping -- reproduced directly by compiling identical source containing such an attribute from two different directories and diffing the .beam output. The clean fix is to prevent this at the compiler invocation instead of trying to scrub it afterwards: erlc's `+deterministic` compile option (or, for `gleam compile-package`'s internal erlc calls, the equivalent ERL_COMPILER_OPTIONS=[deterministic] environment variable, which `compile:file` reads regardless of caller) stops all absolute-path embedding at the source. Applied to every erlc/gleam-compile action: gleam_library.bzl and gleam_test.bzl's `gleam compile-package` invocations (env var), and gleam_binary.bzl and gleam_test.bzl's direct erlc calls for the escript shim and eunit runner shim (+deterministic flag). Verified end-to-end locally: compiling a package and a shim from two different directories, each containing a `-file()` attribute with a different absolute path, now produces byte- identical escript output; without either fix it doesn't. This makes the beam_lib:strip/1 post-processing added two commits ago (and the diagnostic added after it) unnecessary -- reverted both, keeping only the still-needed zip entry timestamp fix in escript_builder.erl. --- .github/workflows/ci.yaml | 39 ------------------------------- gleam/private/escript_builder.erl | 25 ++++---------------- gleam/private/gleam_binary.bzl | 6 ++++- gleam/private/gleam_library.bzl | 8 +++++++ gleam/private/gleam_test.bzl | 7 +++++- 5 files changed, 23 insertions(+), 62 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3dcda1d..11d0bc5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -118,45 +118,6 @@ jobs: if ! cmp -s /tmp/my_app_build_1 /tmp/my_app_build_2; then echo "FAIL: two independent builds of //:my_app produced different output bytes" >&2 cmp /tmp/my_app_build_1 /tmp/my_app_build_2 || true - echo "--- diagnostic: diffing escript archive entries ---" >&2 - erl -noshell -eval ' - {ok, E1} = escript:extract("/tmp/my_app_build_1", []), - {ok, E2} = escript:extract("/tmp/my_app_build_2", []), - {archive, A1} = lists:keyfind(archive, 1, E1), - {archive, A2} = lists:keyfind(archive, 1, E2), - {ok, Z1} = zip:extract(A1, [memory]), - {ok, Z2} = zip:extract(A2, [memory]), - M1 = maps:from_list(Z1), - M2 = maps:from_list(Z2), - Names1 = lists:sort(maps:keys(M1)), - Names2 = lists:sort(maps:keys(M2)), - io:format("names only in build_1: ~p~n", [Names1 -- Names2]), - io:format("names only in build_2: ~p~n", [Names2 -- Names1]), - [ begin - B1 = maps:get(N, M1), - B2 = maps:get(N, M2), - case B1 =:= B2 of - true -> ok; - false -> - FirstDiff = (fun F(I) -> - case {I > byte_size(B1), I > byte_size(B2)} of - {true, _} -> {shorter1, I}; - {_, true} -> {shorter2, I}; - _ -> - case binary:at(B1, I) =:= binary:at(B2, I) of - true -> F(I + 1); - false -> I - end - end - end)(0), - Snip1 = binary:part(B1, FirstDiff, min(60, byte_size(B1) - FirstDiff)), - Snip2 = binary:part(B2, FirstDiff, min(60, byte_size(B2) - FirstDiff)), - io:format("DIFFERS: ~s (size ~p vs ~p, first diff at ~p)~n ~p~n ~p~n", - [N, byte_size(B1), byte_size(B2), FirstDiff, Snip1, Snip2]) - end - end || N <- Names1, maps:is_key(N, M2) ], - halt(). - ' >&2 || true exit 1 fi echo "PASS: //:my_app build is deterministic byte-for-byte across a clean rebuild" diff --git a/gleam/private/escript_builder.erl b/gleam/private/escript_builder.erl index 286f5da..25d7c4a 100644 --- a/gleam/private/escript_builder.erl +++ b/gleam/private/escript_builder.erl @@ -45,38 +45,21 @@ process_path(Path) -> end, lists:sort(Files)); false -> BaseName = filename:basename(Path), - IsBeam = filename:extension(Path) == ".beam", - IsBeamOrApp = IsBeam orelse filename:extension(Path) == ".app", + IsBeamOrApp = filename:extension(Path) == ".beam" orelse filename:extension(Path) == ".app", % Check if file contains "@@" NotInternal = string:str(BaseName, "@@") == 0, if IsBeamOrApp andalso NotInternal -> - Bin = read_file(Path, IsBeam), + Bin = read_file(Path), [{BaseName, Bin, archive_file_info(Bin)}]; true -> [] end end. -read_file(Path, IsBeam) -> +read_file(Path) -> {ok, Bin} = file:read_file(Path), - case IsBeam of - true -> strip_beam(Bin); - false -> Bin - end. - -% erlc embeds the absolute source path (and, in the debug_info chunk, the compiler's cwd) into -% every .beam file it produces -- Gleam's own compiler invokes erlc internally as part of `gleam -% build`, so this isn't something rules_gleam controls directly. Since Bazel runs each action in -% a fresh sandbox directory, that path differs between two otherwise-identical builds, making -% every compiled module byte-different for a reason that has nothing to do with its actual -% content. beam_lib:strip/1 removes the chunks that carry this (and other debug-only metadata -% not needed at runtime), leaving only what's needed to load and run the module. -strip_beam(Bin) -> - case beam_lib:strip(Bin) of - {ok, {_Module, Stripped}} -> Stripped; - _ -> Bin - end. + Bin. % escript:create/2's {archive, Files, _} accepts {Name, Bin} or {Name, Bin, FileInfo}; without % an explicit FileInfo, the zip archive it builds stamps each entry with the current wall-clock diff --git a/gleam/private/gleam_binary.bzl b/gleam/private/gleam_binary.bzl index d4f0421..6bd48a0 100644 --- a/gleam/private/gleam_binary.bzl +++ b/gleam/private/gleam_binary.bzl @@ -28,7 +28,11 @@ def _gleam_binary_impl(ctx): ctx.actions.run( executable = erlc_path, - arguments = ["-o", shim_beam.dirname, shim_src.path], + # +deterministic stops erlc from embedding the compiling machine's absolute source path + # into the shim's .beam (debug/line-number metadata) -- without it, Bazel's per-action + # sandbox path (which differs between separate builds of the same target) would leak + # into the compiled output and make gleam_binary's result non-reproducible. + arguments = ["+deterministic", "-o", shim_beam.dirname, shim_src.path], inputs = [shim_src], outputs = [shim_beam], mnemonic = "CompileGleamShim", diff --git a/gleam/private/gleam_library.bzl b/gleam/private/gleam_library.bzl index 230dbca..a072be9 100644 --- a/gleam/private/gleam_library.bzl +++ b/gleam/private/gleam_library.bzl @@ -93,6 +93,14 @@ def _gleam_library_impl(ctx): tools = depset([gleam_exe_wrapper, underlying_gleam_tool]), inputs = inputs_depset, outputs = [compiled_dir], + # `gleam compile-package` shells out to erlc internally; erlc (like any caller of the + # `compile` module) reads ERL_COMPILER_OPTIONS and applies it to every compilation. + # `deterministic` stops it from embedding the compiling machine's absolute source path + # into each .beam file's debug/line-number metadata -- without it, Bazel's per-action + # sandbox path (which differs between separate builds of the same target) would leak + # into the compiled output and make it non-reproducible for reasons that have nothing + # to do with the actual source. + env = {"ERL_COMPILER_OPTIONS": "[deterministic]"}, progress_message = "Compiling Gleam package: " + package_name, mnemonic = "GleamCompilePackage", ) diff --git a/gleam/private/gleam_test.bzl b/gleam/private/gleam_test.bzl index 1c93a4c..71303a2 100644 --- a/gleam/private/gleam_test.bzl +++ b/gleam/private/gleam_test.bzl @@ -82,6 +82,10 @@ def _gleam_test_impl(ctx): tools = depset([gleam_exe_wrapper, underlying_gleam_tool]), inputs = inputs_depset, outputs = [compiled_dir], + # See gleam_library.bzl's identical env setting for why: erlc (invoked internally by + # `gleam compile-package`) otherwise embeds Bazel's per-action sandbox path into the + # compiled output. + env = {"ERL_COMPILER_OPTIONS": "[deterministic]"}, progress_message = "Compiling Gleam tests: " + package_name, mnemonic = "GleamCompileTest", ) @@ -96,7 +100,8 @@ def _gleam_test_impl(ctx): ctx.actions.run( executable = erlc_path, - arguments = ["-o", runner_beam_dir.path, runner_src.path], + # See gleam_binary.bzl's identical +deterministic flag for why. + arguments = ["+deterministic", "-o", runner_beam_dir.path, runner_src.path], inputs = [runner_src], outputs = [runner_beam_dir], mnemonic = "CompileGleamEunitRunner", From 85f1bcf9bb199a66ec1d81791e74d7926b06e043 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 3 Jul 2026 22:46:41 +0000 Subject: [PATCH 9/9] Test host_platform's amd64 alternate spelling, not just arm64 _ARCH maps both "arm64"->"aarch64" and "amd64"->"x86_64", but only the arm64 side had test coverage (which is exactly what caught the real arm64->aarch64 bug earlier). Add the missing symmetric cases for amd64 on both Linux and macOS, so a typo there wouldn't go unnoticed the same way. --- test/unit/host_repo/host_repo_test.bzl | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/unit/host_repo/host_repo_test.bzl b/test/unit/host_repo/host_repo_test.bzl index a7ad924..62f640e 100644 --- a/test/unit/host_repo/host_repo_test.bzl +++ b/test/unit/host_repo/host_repo_test.bzl @@ -48,6 +48,25 @@ def _resolves_macos_x86_64_test_impl(ctx): resolves_macos_x86_64_test = unittest.make(_resolves_macos_x86_64_test_impl) +def _resolves_linux_amd64_alternate_spelling_test_impl(ctx): + env = unittest.begin(ctx) + + # Same alternate-spelling concern as the arm64 test above, for the other pair: some + # Bazel/OS combinations report "amd64" instead of "x86_64". + ctx_fake = _fake_repository_ctx("linux", "amd64") + asserts.equals(env, "x86_64-unknown-linux-gnu", host_platform(ctx_fake)) + return unittest.end(env) + +resolves_linux_amd64_alternate_spelling_test = unittest.make(_resolves_linux_amd64_alternate_spelling_test_impl) + +def _resolves_macos_amd64_alternate_spelling_test_impl(ctx): + env = unittest.begin(ctx) + ctx_fake = _fake_repository_ctx("mac os x", "amd64") + asserts.equals(env, "x86_64-apple-darwin", host_platform(ctx_fake)) + return unittest.end(env) + +resolves_macos_amd64_alternate_spelling_test = unittest.make(_resolves_macos_amd64_alternate_spelling_test_impl) + def _resolves_windows_regardless_of_arch_test_impl(ctx): env = unittest.begin(ctx) @@ -107,6 +126,8 @@ def host_repo_test_suite(name): resolves_linux_aarch64_test, resolves_macos_arm64_alternate_spelling_test, resolves_macos_x86_64_test, + resolves_linux_amd64_alternate_spelling_test, + resolves_macos_amd64_alternate_spelling_test, resolves_windows_regardless_of_arch_test, ) native.test_suite(