Add gleam_host convenience repository; complete Phase 4

.github/workflows/ci.yaml

@@ -66,6 +66,15 @@ jobs:
         run: |
           echo "Running the gleam Gazelle language extension's Go unit tests"
           bazel test --test_output=errors //gleam_gazelle/...
+      - name: Verify @gleam_host//:gleam is directly runnable
+        run: |
+          # gleam_host_repo (gleam/private/host_repo.bzl) detects the host platform without
+          # needing to know its OS/arch string; prove the resulting binary is real and matches
+          # the pinned toolchain version, not just that the target resolves.
+          echo "Running bazel run @gleam_host//:gleam -- --version"
+          output="$(bazel run @gleam_host//:gleam -- --version)"
+          echo "$output"
+          echo "$output" | grep -q "gleam 1.17.0"
       - name: Verify gazelle generates a BUILD file for a fresh Gleam package
         run: |
           echo "Running bazel run //:gazelle against gleam_gazelle/testdata/sample_package (no BUILD.bazel checked in)"
@@ -91,6 +100,27 @@ jobs:
         run: |
           echo "Running tests in examples/smoke directory"
           bazel test --test_output=errors //...
+      - name: Verify examples/smoke's //:my_app build is deterministic
+        working-directory: examples/smoke
+        run: |
+          # `bazel clean` (not --expunge) drops the output tree so the actions genuinely
+          # re-execute, while keeping already-fetched external repos (Erlang/OTP, the gleam
+          # toolchain, Hex packages) so this doesn't re-download anything. Two builds of the
+          # same target producing byte-different escripts would mean gleam_binary embeds
+          # something nondeterministic (a timestamp, a build-machine path, map/set iteration
+          # order, ...), which would silently break remote caching and bit-for-bit reproducible
+          # builds for every downstream user, not just this example.
+          bazel build //:my_app
+          cp -L bazel-bin/my_app /tmp/my_app_build_1
+          bazel clean
+          bazel build //:my_app
+          cp -L bazel-bin/my_app /tmp/my_app_build_2
+          if ! cmp -s /tmp/my_app_build_1 /tmp/my_app_build_2; then
+            echo "FAIL: two independent builds of //:my_app produced different output bytes" >&2
+            cmp /tmp/my_app_build_1 /tmp/my_app_build_2 || true
+            exit 1
+          fi
+          echo "PASS: //:my_app build is deterministic byte-for-byte across a clean rebuild"
       - name: Run Bazel Tests - examples/nested_smoke Directory
         working-directory: examples/nested_smoke
         run: |

ARCHITECTURE.md

@@ -0,0 +1,93 @@
+# Architecture
+
+An overview of how this repo is laid out and how its test suite is organized. See
+[README.md](README.md) for usage and [CONTRIBUTING.md](CONTRIBUTING.md) for the contributor
+workflow.
+
+## Layout
+
+- **`gleam/`** — the public rules (`gleam_library`, `gleam_binary`, `gleam_test`,
+  `gleam_format_test`, `gleam_release`, `gleam_standalone_release`, `gleam_package`, all
+  re-exported from `gleam/defs.bzl`), the toolchain type and rule (`gleam/toolchain.bzl`), and
+  the bzlmod module extension (`gleam/extensions.bzl`) that downstream `MODULE.bazel` files use
+  to register a Gleam toolchain and Hex packages. `gleam/private/` holds implementation details
+  not meant to be depended on directly: toolchain/Hex-package repository rules (including
+  `gleam_host`, which re-exposes the host platform's toolchain repo under one unconditional
+  name), version/manifest.toml parsers, and similar.
+- **`erlang/`** — the Erlang/OTP toolchain type, split the same way: a hermetic repository rule
+  that downloads a prebuilt OTP release (`erlang/private/hermetic_erlang_repository.bzl`,
+  the default) and a `local_erlang_repository.bzl` alternative that discovers Erlang/OTP on the
+  host's `PATH` instead (opt-in via `gleam.local_erlang_toolchain()`).
+- **`gleam_gazelle/`** — a [Gazelle](https://github.com/bazelbuild/bazel-gazelle) language
+  extension (Go) that generates `gleam_package(...)` targets from `gleam.toml` +
+  `src/**/*.gleam` directories, so BUILD files don't need to be hand-written.
+- **`examples/`** — runnable, CI-exercised projects, each demonstrating one thing end-to-end
+  (see [Test taxonomy](#test-taxonomy) below, item 3).
+- **`docs/`** — `rules.md`, generated from `gleam/defs.bzl`'s docstrings via
+  [Stardoc](https://github.com/bazelbuild/stardoc) (`bazel run //docs:update` regenerates it;
+  `bazel test //docs:rules_test` checks it's up to date).
+- **`test/unit/`** — Starlark unit tests for private helpers (see below).
+
+## Test taxonomy
+
+Four layers, each catching a different class of regression:
+
+1. **Starlark unit tests** (`test/unit/*`, bazel-skylib's `unittest`/`analysistest`) — test pure
+   Starlark functions extracted from repository rules and macros (version parsing, manifest.toml
+   parsing, the otp-tree/erl-invocation resolution `gleam_release` depends on, `gleam_host`'s
+   host-platform detection) directly, without needing a full build or a real toolchain. Also
+   used to prove a `fail()` actually fires for a bad input, via
+   `analysistest.make(expect_failure = True)` plus `asserts.expect_failure(env, msg)`. These are
+   the cheapest and fastest layer, and the first to write when adding logic to a repository rule
+   or module extension — the majority of these functions were literally extracted from an
+   existing rule implementation specifically to make them unit-testable in isolation.
+2. **`gleam_gazelle`'s own Go unit tests + golden files** (`gleam_gazelle/*_test.go`,
+   `gleam_gazelle/testdata/golden/*`) — since the Gazelle extension is Go, not Starlark, it's
+   tested the same way `bazel-gazelle`'s own language extensions are: each fixture directory
+   under `testdata/golden/` pairs a `gleam.toml` + sources with a checked-in `BUILD.want`, and
+   `go test ./gleam_gazelle/... -run TestGolden -update` regenerates them after an intentional
+   output change. This calls `GenerateRules` directly and does not include the `load(...)`
+   statement Gazelle's own resolver inserts — that's covered separately by a real compiled
+   `gazelle_binary` run against `gleam_gazelle/testdata/sample_package` in CI (see below).
+3. **Example projects** (`examples/*`, exercised by `bazel test //...`/`bazel build //...` in
+   CI on both Linux and macOS) — end-to-end proof that the public rules actually work together
+   under real Bazel, real toolchains, and (for the Hex-dependent examples) real downloaded
+   packages. Each example is scoped to demonstrate one thing:
+   - **[smoke](examples/smoke)** — the minimal `gleam_package` example from the README's Quick
+     start.
+   - **[gazelle_smoke](examples/gazelle_smoke)** — wiring `gleam_gazelle` into a downstream
+     project's own `gazelle_binary`.
+   - **[nested_smoke](examples/nested_smoke)** — a package nested below the repo root, and the
+     one example that opts out of the hermetic Erlang toolchain via
+     `gleam.local_erlang_toolchain()` (so it needs a system Erlang on `PATH`; see the "Set up
+     Erlang" CI steps).
+   - **[web_service](examples/web_service)** — a runnable HTTP service, a `manifest.toml`-driven
+     bulk Hex dependency declaration (`gleam.hex_manifest(...)`), and (Linux only) a Docker-based
+     proof that `gleam_release`'s bundled OTP tree makes the result portable to a machine with no
+     Erlang installed at all.
+   - **[standalone_cli](examples/standalone_cli)** — same portability proof as `web_service`,
+     for `gleam_standalone_release` (which requires the hermetic toolchain outright).
+   - **[hermetic_erlang](examples/hermetic_erlang)** — pins a specific, non-default
+     `otp_version` via `gleam.erlang_toolchain(...)`, proving a second OTP release works
+     end-to-end (not just the built-in default).
+   - **[expect_fail](examples/expect_fail)** — deliberately broken fixtures (a failing test, a
+     misformatted file); CI asserts `bazel test` actually fails on them, proving `gleam_test`/
+     `gleam_format_test` detect real failures rather than always passing.
+   - **[hello_world](examples/hello_world)** — the smallest possible `gleam_binary` (escript),
+     including a check that the built escript actually runs.
+4. **CI-only verification steps** (`.github/workflows/ci.yaml`, no corresponding
+   `bazel test` target) — checks that don't fit cleanly into a hermetic Bazel test, either
+   because they need a genuinely fresh environment (the Docker-based portability checks above)
+   or because they exercise a real compiled binary rather than in-process Go code (the
+   `gazelle_binary` vs. `testdata/sample_package` diff, `bazel run @gleam_host//:gleam --
+--version`).
+
+When adding a rule or macro with any non-trivial branching logic, prefer extracting the logic
+into a plain function and covering it with layer 1 first — it's the cheapest to write and the
+fastest to run. The macOS hermetic Erlang arch-naming bug and `examples/hermetic_erlang`'s
+never-actually-published OTP pin were both only caught after a real build broke, because the
+logic at fault lived inside a repository rule's download/extract path rather than a
+separately-testable function; `gleam_host`'s own `arm64`→`aarch64` arch-normalization mistake,
+by contrast, was caught immediately by its layer-1 unit test the first time it ran under real
+Bazel on a real macOS runner (`repository_ctx.os.arch` reports `"arm64"` there, not `"aarch64"`)
+-- exactly the kind of bug this layer exists to catch cheaply.

CONTRIBUTING.md

@@ -1,5 +1,8 @@
 # How to Contribute
 
+See [ARCHITECTURE.md](ARCHITECTURE.md) for how the repo is laid out and how its test suite is
+organized before diving in.
+
 ## Using devcontainers
 
 If you are using [devcontainers](https://code.visualstudio.com/docs/devcontainers/containers)

MODULE.bazel

@@ -20,7 +20,7 @@ go_sdk.download(version = "1.24.12")
 
 gleam = use_extension("//gleam:extensions.bzl", "gleam")
 gleam.toolchain(version = "1.17.0")
-use_repo(gleam, "gleam_toolchains", "local_config_erlang")
+use_repo(gleam, "gleam_host", "gleam_toolchains", "local_config_erlang")
 
 register_toolchains("@gleam_toolchains//:all")
 

README.md

@@ -1,7 +1,8 @@
 # Bazel rules for Gleam
 
 Bazel rules for building, testing, and packaging [Gleam](https://gleam.run) projects on the
-Erlang/OTP target.
+Erlang/OTP target. See [ARCHITECTURE.md](ARCHITECTURE.md) for how the repo is laid out and how
+its test suite is organized.
 
 ## Status
 
@@ -174,3 +175,7 @@ extension turns any `gleam.toml` + `src/**/*.gleam` directory into a `gleam_pack
 Wire it into your own `gazelle_binary`'s `languages` list alongside `@rules_gleam//gleam_gazelle`
 and run `bazel run //:gazelle` -- see [examples/gazelle_smoke](examples/gazelle_smoke) for a
 worked example of wiring this into a downstream project's own `MODULE.bazel`/`BUILD.bazel`.
+
+To run the `gleam` CLI itself (e.g. `gleam format`) without knowing your machine's OS/arch
+string, add `"gleam_host"` to the extension's `use_repo(...)` call and run
+`bazel run @gleam_host//:gleam -- format`.

examples/hermetic_erlang/MODULE.bazel

@@ -17,10 +17,15 @@ gleam.toolchain(version = "1.17.0")
 # one, so CI actually exercises a second OTP release end-to-end and can catch BEAM-compat
 # regressions across versions (gleam itself requires OTP 27.0+, so this can't go below that).
 # Confirmed available on both builds.hex.pm (Linux) and erlef/otp_builds (macOS) before
-# picking it. sha256 is not yet pinned -- the first CI run against this version will download
-# unverified and print the actual checksum to pin here.
+# picking it. Both checksums below are what CI actually observed downloading this release
+# unverified (CI's macos-latest runner is Apple Silicon, hence macos_arm64 rather than
+# macos_amd64).
 gleam.erlang_toolchain(
     otp_version = "29.0.2",
+    sha256 = {
+        "linux_amd64": "2e9d154a52de72653a1a3c1269b10df3b4ff5e058753e167bc92602681038e9a",
+        "macos_arm64": "ec6e09dee496adf646b0a22eef26b769e7f8032c83dd10292c58d50fb38e2210",
+    },
 )
 gleam.hex_package(
     name = "gleam_stdlib",

gleam/extensions.bzl

@@ -39,6 +39,16 @@ project's manifest.toml lockfile:
 ```starlark
 gleam.hex_manifest(manifest = "//path/to:manifest.toml")
 ```
+
+A `@<name>_host` repository (e.g. `@gleam_host` for the default toolchain name) is always
+created alongside the per-platform ones, exposing a directly `bazel run`-able `gleam` CLI for
+whatever platform Bazel happens to be running on, without needing to know its OS/arch string:
+```starlark
+use_repo(gleam, "gleam_host", ...)
+```
+```shell
+bazel run @gleam_host//:gleam -- format
+```
 """
 
 load("//erlang/private:hermetic_erlang_repository.bzl", "hermetic_erlang_repository")  # buildifier: disable=bzl-visibility
@@ -50,12 +60,16 @@ load(":repositories.bzl", "gleam_register_toolchains")
 _DEFAULT_NAME = "gleam"
 
 # Used when Erlang is hermetic by default (no gleam.erlang_toolchain(...) call) -- see
-# erlang/private/hermetic_erlang_repository.bzl. No checksum is pinned yet for this version --
-# the first CI run downloads unverified and prints the observed checksum, same as an explicit
-# gleam.erlang_toolchain(...) call with no matching sha256 entry.
+# erlang/private/hermetic_erlang_repository.bzl. Checksums below are the actual digests CI
+# observed downloading OTP-28.1 unverified (printed by hermetic_erlang_repository.bzl's own
+# NOTE: log line); platforms CI doesn't run on (linux_arm64, macos_amd64) are left unpinned
+# until a run on that platform observes them, same as any other platform's first use.
 _DEFAULT_HERMETIC_OTP_VERSION = "28.1"
 _DEFAULT_HERMETIC_OS_VERSION = "ubuntu-22.04"
-_DEFAULT_HERMETIC_SHA256 = {}
+_DEFAULT_HERMETIC_SHA256 = {
+    "linux_amd64": "60c1083df707642f20831c762a68db191984314fef1d4d80b05bb8caf70b70bf",
+    "macos_arm64": "831fcfb46929c752abf0a921409a46b306e7019061653323fab7f0958ad4db25",
+}
 
 gleam_toolchain = tag_class(attrs = {
     "name": attr.string(doc = """\

gleam/private/BUILD.bazel

@@ -64,6 +64,13 @@ bzl_library(
     visibility = ["//gleam:__subpackages__"],
 )
 
+bzl_library(
+    name = "host_repo",
+    srcs = ["host_repo.bzl"],
+    visibility = ["//gleam:__subpackages__"],
+    deps = [":toolchains_repo"],
+)
+
 bzl_library(
     name = "versions",
     srcs = ["versions.bzl"],

gleam/private/escript_builder.erl

@@ -1,6 +1,8 @@
 -module(escript_builder).
 -export([main/1]).
 
+-include_lib("kernel/include/file.hrl").
+
 main([OutFile, PackageName, EntryModule, EntryFunction | Files]) ->
     % Read files
     ArchiveFiles = lists:flatmap(fun(Path) ->
@@ -32,19 +34,24 @@ main([OutFile, PackageName, EntryModule, EntryFunction | Files]) ->
 process_path(Path) ->
     case filelib:is_dir(Path) of
         true ->
+            % file:list_dir/1 does not guarantee any particular order (it reflects whatever
+            % order the underlying filesystem happens to return directory entries in, which can
+            % differ between a from-scratch build and a rebuild of the same sources) -- sort so
+            % the resulting escript's zip archive has a deterministic entry order.
             {ok, Files} = file:list_dir(Path),
             lists:flatmap(fun(File) ->
                 Full = filename:join(Path, File),
                 process_path(Full)
-            end, Files);
+            end, lists:sort(Files));
         false ->
             BaseName = filename:basename(Path),
             IsBeamOrApp = filename:extension(Path) == ".beam" orelse filename:extension(Path) == ".app",
             % Check if file contains "@@"
             NotInternal = string:str(BaseName, "@@") == 0,
             if
                 IsBeamOrApp andalso NotInternal ->
-                    [{BaseName, read_file(Path)}];
+                    Bin = read_file(Path),
+                    [{BaseName, Bin, archive_file_info(Bin)}];
                 true ->
                     []
             end
@@ -53,3 +60,26 @@ process_path(Path) ->
 read_file(Path) ->
     {ok, Bin} = file:read_file(Path),
     Bin.
+
+% escript:create/2's {archive, Files, _} accepts {Name, Bin} or {Name, Bin, FileInfo}; without
+% an explicit FileInfo, the zip archive it builds stamps each entry with the current wall-clock
+% time (when this escript_builder run happens to execute), not anything about the input files --
+% two otherwise-identical builds run even a couple of seconds apart get a byte-different escript
+% as a result. Pin every field to a fixed value instead.
+archive_file_info(Bin) ->
+    FixedTime = {{1980, 1, 1}, {0, 0, 0}},
+    #file_info{
+        size = byte_size(Bin),
+        type = regular,
+        access = read_write,
+        atime = FixedTime,
+        mtime = FixedTime,
+        ctime = FixedTime,
+        mode = 8#100644,
+        links = 1,
+        major_device = 0,
+        minor_device = 0,
+        inode = 0,
+        uid = 0,
+        gid = 0
+    }.

gleam/private/gleam_binary.bzl

@@ -28,7 +28,11 @@ def _gleam_binary_impl(ctx):
 
     ctx.actions.run(
         executable = erlc_path,
-        arguments = ["-o", shim_beam.dirname, shim_src.path],
+        # +deterministic stops erlc from embedding the compiling machine's absolute source path
+        # into the shim's .beam (debug/line-number metadata) -- without it, Bazel's per-action
+        # sandbox path (which differs between separate builds of the same target) would leak
+        # into the compiled output and make gleam_binary's result non-reproducible.
+        arguments = ["+deterministic", "-o", shim_beam.dirname, shim_src.path],
         inputs = [shim_src],
         outputs = [shim_beam],
         mnemonic = "CompileGleamShim",

gleam/private/gleam_library.bzl

@@ -93,6 +93,14 @@ def _gleam_library_impl(ctx):
         tools = depset([gleam_exe_wrapper, underlying_gleam_tool]),
         inputs = inputs_depset,
         outputs = [compiled_dir],
+        # `gleam compile-package` shells out to erlc internally; erlc (like any caller of the
+        # `compile` module) reads ERL_COMPILER_OPTIONS and applies it to every compilation.
+        # `deterministic` stops it from embedding the compiling machine's absolute source path
+        # into each .beam file's debug/line-number metadata -- without it, Bazel's per-action
+        # sandbox path (which differs between separate builds of the same target) would leak
+        # into the compiled output and make it non-reproducible for reasons that have nothing
+        # to do with the actual source.
+        env = {"ERL_COMPILER_OPTIONS": "[deterministic]"},
         progress_message = "Compiling Gleam package: " + package_name,
         mnemonic = "GleamCompilePackage",
     )

gleam/private/gleam_test.bzl

@@ -82,6 +82,10 @@ def _gleam_test_impl(ctx):
         tools = depset([gleam_exe_wrapper, underlying_gleam_tool]),
         inputs = inputs_depset,
         outputs = [compiled_dir],
+        # See gleam_library.bzl's identical env setting for why: erlc (invoked internally by
+        # `gleam compile-package`) otherwise embeds Bazel's per-action sandbox path into the
+        # compiled output.
+        env = {"ERL_COMPILER_OPTIONS": "[deterministic]"},
         progress_message = "Compiling Gleam tests: " + package_name,
         mnemonic = "GleamCompileTest",
     )
@@ -96,7 +100,8 @@ def _gleam_test_impl(ctx):
 
     ctx.actions.run(
         executable = erlc_path,
-        arguments = ["-o", runner_beam_dir.path, runner_src.path],
+        # See gleam_binary.bzl's identical +deterministic flag for why.
+        arguments = ["+deterministic", "-o", runner_beam_dir.path, runner_src.path],
         inputs = [runner_src],
         outputs = [runner_beam_dir],
         mnemonic = "CompileGleamEunitRunner",