From 7c293ef97f8f6fb7fcee1802c812ed5817a3ef04 Mon Sep 17 00:00:00 2001 From: Kenneth Sinder Date: Sat, 18 Jul 2026 08:03:43 -0700 Subject: [PATCH] fix(runner-availability): scope detection to runner groups The detector read a single unpaginated page of /orgs/{org}/actions/runners. Managed-runner vendors register one ephemeral runner per job into their own groups and never deregister them, so that list had grown to 1015 entries for burin-labs, 1002 of them Blacksmith ephemerals and 997 of those offline. The org's own runners sorted to index 1002+, past the first page. Every caller therefore saw linux=false/macos=false while 9 Linux and 4 macOS runners were online and idle, and routed onto paid GitHub-hosted runners instead. The probe emitted no warning, because "I could not see the fleet" and "the fleet is busy" produced the same output. burin-code CI spend rose from $516/mo in May to a $4815/mo run rate by mid-July with this active. Query the declared runner groups instead of the flat org list. That keeps the request count proportional to the number of groups rather than to vendor fleet size; paginating the flat list would only have deferred the same failure, since the ephemeral fleet grew by roughly 170 stale registrations per day. Also make every fall-through loud. An unresolvable group name and a group set containing no runners are now warnings, not silent downgrades to the expensive path. Verified against the live burin-labs org across 8 cases: happy path (13 runners, linux=true macos=true), missing group, absent token, 401, vendor-group-only, whitespace and empty entries in the input, and single-group. actionlint and shellcheck clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/runner-availability.yml | 80 +++++++++++++++++++++-- 1 file changed, 75 insertions(+), 5 deletions(-) diff --git a/.github/workflows/runner-availability.yml b/.github/workflows/runner-availability.yml index dd1f5d4..bffe282 100644 --- a/.github/workflows/runner-availability.yml +++ b/.github/workflows/runner-availability.yml @@ -18,6 +18,17 @@ name: runner-availability (reusable) # The `.busy | not` filter means a busy/absent self-hosted pool reports that OS # as unavailable, so callers route to hosted runners instead of queueing behind # active local work. +# +# Detection is scoped to the runner GROUPS named by `runner_groups`, never the +# flat `/orgs/{org}/actions/runners` list. Managed-runner vendors (Blacksmith, +# Namespace, ...) register one ephemeral runner per job into their own groups +# and do not deregister them, so that flat list grows without bound: burin-labs +# reached 1015 registrations, 1002 of them vendor ephemerals and 997 of those +# offline. A single unpaginated page of it returned zero self-hosted matches +# while 9 Linux and 4 macOS runners sat online and idle, so every caller routed +# to paid GitHub-hosted runners. Group scoping keeps the query O(#groups) +# regardless of vendor fleet size; paginating the flat list would only have +# deferred the same failure. on: workflow_call: inputs: @@ -26,6 +37,13 @@ on: type: string default: harn-ci required: false + runner_groups: + description: >- + Comma-separated runner group names holding this org's own self-hosted + capacity. Must exclude vendor-managed groups. + type: string + default: Default,burin-labs-mac-mbp + required: false linux_runner_tag: description: "Linux-specific self-hosted runner label" type: string @@ -98,6 +116,7 @@ jobs: LINUX_TAG: ${{ inputs.linux_runner_tag }} MACOS_TAG: ${{ inputs.macos_runner_tag }} WINDOWS_TAG: ${{ inputs.windows_runner_tag }} + RUNNER_GROUPS: ${{ inputs.runner_groups }} run: | set -euo pipefail if [ -z "${GH_TOKEN:-}" ]; then @@ -111,17 +130,68 @@ jobs: fi gh_api_stderr="$(mktemp)" - runner_list="$(gh api "/orgs/${OWNER}/actions/runners?per_page=100" 2>"$gh_api_stderr" || true)" - if [ -z "$runner_list" ] || ! echo "$runner_list" | jq -e '.runners' > /dev/null 2>&1; then - err_excerpt="$(head -c 300 "$gh_api_stderr" | tr '\n' ' ')" - body_excerpt="$(echo "$runner_list" | head -c 300 | tr '\n' ' ')" - echo "::warning::runner-availability: could not list /orgs/${OWNER}/actions/runners; falling back to GitHub-hosted. gh stderr: ${err_excerpt} | response body: ${body_excerpt}" + + # Every fall-through below is a FALSE report of "no self-hosted + # capacity" that silently routes the caller onto paid GitHub-hosted + # runners. It must always be loud: a probe that cannot see the fleet + # is not the same as a fleet that is busy, and the previous version + # of this detector conflated the two for six days. + fall_through() { + echo "::warning::runner-availability: $1; falling back to GitHub-hosted runners" { echo "linux=false" echo "windows=false" echo "macos=false" } >> "$GITHUB_OUTPUT" exit 0 + } + + group_list="$(gh api --paginate --slurp \ + "/orgs/${OWNER}/actions/runner-groups?per_page=100" \ + 2>"$gh_api_stderr" || true)" + if [ -z "$group_list" ] || ! echo "$group_list" | jq -e '.[0].runner_groups' > /dev/null 2>&1; then + err_excerpt="$(head -c 300 "$gh_api_stderr" | tr '\n' ' ')" + body_excerpt="$(echo "$group_list" | head -c 300 | tr '\n' ' ')" + fall_through "could not list /orgs/${OWNER}/actions/runner-groups. gh stderr: ${err_excerpt} | response body: ${body_excerpt}" + fi + + # Resolve the declared group names to ids. An unresolved name means + # the group was renamed or deleted, which silently shrinks the + # observable fleet, so treat it as a hard detection failure rather + # than quietly probing fewer groups. + resolved="$(echo "$group_list" | jq -c --arg names "$RUNNER_GROUPS" ' + ([$names | split(",") | .[] | gsub("^\\s+|\\s+$"; "")] | map(select(. != ""))) as $want + | ([.[].runner_groups[]] | map({key: .name, value: .id}) | from_entries) as $have + | { + found: [$want[] | select($have[.] != null) | {name: ., id: $have[.]}], + missing: [$want[] | select($have[.] == null)], + }')" + missing="$(echo "$resolved" | jq -r '.missing | join(", ")')" + if [ -n "$missing" ]; then + known="$(echo "$group_list" | jq -r '[.[].runner_groups[].name] | join(", ")')" + fall_through "runner group(s) not found: ${missing}. Known groups: ${known}. Update the caller's runner_groups input" + fi + + # Query each declared group instead of the flat org runner list. The + # flat list is dominated by vendor ephemerals (see header comment). + runners="[]" + for group_id in $(echo "$resolved" | jq -r '.found[].id'); do + group_runners="$(gh api --paginate --slurp \ + "/orgs/${OWNER}/actions/runner-groups/${group_id}/runners?per_page=100" \ + 2>"$gh_api_stderr" || true)" + if [ -z "$group_runners" ] || ! echo "$group_runners" | jq -e '.[0].runners' > /dev/null 2>&1; then + err_excerpt="$(head -c 300 "$gh_api_stderr" | tr '\n' ' ')" + fall_through "could not list runners in group ${group_id}. gh stderr: ${err_excerpt}" + fi + runners="$(jq -c -n --argjson acc "$runners" --argjson next "$group_runners" \ + '$acc + [$next[].runners[]]')" + done + + runner_list="$(jq -c -n --argjson r "$runners" '{runners: $r}')" + observed="$(echo "$runner_list" | jq -r '.runners | length')" + echo "::notice::runner-availability: observed ${observed} runner(s) across group(s) ${RUNNER_GROUPS}" + if [ "$observed" -eq 0 ]; then + fall_through "declared groups (${RUNNER_GROUPS}) contain no runners at all, which indicates a misconfigured runner_groups input rather than a busy fleet" fi for os_label in Linux Windows macOS; do