From 9cde368b4ca0da1979ab3454b0c1dca64c69a2e5 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Fri, 8 Aug 2025 18:51:04 +0200 Subject: [PATCH 1/2] Revert "workflows/eval: disable swap" This reverts commit f2648b263b69aecca529c14fd5441503850a7c0d. While the idea to never use swap was fine, in practice this meant that when nix ran OOM, some other process was killed instead. This lead to the job not being possible to be cancelled anymore and thus needing to timeout, before subsequent jobs could be scheduled. This can take up to 6 hours for GitHub Actions by default. Re-enabling the swap file to catch this case more gracefully. It's still the goal to never actually *use* the swap file during Eval and just a safeguard. Keeping the changed chunkSize and not reverting it - this makes it slightly less likely to hit the swap file when running with Lix. --- .github/workflows/eval.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 805f8572cf152..3643c2340b83b 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -32,11 +32,16 @@ jobs: outputs: targetRunId: ${{ steps.targetRunId.outputs.targetRunId }} steps: - # We'd rather fail than run slow eval on swap. - # Failing allows us to adjust the chunkSize to remain fast. - - name: Disable swap + # This is not supposed to be used and just acts as a fallback. + # Without swap, when Eval runs OOM, it will fail badly with a + # job that is sometimes not interruptible anymore. + # If Eval starts swapping, decrease chunkSize to keep it fast. + - name: Enable swap run: | - sudo swapoff -a + sudo fallocate -l 10G /swap + sudo chmod 600 /swap + sudo mkswap /swap + sudo swapon /swap - name: Check out the PR at the test merge commit uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 From 436d54174dd639cf13c51d46f774822f4d60148f Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sat, 9 Aug 2025 17:19:07 +0200 Subject: [PATCH 2/2] .github/workflows: set timeouts None of our jobs is expected to run for 6 hours, the GitHub limit. These limits are generous and take into accounts that some jobs need to wait for others. If jobs exceed these times, most likely something else is wrong and needs investigation. --- .github/workflows/backport.yml | 1 + .github/workflows/build.yml | 1 + .github/workflows/check.yml | 1 + .github/workflows/codeowners-v2.yml | 2 ++ .github/workflows/dismissed-review.yml | 1 + .github/workflows/edited.yml | 1 + .github/workflows/eval.yml | 3 +++ .github/workflows/lint.yml | 3 ++- .github/workflows/periodic-merge.yml | 1 + .github/workflows/reviewers.yml | 1 + 10 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 709d8a6edd8f9..57c0719ac5e2a 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -23,6 +23,7 @@ jobs: name: Backport Pull Request if: vars.NIXPKGS_CI_APP_ID && github.event.pull_request.merged == true && (github.event.action != 'labeled' || startsWith(github.event.label.name, 'backport')) runs-on: ubuntu-24.04-arm + timeout-minutes: 3 steps: # Use a GitHub App to create the PR so that CI gets triggered # The App is scoped to Repository > Contents and Pull Requests: write for Nixpkgs diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7c4d9ac48a9bc..e9e090cb5b2ae 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,6 +43,7 @@ jobs: desc: shell name: '${{ matrix.system }}: ${{ matrix.desc }}' runs-on: ${{ matrix.runner }} + timeout-minutes: 60 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 2b8dffb9f163a..c04a6dc491863 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -38,6 +38,7 @@ jobs: permissions: pull-requests: write runs-on: ubuntu-24.04-arm + timeout-minutes: 3 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: diff --git a/.github/workflows/codeowners-v2.yml b/.github/workflows/codeowners-v2.yml index 97fcfc63bdd2c..bcc85faed3526 100644 --- a/.github/workflows/codeowners-v2.yml +++ b/.github/workflows/codeowners-v2.yml @@ -49,6 +49,7 @@ jobs: check: name: Check runs-on: ubuntu-24.04-arm + timeout-minutes: 5 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -106,6 +107,7 @@ jobs: request: name: Request runs-on: ubuntu-24.04-arm + timeout-minutes: 5 steps: - uses: cachix/install-nix-action@f0fe604f8a612776892427721526b4c7cfb23aba # v31 diff --git a/.github/workflows/dismissed-review.yml b/.github/workflows/dismissed-review.yml index e8ab48bda0755..1d99c1c9cf063 100644 --- a/.github/workflows/dismissed-review.yml +++ b/.github/workflows/dismissed-review.yml @@ -25,6 +25,7 @@ jobs: minimize: name: Minimize as resolved runs-on: ubuntu-24.04-arm + timeout-minutes: 2 steps: - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 with: diff --git a/.github/workflows/edited.yml b/.github/workflows/edited.yml index 49fccb5f48ba0..ccc55e820e3a1 100644 --- a/.github/workflows/edited.yml +++ b/.github/workflows/edited.yml @@ -31,6 +31,7 @@ jobs: name: Trigger jobs runs-on: ubuntu-24.04 if: github.event.changes.base.ref.from && github.event.changes.base.ref.from != github.event.pull_request.base.ref + timeout-minutes: 2 steps: # Use a GitHub App to create the PR so that CI gets triggered # The App is scoped to Repository > Contents and Pull Requests: write for Nixpkgs diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 3643c2340b83b..64f6d2ba668ee 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -31,6 +31,7 @@ jobs: name: ${{ matrix.system }} outputs: targetRunId: ${{ steps.targetRunId.outputs.targetRunId }} + timeout-minutes: 15 steps: # This is not supposed to be used and just acts as a fallback. # Without swap, when Eval runs OOM, it will fail badly with a @@ -155,6 +156,7 @@ jobs: if: needs.eval.outputs.targetRunId permissions: statuses: write + timeout-minutes: 5 steps: - name: Download output paths and eval stats for all systems uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 @@ -237,6 +239,7 @@ jobs: misc: if: ${{ github.event_name != 'push' }} runs-on: ubuntu-24.04-arm + timeout-minutes: 10 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 0b2da1070d200..ccc2846dab0f3 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -19,6 +19,7 @@ defaults: jobs: treefmt: runs-on: ubuntu-24.04-arm + timeout-minutes: 10 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -51,6 +52,7 @@ jobs: parse: runs-on: ubuntu-24.04-arm + timeout-minutes: 10 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -72,7 +74,6 @@ jobs: nixpkgs-vet: runs-on: ubuntu-24.04-arm - # This should take 1 minute at most, but let's be generous. The default of 6 hours is definitely too long. timeout-minutes: 10 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 diff --git a/.github/workflows/periodic-merge.yml b/.github/workflows/periodic-merge.yml index 046a0d2fb4dde..6fa5e699fc5d8 100644 --- a/.github/workflows/periodic-merge.yml +++ b/.github/workflows/periodic-merge.yml @@ -19,6 +19,7 @@ defaults: jobs: merge: runs-on: ubuntu-24.04-arm + timeout-minutes: 5 steps: # Use a GitHub App to create the PR so that CI gets triggered # The App is scoped to Repository > Contents and Pull Requests: write for Nixpkgs diff --git a/.github/workflows/reviewers.yml b/.github/workflows/reviewers.yml index 1d518133cf6d2..797fa9e919119 100644 --- a/.github/workflows/reviewers.yml +++ b/.github/workflows/reviewers.yml @@ -27,6 +27,7 @@ defaults: jobs: request: runs-on: ubuntu-24.04-arm + timeout-minutes: 20 steps: - name: Check out the PR at the base commit uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2