diff --git a/.github/actions/checkout/action.yml b/.github/actions/checkout/action.yml index e14c5936e62d7..868d832bf8f37 100644 --- a/.github/actions/checkout/action.yml +++ b/.github/actions/checkout/action.yml @@ -5,8 +5,6 @@ description: 'Checkout into trusted / untrusted / pinned folders consistently.' inputs: merged-as-untrusted-at: description: "Whether and which SHA to checkout for the merge commit in the ./untrusted folder." - pinned-from: - description: "Whether to checkout the pinned nixpkgs for CI and from where (trusted, untrusted)." target-as-trusted-at: description: "Whether and which SHA to checkout for the target commit in the ./trusted folder." @@ -16,7 +14,6 @@ runs: - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 env: MERGED_SHA: ${{ inputs.merged-as-untrusted-at }} - PINNED_FROM: ${{ inputs.pinned-from }} TARGET_SHA: ${{ inputs.target-as-trusted-at }} with: script: | @@ -45,6 +42,7 @@ runs: // A single fetch call comes with a lot less overhead. The fetch takes essentially the // same time no matter whether its 1, 2 or 3 commits at once. async function getPinnedSha(ref) { + if (!ref) return undefined const { content, encoding } = (await github.rest.repos.getContent({ ...context.repo, path: 'ci/pinned.json', @@ -60,16 +58,16 @@ runs: path: 'untrusted', }, { - sha: process.env.PINNED_FROM === 'untrusted' && (await getPinnedSha(process.env.MERGED_SHA)), - path: 'pinned' + sha: await getPinnedSha(process.env.MERGED_SHA), + path: 'untrusted-pinned' }, { sha: process.env.TARGET_SHA, path: 'trusted', }, { - sha: process.env.PINNED_FROM === 'trusted' && (await getPinnedSha(process.env.TARGET_SHA)), - path: 'pinned' + sha: await getPinnedSha(process.env.TARGET_SHA), + path: 'trusted-pinned' } ].filter(({ sha }) => Boolean(sha)) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 20fcb47dc7a04..0acb7dab64f04 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,7 +51,6 @@ jobs: uses: ./.github/actions/checkout with: merged-as-untrusted-at: ${{ inputs.mergedSha }} - pinned-from: untrusted - uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31 with: @@ -66,33 +65,33 @@ jobs: authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} pushFilter: '(-source$|-nixpkgs-tarball-)' - - run: nix-env --install -f nixpkgs/pinned -A nix-build-uncached + - run: nix-env --install -f nixpkgs/untrusted-pinned -A nix-build-uncached - name: Build shell if: contains(matrix.builds, 'shell') - run: echo "${{ matrix.systems }}" | xargs -n1 nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/pinned -A shell --argstr system + run: echo "${{ matrix.systems }}" | xargs -n1 nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A shell --argstr system - name: Build NixOS manual if: | contains(matrix.builds, 'manual-nixos') && !cancelled() && contains(fromJSON(inputs.baseBranch).type, 'primary') - run: nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/pinned -A manual-nixos --out-link nixos-manual + run: nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A manual-nixos --out-link nixos-manual - name: Build Nixpkgs manual if: contains(matrix.builds, 'manual-nixpkgs') && !cancelled() - run: nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/pinned -A manual-nixpkgs -A manual-nixpkgs-tests + run: nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A manual-nixpkgs -A manual-nixpkgs-tests - name: Build Nixpkgs manual tests if: contains(matrix.builds, 'manual-nixpkgs-tests') && !cancelled() - run: nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/pinned -A manual-nixpkgs-tests + run: nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A manual-nixpkgs-tests - name: Build lib tests if: contains(matrix.builds, 'lib-tests') && !cancelled() - run: nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/pinned -A lib-tests + run: nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A lib-tests - name: Build tarball if: contains(matrix.builds, 'tarball') && !cancelled() - run: nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/pinned -A tarball + run: nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A tarball - name: Upload NixOS manual if: | diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 287d4f80d557f..ded8bad536c47 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -103,7 +103,6 @@ jobs: uses: ./.github/actions/checkout with: merged-as-untrusted-at: ${{ inputs.mergedSha }} - pinned-from: trusted target-as-trusted-at: ${{ inputs.targetSha }} - uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31 @@ -117,7 +116,7 @@ jobs: pushFilter: -source$ - name: Build codeowners validator - run: nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/pinned -A codeownersValidator + run: nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A codeownersValidator - uses: actions/create-github-app-token@0f859bf9e69e887678d5bbfbee594437cb440ffe # v2.1.0 if: github.event_name == 'pull_request_target' && vars.OWNER_RO_APP_ID diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index c3b1c70034be6..7efa71913ac70 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -71,8 +71,6 @@ jobs: # to not interrupt main Eval's compare step. continue-on-error: ${{ matrix.version != '' }} name: ${{ matrix.system }}${{ matrix.version && format(' @ {0}', matrix.version) || '' }} - outputs: - targetRunId: ${{ steps.targetRunId.outputs.targetRunId }} timeout-minutes: 15 steps: # This is not supposed to be used and just acts as a fallback. @@ -89,11 +87,11 @@ jobs: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: sparse-checkout: .github/actions - - name: Check out the PR at the test merge commit + - name: Check out the PR at merged and target commits uses: ./.github/actions/checkout with: merged-as-untrusted-at: ${{ inputs.mergedSha }} - pinned-from: untrusted + target-as-trusted-at: ${{ inputs.targetSha }} - name: Install Nix uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31 @@ -106,98 +104,64 @@ jobs: authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} pushFilter: '(-source|-single-chunk)$' - - name: Evaluate the ${{ matrix.system }} output paths for all derivation attributes + - name: Evaluate the ${{ matrix.system }} output paths at the merge commit env: MATRIX_SYSTEM: ${{ matrix.system }} MATRIX_VERSION: ${{ matrix.version || 'nixVersions.latest' }} run: | - nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/pinned -A eval.singleSystem \ + nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.singleSystem \ --argstr evalSystem "$MATRIX_SYSTEM" \ --arg chunkSize 8000 \ --argstr nixPath "$MATRIX_VERSION" \ --out-link merged - # If it uses too much memory, slightly decrease chunkSize - - - name: Upload the output paths and eval stats - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}merged-${{ matrix.system }} - path: merged/* - - - name: Log current API rate limits + # If it uses too much memory, slightly decrease chunkSize. + # Note: Keep the same further down in sync! + + # Running the attrpath generation step separately from the outpath step afterwards. + # The idea is that, *if* Eval on the target branch has not finished, yet, we will + # generate the attrpaths in the meantime - and the separate command command afterwards + # will check cachix again for whether Eval has finished. If no Eval result from the + # target branch can be found the second time, we proceed to run it in here. Attrpaths + # generation takes roughly 30 seconds, so for every normal use-case this should be more + # than enough of a head start for Eval on the target branch to finish. + # This edge-case, that Eval on the target branch is delayed is unlikely to happen anyway: + # For a commit to become the target commit of a PR, it must *already* be on the branch. + # Normally, CI should always start running on that push event *before* it starts running + # on the PR. + - name: Evaluate the ${{ matrix.system }} attribute paths at the target commit + if: inputs.targetSha env: - GH_TOKEN: ${{ github.token }} - run: gh api /rate_limit | jq + MATRIX_SYSTEM: ${{ matrix.system }} + run: | + nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.attrpathsSuperset \ + --argstr evalSystem "$MATRIX_SYSTEM" \ + --argstr nixPath "nixVersions.latest" - - name: Get target run id + - name: Evaluate the ${{ matrix.system }} output paths at the target commit if: inputs.targetSha - id: targetRunId - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 env: MATRIX_SYSTEM: ${{ matrix.system }} - TARGET_SHA: ${{ inputs.targetSha }} - with: - script: | - const system = process.env.MATRIX_SYSTEM - const targetSha = process.env.TARGET_SHA - - let run_id - try { - run_id = (await github.rest.actions.listWorkflowRuns({ - ...context.repo, - workflow_id: 'push.yml', - event: 'push', - head_sha: targetSha - })).data.workflow_runs[0].id - } catch { - throw new Error(`Could not find a push.yml workflow run for ${targetSha}.`) - } - - // Waiting 120 * 5 sec = 10 min. max. - // Eval takes max 5-6 minutes, normally. - for (let i = 0; i < 120; i++) { - const result = await github.rest.actions.listWorkflowRunArtifacts({ - ...context.repo, - run_id, - name: `merged-${system}` - }) - if (result.data.total_count > 0) { - core.setOutput('targetRunId', run_id) - return - } - await new Promise(resolve => setTimeout(resolve, 5000)) - } - // No artifact found at this stage. This usually means that Eval failed on the target branch. - // This should only happen when Eval is broken on the target branch and this PR fixes it. - // Continue without targetRunId to skip the remaining steps, but pass the job. - - - name: Log current API rate limits - env: - GH_TOKEN: ${{ github.token }} - run: gh api /rate_limit | jq - - - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 - if: steps.targetRunId.outputs.targetRunId - with: - run-id: ${{ steps.targetRunId.outputs.targetRunId }} - name: merged-${{ matrix.system }} - path: target - github-token: ${{ github.token }} - merge-multiple: true + # This should be very quick, because it pulls the eval results from Cachix. + run: | + nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.singleSystem \ + --argstr evalSystem "$MATRIX_SYSTEM" \ + --arg chunkSize 8000 \ + --argstr nixPath "nixVersions.latest" \ + --out-link target - name: Compare outpaths against the target branch - if: steps.targetRunId.outputs.targetRunId + if: inputs.targetSha env: MATRIX_SYSTEM: ${{ matrix.system }} run: | - nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/pinned -A eval.diff \ - --arg beforeDir ./target \ + nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.diff \ + --arg beforeDir "$(readlink ./target)" \ --arg afterDir "$(readlink ./merged)" \ --argstr evalSystem "$MATRIX_SYSTEM" \ --out-link diff - name: Upload outpaths diff and stats - if: steps.targetRunId.outputs.targetRunId + if: inputs.targetSha uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}diff-${{ matrix.system }} @@ -206,7 +170,7 @@ jobs: compare: runs-on: ubuntu-24.04-arm needs: [eval] - if: needs.eval.outputs.targetRunId && !cancelled() && !failure() + if: inputs.targetSha && !cancelled() && !failure() permissions: statuses: write timeout-minutes: 5 @@ -219,7 +183,6 @@ jobs: with: merged-as-untrusted-at: ${{ inputs.mergedSha }} target-as-trusted-at: ${{ inputs.targetSha }} - pinned-from: trusted - name: Download output paths and eval stats for all systems uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 @@ -233,7 +196,7 @@ jobs: - name: Combine all output paths and eval stats run: | - nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/pinned -A eval.combine \ + nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.combine \ --arg diffDir ./diff \ --out-link combined @@ -245,7 +208,7 @@ jobs: | jq --raw-input --slurp 'split("\n")[:-1]' > touched-files.json # Use the target branch to get accurate maintainer info - nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/pinned -A eval.compare \ + nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.compare \ --arg combinedDir "$(realpath ./combined)" \ --arg touchedFilesJson ./touched-files.json \ --argstr githubAuthorId "$AUTHOR_ID" \ diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 771a64c4a5d6f..4d94df1578fad 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -31,7 +31,6 @@ jobs: uses: ./.github/actions/checkout with: merged-as-untrusted-at: ${{ inputs.mergedSha }} - pinned-from: untrusted - uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31 @@ -44,7 +43,7 @@ jobs: # Note that it's fine to run this on untrusted code because: # - There's no secrets accessible here # - The build is sandboxed - if ! nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/pinned -A fmt.check; then + if ! nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A fmt.check; then echo "Some files are not properly formatted" echo "Please format them by going to the Nixpkgs root directory and running one of:" echo " nix-shell --run treefmt" @@ -66,7 +65,6 @@ jobs: uses: ./.github/actions/checkout with: merged-as-untrusted-at: ${{ inputs.mergedSha }} - pinned-from: untrusted - uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31 @@ -81,7 +79,7 @@ jobs: - name: Parse all nix files run: | # Tests multiple versions at once, let's make sure all of them run, so keep-going. - nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/pinned -A parse --keep-going + nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A parse --keep-going nixpkgs-vet: runs-on: ubuntu-24.04-arm @@ -94,7 +92,6 @@ jobs: uses: ./.github/actions/checkout with: merged-as-untrusted-at: ${{ inputs.mergedSha }} - pinned-from: untrusted target-as-trusted-at: ${{ inputs.targetSha }} - uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31 @@ -112,7 +109,7 @@ jobs: # Force terminal colors to be enabled. The library that `nixpkgs-vet` uses respects https://bixense.com/clicolors/ CLICOLOR_FORCE: 1 run: | - if nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/pinned -A nixpkgs-vet --arg base "./nixpkgs/trusted" --arg head "./nixpkgs/untrusted"; then + if nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A nixpkgs-vet --arg base "./nixpkgs/trusted" --arg head "./nixpkgs/untrusted"; then exit 0 else exitCode=$? diff --git a/ci/eval/default.nix b/ci/eval/default.nix index 363804cce7168..8d79034db59e6 100644 --- a/ci/eval/default.nix +++ b/ci/eval/default.nix @@ -141,6 +141,8 @@ let env = { inherit evalSystem chunkSize; }; + __structuredAttrs = true; + unsafeDiscardReferences.out = true; } '' export NIX_STATE_DIR=$(mktemp -d)