From 51e6b0e40bd10815ef3c64ea90e672eaa43d4706 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Tue, 12 Aug 2025 10:13:46 +0200 Subject: [PATCH 1/4] workflows: fix actions/download-artifact hashes The Dependabot update change the hashes to the latest main branch commit instead of the v5.0.0 tag - also it didn't adjust the tags in the comments accordingly. Last but not least, one of the references used a `@v5` reference instead of the commit hash. The latter is probably what Dependabot tripped on. --- .github/workflows/eval.yml | 4 ++-- .github/workflows/reviewers.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index dc9889545857e..835fa4c43d99f 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -125,7 +125,7 @@ jobs: GH_TOKEN: ${{ github.token }} run: gh api /rate_limit | jq - - uses: actions/download-artifact@v5 + - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 if: steps.targetRunId.outputs.targetRunId with: run-id: ${{ steps.targetRunId.outputs.targetRunId }} @@ -171,7 +171,7 @@ jobs: pinnedFrom: trusted - name: Download output paths and eval stats for all systems - uses: actions/download-artifact@de96f4613b77ec03b5cf633e7c350c32bd3c5660 # v4.1.8 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: pattern: diff-* path: diff diff --git a/.github/workflows/reviewers.yml b/.github/workflows/reviewers.yml index 2cddb17a10186..58c2017242730 100644 --- a/.github/workflows/reviewers.yml +++ b/.github/workflows/reviewers.yml @@ -96,7 +96,7 @@ jobs: run: gh api /rate_limit | jq - name: Download the comparison results - uses: actions/download-artifact@de96f4613b77ec03b5cf633e7c350c32bd3c5660 # v4.1.8 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: run-id: ${{ steps.eval.outputs.run-id }} github-token: ${{ github.token }} From f05895fb3cf7949b2412a1b2f3a99e7c5b962366 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Fri, 8 Aug 2025 19:15:38 +0200 Subject: [PATCH 2/4] ci/eval/compare: reorder step summary Move "Packages" up, because it's much shorter and easier to scroll past. This way both Packages and Performance are visible immediately. --- ci/eval/compare/default.nix | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/ci/eval/compare/default.nix b/ci/eval/compare/default.nix index 81eebcc9adb12..9060d357d6620 100644 --- a/ci/eval/compare/default.nix +++ b/ci/eval/compare/default.nix @@ -146,6 +146,12 @@ runCommand "compare" cp ${changed-paths} $out/changed-paths.json + { + echo + echo "# Packages" + echo + jq -r -f ${./generate-step-summary.jq} < ${changed-paths} + } >> $out/step-summary.md if jq -e '(.attrdiff.added | length == 0) and (.attrdiff.removed | length == 0)' "${changed-paths}" > /dev/null; then # Chunks have changed between revisions @@ -175,12 +181,5 @@ runCommand "compare" } >> $out/step-summary.md fi - { - echo - echo "# Packages" - echo - jq -r -f ${./generate-step-summary.jq} < ${changed-paths} - } >> $out/step-summary.md - cp "$maintainersPath" "$out/maintainers.json" '' From b523f257ac01c54d709907fae166c4b82684b382 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 23 Jul 2025 13:28:44 +0200 Subject: [PATCH 3/4] workflows/eval: test all available versions With this change, we start running Eval on all available Lix and Nix versions. Because this requires a lot of resources, this complete test is only run when `ci/pinned.json` is updated. The resulting outpaths are checked for consistency with the target branch. A difference will cause the `report` job to fail, thus blocking the merge, ensuring Eval consistency for Nixpkgs across different versions. This implements a kind of "ratchet style" check: Since we originally confirmed that the versions currently in Nixpkgs at the time of this commit match Eval behavior of Nix 2.3, we can ensure consistency with Nix 2.3 down the road, even without testing for it explicitly. There had been one regression in Eval consistency for Nix between 2.18 and 2.24 - two tests in `tests.devShellTools` produce different results between Lix 2.91+ (which was forked from Nix 2.18) and Nix 2.24+. I assume it's unlikely that such a change would be "fixed" by now, thus I added an exception for these. As a bonus, we also present the total time in seconds it takes for Eval to complete for every tested version in a summary table. This allows us to easily see performance improvements for Eval due to version updates. At this stage, this time only includes the "outpaths" step of Eval, but not the generation of attrpaths beforehand. --- .github/workflows/eval.yml | 133 ++++++++++++++++++++++++++++++++++++- .github/workflows/pr.yml | 16 +++++ ci/default.nix | 3 +- ci/supportedVersions.nix | 32 +++++++++ 4 files changed, 180 insertions(+), 4 deletions(-) create mode 100755 ci/supportedVersions.nix diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 835fa4c43d99f..1ac9c846e6f05 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -11,6 +11,10 @@ on: systems: required: true type: string + testVersions: + required: false + default: false + type: boolean secrets: OWNER_APP_PRIVATE_KEY: required: false @@ -22,13 +26,49 @@ defaults: shell: bash jobs: + versions: + if: inputs.testVersions + runs-on: ubuntu-24.04-arm + outputs: + versions: ${{ steps.versions.outputs.versions }} + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + path: trusted + sparse-checkout: | + ci/supportedVersions.nix + + - name: Check out the PR at the test merge commit + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + ref: ${{ inputs.mergedSha }} + path: untrusted + sparse-checkout: | + ci/pinned.json + + - name: Install Nix + uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31 + + - name: Load supported versions + id: versions + run: | + echo "versions=$(trusted/ci/supportedVersions.nix --arg pinnedJson untrusted/ci/pinned.json)" >> "$GITHUB_OUTPUT" + eval: runs-on: ubuntu-24.04-arm + needs: versions + if: ${{ !cancelled() }} strategy: fail-fast: false matrix: system: ${{ fromJSON(inputs.systems) }} - name: ${{ matrix.system }} + version: + - "" # Default Eval triggering rebuild labels and such. + - ${{ fromJSON(needs.versions.outputs.versions || '[]') }} # Only for ci/pinned.json updates. + # Failures for versioned Evals will be collected in a separate job below + # to not interrupt main Eval's compare step. + continue-on-error: ${{ matrix.version != '' }} + name: ${{ matrix.system }}${{ matrix.version && format(' @ {0}', matrix.version) || '' }} outputs: targetRunId: ${{ steps.targetRunId.outputs.targetRunId }} timeout-minutes: 15 @@ -60,17 +100,19 @@ jobs: - name: Evaluate the ${{ matrix.system }} output paths for all derivation attributes env: MATRIX_SYSTEM: ${{ matrix.system }} + MATRIX_VERSION: ${{ matrix.version || 'nixVersions.latest' }} run: | nix-build untrusted/ci --arg nixpkgs ./pinned -A eval.singleSystem \ --argstr evalSystem "$MATRIX_SYSTEM" \ --arg chunkSize 8000 \ + --argstr nixPath "$MATRIX_VERSION" \ --out-link merged # If it uses too much memory, slightly decrease chunkSize - name: Upload the output paths and eval stats uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: merged-${{ matrix.system }} + name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}merged-${{ matrix.system }} path: merged/* - name: Log current API rate limits @@ -149,7 +191,7 @@ jobs: if: steps.targetRunId.outputs.targetRunId uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: diff-${{ matrix.system }} + name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}diff-${{ matrix.system }} path: diff/* compare: @@ -240,6 +282,91 @@ jobs: target_url }) + # Creates a matrix of Eval performance for various versions and systems. + report: + runs-on: ubuntu-24.04-arm + needs: [versions, eval] + steps: + - name: Download output paths and eval stats for all versions + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + with: + pattern: "*-diff-*" + path: versions + + - name: Add version comparison table to job summary + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + SYSTEMS: ${{ inputs.systems }} + VERSIONS: ${{ needs.versions.outputs.versions }} + with: + script: | + const { readFileSync } = require('node:fs') + const path = require('node:path') + + const systems = JSON.parse(process.env.SYSTEMS) + const versions = JSON.parse(process.env.VERSIONS) + + core.summary.addHeading('Lix/Nix version comparison') + core.summary.addTable( + [].concat( + [ + [{ data: 'Version', header: true }].concat( + systems.map((system) => ({ data: system, header: true })), + ), + ], + versions.map((version) => + [{ data: version }].concat( + systems.map((system) => { + try { + const artifact = path.join('versions', `${version}-diff-${system}`) + const time = Math.round( + parseFloat( + readFileSync( + path.join(artifact, 'after', system, 'total-time'), + 'utf-8', + ), + ), + ) + const diff = JSON.parse( + readFileSync(path.join(artifact, system, 'diff.json'), 'utf-8'), + ) + const attrs = [].concat( + diff.added, + diff.removed, + diff.changed, + diff.rebuilds + ).filter(attr => + // Exceptions related to dev shells, which changed at some time between 2.18 and 2.24. + !attr.startsWith('tests.devShellTools.nixos.') && + !attr.startsWith('tests.devShellTools.unstructuredDerivationInputEnv.') + ) + if (attrs.length > 0) { + core.setFailed( + `${version} on ${system} has changed outpaths!\nNote: Please make sure to update ci/pinned.json separately from changes to other packages.`, + ) + return { data: ':x:' } + } + return { data: time } + } catch { + core.warning(`${version} on ${system} did not produce artifact.`) + return { data: ':warning:' } + } + }), + ), + ), + ), + ) + core.summary.addRaw( + '\n*Evaluation time in seconds without downloading dependencies.*', + true, + ) + core.summary.addRaw('\n*:warning: Job did not report a result.*', true) + core.summary.addRaw( + '\n*:x: Job produced different outpaths than the target branch.*', + true, + ) + core.summary.write() + misc: if: ${{ github.event_name != 'push' }} runs-on: ubuntu-24.04-arm diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 086b14268f5b8..db261a904ef54 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -28,6 +28,7 @@ jobs: mergedSha: ${{ steps.get-merge-commit.outputs.mergedSha }} targetSha: ${{ steps.get-merge-commit.outputs.targetSha }} systems: ${{ steps.systems.outputs.systems }} + touched: ${{ steps.files.outputs.touched }} steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: @@ -64,6 +65,20 @@ jobs: core.setOutput('head', headClassification) core.info('head classification:', headClassification) + - name: Determine changed files + id: files + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const files = (await github.paginate(github.rest.pulls.listFiles, { + ...context.repo, + pull_number: context.payload.pull_request.number, + per_page: 100, + })).map(file => file.filename) + + if (files.includes('ci/pinned.json')) core.setOutput('touched', ['pinned']) + else core.setOutput('touched', []) + check: name: Check needs: [prepare] @@ -96,6 +111,7 @@ jobs: mergedSha: ${{ needs.prepare.outputs.mergedSha }} targetSha: ${{ needs.prepare.outputs.targetSha }} systems: ${{ needs.prepare.outputs.systems }} + testVersions: ${{ contains(fromJSON(needs.prepare.outputs.touched), 'pinned') && !contains(fromJSON(needs.prepare.outputs.headBranch).type, 'development') }} labels: name: Labels diff --git a/ci/default.nix b/ci/default.nix index 603e11513aa7e..2bae87eca0587 100644 --- a/ci/default.nix +++ b/ci/default.nix @@ -5,6 +5,7 @@ in system ? builtins.currentSystem, nixpkgs ? null, + nixPath ? "nixVersions.latest", }: let nixpkgs' = @@ -115,7 +116,7 @@ rec { # (nixVersions.stable and Lix) here somehow at some point to ensure we don't # have eval divergence. eval = pkgs.callPackage ./eval { - nix = pkgs.nixVersions.latest; + nix = pkgs.lib.getAttrFromPath (pkgs.lib.splitString "." nixPath) pkgs; }; # CI jobs diff --git a/ci/supportedVersions.nix b/ci/supportedVersions.nix new file mode 100755 index 0000000000000..77f6f7dd14751 --- /dev/null +++ b/ci/supportedVersions.nix @@ -0,0 +1,32 @@ +#!/usr/bin/env -S nix-instantiate --eval --strict --json --arg unused true +# Unused argument to trigger nix-instantiate calling this function with the default arguments. +{ + pinnedJson ? ./pinned.json, +}: +let + pinned = (builtins.fromJSON (builtins.readFile pinnedJson)).pins; + nixpkgs = fetchTarball { + inherit (pinned.nixpkgs) url; + sha256 = pinned.nixpkgs.hash; + }; + pkgs = import nixpkgs { + config.allowAliases = false; + }; + + inherit (pkgs) lib; + + lix = lib.pipe pkgs.lixPackageSets [ + (lib.filterAttrs (_: set: lib.isDerivation set.lix or null && set.lix.meta.available)) + lib.attrNames + (lib.filter (name: lib.match "lix_[0-9_]+|git" name != null)) + (map (name: "lixPackageSets.${name}.lix")) + ]; + + nix = lib.pipe pkgs.nixVersions [ + (lib.filterAttrs (_: drv: lib.isDerivation drv && drv.meta.available)) + lib.attrNames + (lib.filter (name: lib.match "nix_[0-9_]+|git" name != null)) + (map (name: "nixVersions.${name}")) + ]; +in +lix ++ nix From 14a6d9de4600fbe6d86abe1ea2a538522069e8d0 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Thu, 7 Aug 2025 14:42:55 +0200 Subject: [PATCH 4/4] ci/pinned: update This gives us a fixed `lixPackageSets.git`. From the nixpkgs-unstable channel: https://hydra.nixos.org/build/304569381#tabs-buildinputs Changes for treefmt-nix: https://github.com/numtide/treefmt-nix/compare/58bd4da459f0a39e506847109a2a5cfceb837796...7d81f6fb2e19bf84f1c65135d1060d829fae2408 --- ci/pinned.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/pinned.json b/ci/pinned.json index 1f1af448a2e87..14a454c2e258d 100644 --- a/ci/pinned.json +++ b/ci/pinned.json @@ -9,9 +9,9 @@ }, "branch": "nixpkgs-unstable", "submodules": false, - "revision": "6a489c9482ca676ce23c0bcd7f2e1795383325fa", - "url": "https://github.com/NixOS/nixpkgs/archive/6a489c9482ca676ce23c0bcd7f2e1795383325fa.tar.gz", - "hash": "0vsvkhy3gb8yzq62vazhmpqixssmd4xinnll7w73l4vrqd611wlf" + "revision": "641d909c4a7538f1539da9240dedb1755c907e40", + "url": "https://github.com/NixOS/nixpkgs/archive/641d909c4a7538f1539da9240dedb1755c907e40.tar.gz", + "hash": "10hpb1aw884k3zzcy1mhf47dqvfagiyx7kr6hg0p5xcwg04mkx8x" }, "treefmt-nix": { "type": "Git", @@ -22,9 +22,9 @@ }, "branch": "main", "submodules": false, - "revision": "58bd4da459f0a39e506847109a2a5cfceb837796", - "url": "https://github.com/numtide/treefmt-nix/archive/58bd4da459f0a39e506847109a2a5cfceb837796.tar.gz", - "hash": "01bg9b4xzlv6s5q1q78vib6l2csw02b3rk5bm5yj4gx2sk2hvmrq" + "revision": "7d81f6fb2e19bf84f1c65135d1060d829fae2408", + "url": "https://github.com/numtide/treefmt-nix/archive/7d81f6fb2e19bf84f1c65135d1060d829fae2408.tar.gz", + "hash": "1cg20q8ja8k2nb7mzy95hgmd8whxapc3fbyndh1ip5dr6d1grxfs" } }, "version": 5