workflows/eval: compare eval consistency and performance between Lix / Nix versions

.github/workflows/eval.yml

@@ -11,6 +11,10 @@ on:
       systems:
         required: true
         type: string
+      testVersions:
+        required: false
+        default: false
+        type: boolean
     secrets:
       OWNER_APP_PRIVATE_KEY:
         required: false
@@ -22,13 +26,49 @@ defaults:
     shell: bash
 
 jobs:
+  versions:
+    if: inputs.testVersions
+    runs-on: ubuntu-24.04-arm
+    outputs:
+      versions: ${{ steps.versions.outputs.versions }}
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          path: trusted
+          sparse-checkout: |
+            ci/supportedVersions.nix
+
+      - name: Check out the PR at the test merge commit
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          ref: ${{ inputs.mergedSha }}
+          path: untrusted
+          sparse-checkout: |
+            ci/pinned.json
+
+      - name: Install Nix
+        uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31
+
+      - name: Load supported versions
+        id: versions
+        run: |
+          echo "versions=$(trusted/ci/supportedVersions.nix --arg pinnedJson untrusted/ci/pinned.json)" >> "$GITHUB_OUTPUT"
+
   eval:
     runs-on: ubuntu-24.04-arm
+    needs: versions
+    if: ${{ !cancelled() }}
     strategy:
       fail-fast: false
       matrix:
         system: ${{ fromJSON(inputs.systems) }}
-    name: ${{ matrix.system }}
+        version:
+          - "" # Default Eval triggering rebuild labels and such.
+          - ${{ fromJSON(needs.versions.outputs.versions || '[]') }} # Only for ci/pinned.json updates.
+    # Failures for versioned Evals will be collected in a separate job below
+    # to not interrupt main Eval's compare step.
+    continue-on-error: ${{ matrix.version != '' }}
+    name: ${{ matrix.system }}${{ matrix.version && format(' @ {0}', matrix.version) || '' }}
     outputs:
       targetRunId: ${{ steps.targetRunId.outputs.targetRunId }}
     timeout-minutes: 15
@@ -60,17 +100,19 @@ jobs:
       - name: Evaluate the ${{ matrix.system }} output paths for all derivation attributes
         env:
           MATRIX_SYSTEM: ${{ matrix.system }}
+          MATRIX_VERSION: ${{ matrix.version || 'nixVersions.latest' }}
         run: |
           nix-build untrusted/ci --arg nixpkgs ./pinned  -A eval.singleSystem \
             --argstr evalSystem "$MATRIX_SYSTEM" \
             --arg chunkSize 8000 \
+            --argstr nixPath "$MATRIX_VERSION" \
             --out-link merged
           # If it uses too much memory, slightly decrease chunkSize
 
       - name: Upload the output paths and eval stats
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
-          name: merged-${{ matrix.system }}
+          name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}merged-${{ matrix.system }}
           path: merged/*
 
       - name: Log current API rate limits
@@ -125,7 +167,7 @@ jobs:
           GH_TOKEN: ${{ github.token }}
         run: gh api /rate_limit | jq
 
-      - uses: actions/download-artifact@v5
+      - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
         if: steps.targetRunId.outputs.targetRunId
         with:
           run-id: ${{ steps.targetRunId.outputs.targetRunId }}
@@ -149,7 +191,7 @@ jobs:
         if: steps.targetRunId.outputs.targetRunId
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
-          name: diff-${{ matrix.system }}
+          name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}diff-${{ matrix.system }}
           path: diff/*
 
   compare:
@@ -171,7 +213,7 @@ jobs:
           pinnedFrom: trusted
 
       - name: Download output paths and eval stats for all systems
-        uses: actions/download-artifact@de96f4613b77ec03b5cf633e7c350c32bd3c5660 # v4.1.8
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
         with:
           pattern: diff-*
           path: diff
@@ -240,6 +282,91 @@ jobs:
               target_url
             })
 
+  # Creates a matrix of Eval performance for various versions and systems.
+  report:
+    runs-on: ubuntu-24.04-arm
+    needs: [versions, eval]
+    steps:
+      - name: Download output paths and eval stats for all versions
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
+        with:
+          pattern: "*-diff-*"
+          path: versions
+
+      - name: Add version comparison table to job summary
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        env:
+          SYSTEMS: ${{ inputs.systems }}
+          VERSIONS: ${{ needs.versions.outputs.versions }}
+        with:
+          script: |
+            const { readFileSync } = require('node:fs')
+            const path = require('node:path')
+
+            const systems = JSON.parse(process.env.SYSTEMS)
+            const versions = JSON.parse(process.env.VERSIONS)
+
+            core.summary.addHeading('Lix/Nix version comparison')
+            core.summary.addTable(
+              [].concat(
+                [
+                  [{ data: 'Version', header: true }].concat(
+                    systems.map((system) => ({ data: system, header: true })),
+                  ),
+                ],
+                versions.map((version) =>
+                  [{ data: version }].concat(
+                    systems.map((system) => {
+                      try {
+                        const artifact = path.join('versions', `${version}-diff-${system}`)
+                        const time = Math.round(
+                          parseFloat(
+                            readFileSync(
+                              path.join(artifact, 'after', system, 'total-time'),
+                              'utf-8',
+                            ),
+                          ),
+                        )
+                        const diff = JSON.parse(
+                          readFileSync(path.join(artifact, system, 'diff.json'), 'utf-8'),
+                        )
+                        const attrs = [].concat(
+                          diff.added,
+                          diff.removed,
+                          diff.changed,
+                          diff.rebuilds
+                        ).filter(attr =>
+                          // Exceptions related to dev shells, which changed at some time between 2.18 and 2.24.
+                          !attr.startsWith('tests.devShellTools.nixos.') &&
+                          !attr.startsWith('tests.devShellTools.unstructuredDerivationInputEnv.')
+                        )
+                        if (attrs.length > 0) {
+                          core.setFailed(
+                            `${version} on ${system} has changed outpaths!\nNote: Please make sure to update ci/pinned.json separately from changes to other packages.`,
+                          )
+                          return { data: ':x:' }
+                        }
+                        return { data: time }
+                      } catch {
+                        core.warning(`${version} on ${system} did not produce artifact.`)
+                        return { data: ':warning:' }
+                      }
+                    }),
+                  ),
+                ),
+              ),
+            )
+            core.summary.addRaw(
+              '\n*Evaluation time in seconds without downloading dependencies.*',
+              true,
+            )
+            core.summary.addRaw('\n*:warning: Job did not report a result.*', true)
+            core.summary.addRaw(
+              '\n*:x: Job produced different outpaths than the target branch.*',
+              true,
+            )
+            core.summary.write()
+
   misc:
     if: ${{ github.event_name != 'push' }}
     runs-on: ubuntu-24.04-arm

.github/workflows/pr.yml

@@ -28,6 +28,7 @@ jobs:
       mergedSha: ${{ steps.get-merge-commit.outputs.mergedSha }}
       targetSha: ${{ steps.get-merge-commit.outputs.targetSha }}
       systems: ${{ steps.systems.outputs.systems }}
+      touched: ${{ steps.files.outputs.touched }}
     steps:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
@@ -64,6 +65,20 @@ jobs:
             core.setOutput('head', headClassification)
             core.info('head classification:', headClassification)
 
+      - name: Determine changed files
+        id: files
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        with:
+          script: |
+            const files = (await github.paginate(github.rest.pulls.listFiles, {
+              ...context.repo,
+              pull_number: context.payload.pull_request.number,
+              per_page: 100,
+            })).map(file => file.filename)
+
+            if (files.includes('ci/pinned.json')) core.setOutput('touched', ['pinned'])
+            else core.setOutput('touched', [])
+
   check:
     name: Check
     needs: [prepare]
@@ -96,6 +111,7 @@ jobs:
       mergedSha: ${{ needs.prepare.outputs.mergedSha }}
       targetSha: ${{ needs.prepare.outputs.targetSha }}
       systems: ${{ needs.prepare.outputs.systems }}
+      testVersions: ${{ contains(fromJSON(needs.prepare.outputs.touched), 'pinned') && !contains(fromJSON(needs.prepare.outputs.headBranch).type, 'development') }}
 
   labels:
     name: Labels

.github/workflows/reviewers.yml

@@ -96,7 +96,7 @@ jobs:
         run: gh api /rate_limit | jq
 
       - name: Download the comparison results
-        uses: actions/download-artifact@de96f4613b77ec03b5cf633e7c350c32bd3c5660 # v4.1.8
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
         with:
           run-id: ${{ steps.eval.outputs.run-id }}
           github-token: ${{ github.token }}

ci/default.nix

@@ -5,6 +5,7 @@ in
   system ? builtins.currentSystem,
 
   nixpkgs ? null,
+  nixPath ? "nixVersions.latest",
 }:
 let
   nixpkgs' =
@@ -115,7 +116,7 @@ rec {
   # (nixVersions.stable and Lix) here somehow at some point to ensure we don't
   # have eval divergence.
   eval = pkgs.callPackage ./eval {
-    nix = pkgs.nixVersions.latest;
+    nix = pkgs.lib.getAttrFromPath (pkgs.lib.splitString "." nixPath) pkgs;
   };
 
   # CI jobs

ci/eval/compare/default.nix

@@ -146,6 +146,12 @@ runCommand "compare"
 
     cp ${changed-paths} $out/changed-paths.json
 
+    {
+      echo
+      echo "# Packages"
+      echo
+      jq -r -f ${./generate-step-summary.jq} < ${changed-paths}
+    } >> $out/step-summary.md
 
     if jq -e '(.attrdiff.added | length == 0) and (.attrdiff.removed | length == 0)' "${changed-paths}" > /dev/null; then
       # Chunks have changed between revisions
@@ -175,12 +181,5 @@ runCommand "compare"
       } >> $out/step-summary.md
     fi
 
-    {
-      echo
-      echo "# Packages"
-      echo
-      jq -r -f ${./generate-step-summary.jq} < ${changed-paths}
-    } >> $out/step-summary.md
-
     cp "$maintainersPath" "$out/maintainers.json"
   ''

ci/pinned.json

@@ -9,9 +9,9 @@
       },
       "branch": "nixpkgs-unstable",
       "submodules": false,
-      "revision": "6a489c9482ca676ce23c0bcd7f2e1795383325fa",
-      "url": "https://github.com/NixOS/nixpkgs/archive/6a489c9482ca676ce23c0bcd7f2e1795383325fa.tar.gz",
-      "hash": "0vsvkhy3gb8yzq62vazhmpqixssmd4xinnll7w73l4vrqd611wlf"
+      "revision": "641d909c4a7538f1539da9240dedb1755c907e40",
+      "url": "https://github.com/NixOS/nixpkgs/archive/641d909c4a7538f1539da9240dedb1755c907e40.tar.gz",
+      "hash": "10hpb1aw884k3zzcy1mhf47dqvfagiyx7kr6hg0p5xcwg04mkx8x"
     },
     "treefmt-nix": {
       "type": "Git",
@@ -22,9 +22,9 @@
       },
       "branch": "main",
       "submodules": false,
-      "revision": "58bd4da459f0a39e506847109a2a5cfceb837796",
-      "url": "https://github.com/numtide/treefmt-nix/archive/58bd4da459f0a39e506847109a2a5cfceb837796.tar.gz",
-      "hash": "01bg9b4xzlv6s5q1q78vib6l2csw02b3rk5bm5yj4gx2sk2hvmrq"
+      "revision": "7d81f6fb2e19bf84f1c65135d1060d829fae2408",
+      "url": "https://github.com/numtide/treefmt-nix/archive/7d81f6fb2e19bf84f1c65135d1060d829fae2408.tar.gz",
+      "hash": "1cg20q8ja8k2nb7mzy95hgmd8whxapc3fbyndh1ip5dr6d1grxfs"
     }
   },
   "version": 5

ci/supportedVersions.nix

@@ -0,0 +1,32 @@
+#!/usr/bin/env -S nix-instantiate --eval --strict --json --arg unused true
+# Unused argument to trigger nix-instantiate calling this function with the default arguments.
+{
+  pinnedJson ? ./pinned.json,
+}:
+let
+  pinned = (builtins.fromJSON (builtins.readFile pinnedJson)).pins;
+  nixpkgs = fetchTarball {
+    inherit (pinned.nixpkgs) url;
+    sha256 = pinned.nixpkgs.hash;
+  };
+  pkgs = import nixpkgs {
+    config.allowAliases = false;
+  };
+
+  inherit (pkgs) lib;
+
+  lix = lib.pipe pkgs.lixPackageSets [
+    (lib.filterAttrs (_: set: lib.isDerivation set.lix or null && set.lix.meta.available))
+    lib.attrNames
+    (lib.filter (name: lib.match "lix_[0-9_]+|git" name != null))
+    (map (name: "lixPackageSets.${name}.lix"))
+  ];
+
+  nix = lib.pipe pkgs.nixVersions [
+    (lib.filterAttrs (_: drv: lib.isDerivation drv && drv.meta.available))
+    lib.attrNames
+    (lib.filter (name: lib.match "nix_[0-9_]+|git" name != null))
+    (map (name: "nixVersions.${name}"))
+  ];
+in
+lix ++ nix