Fix GitHub Actions artifact ordering

.github/scripts/extract_matrix.py

@@ -360,7 +360,7 @@ def __gt__(self, other):
         return NotImplemented
 
 
-def get_tagged_jobs(buildspec, target, filter=None):
+def get_tagged_jobs(buildspec, target, filter=None, artifact_mode="all"):
     jobs = [Job({"name": target}).to_dict()]
     for job in sorted([Job(build) for build in buildspec.get("builds", [])]):
         if not any(t for t in job.targets if t in [target]):
@@ -371,15 +371,19 @@ def get_tagged_jobs(buildspec, target, filter=None):
             continue
         if [x for x in JOB_EXCLUSION_TERMS if x in str(job)]:
             continue
+        if artifact_mode == "providers" and not job.upload_artifact:
+            continue
+        if artifact_mode == "non-providers" and job.upload_artifact:
+            continue
         jobs.append(job.to_dict())
     return jobs
 
 
-def main(jsonnet_bin, ci_jsonnet, target, filter=None, indent=False):
+def main(jsonnet_bin, ci_jsonnet, target, filter=None, artifact_mode="all", indent=False):
 
     result = subprocess.check_output([jsonnet_bin, ci_jsonnet], text=True)
     buildspec = json.loads(result)
-    tagged_jobs = get_tagged_jobs(buildspec, target, filter=filter)
+    tagged_jobs = get_tagged_jobs(buildspec, target, filter=filter, artifact_mode=artifact_mode)
     matrix = tagged_jobs
     print(json.dumps(matrix, indent=2 if indent else None))
 
@@ -390,12 +394,19 @@ def main(jsonnet_bin, ci_jsonnet, target, filter=None, indent=False):
     parser.add_argument("ci_jsonnet", help="Path to ci.jsonnet spec")
     parser.add_argument("target", help="Target name (e.g., tier1)")
     parser.add_argument("filter", nargs="?", default=None, help="Regex filter for job names (optional)")
+    parser.add_argument(
+        "--artifact-mode",
+        choices=["all", "providers", "non-providers"],
+        default="all",
+        help="Select all jobs, only artifact providers, or only jobs that do not publish artifacts.",
+    )
     parser.add_argument('--indent', action='store_true', help='Indent output JSON')
     args = parser.parse_args()
     main(
         jsonnet_bin=args.jsonnet_bin,
         ci_jsonnet=args.ci_jsonnet,
         target=args.target,
         filter=args.filter,
+        artifact_mode=args.artifact_mode,
         indent=args.indent or sys.stdout.isatty(),
     )

.github/scripts/unpack-artifact

@@ -1,17 +1,24 @@
 #!/bin/bash
+set -euo pipefail
 
-to_extract="$ARTIFACT_PATHS"
+to_extract="${ARTIFACT_PATHS:-}"
 if [ -n "$1" ]; then
    to_extract="${ARTIFACT_PATH_PREFIX}/${1}.tar"
 fi
 
+if [ -z "$to_extract" ]; then
+   echo "::error::No artifact archive path was provided."
+   exit 1
+fi
+
 for i in $to_extract; do
    if [ -f "$i" ]; then
       tar -x -f "$i" -C "${ARTIFACT_PATH_PREFIX}" -v
       echo "Unpacked $i in $(pwd)/${ARTIFACT_PATH_PREFIX}"
    else
-      echo "file not found in ${ARTIFACT_PATH_PREFIX}"
+      echo "::error::Required artifact archive not found: $i"
       ls "${ARTIFACT_PATH_PREFIX}"
+      exit 1
    fi
 
-done
+done

.github/workflows/ci-matrix-gen.yml

@@ -29,6 +29,7 @@ jobs:
   generate-tier1:
     runs-on: ubuntu-latest
     outputs:
+      artifact_matrix: ${{ steps.set-matrix.outputs.artifact_matrix }}
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     env:
       TARGET: tier1
@@ -42,13 +43,17 @@ jobs:
       - name: Extract job matrix
         id: set-matrix
         run: |
-          python3 .github/scripts/extract_matrix.py ./sjsonnet ./ci.jsonnet ${TARGET} ${JOBS} > matrix.json
+          python3 .github/scripts/extract_matrix.py ./sjsonnet ./ci.jsonnet ${TARGET} ${JOBS} --artifact-mode providers > artifact-matrix.json
+          python3 .github/scripts/extract_matrix.py ./sjsonnet ./ci.jsonnet ${TARGET} ${JOBS} --artifact-mode non-providers > matrix.json
+          cat artifact-matrix.json
           cat matrix.json
+          echo "artifact_matrix=$(cat artifact-matrix.json)" >> $GITHUB_OUTPUT
           echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
 
   generate-tier2:
     runs-on: ubuntu-latest
     outputs:
+      artifact_matrix: ${{ steps.set-matrix.outputs.artifact_matrix }}
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     env:
       TARGET: tier2
@@ -58,19 +63,20 @@ jobs:
   generate-tier3:
     runs-on: ubuntu-latest
     outputs:
+      artifact_matrix: ${{ steps.set-matrix.outputs.artifact_matrix }}
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     env:
       TARGET: tier3
       JOBS: ${{ inputs.jobs_to_run }}
     steps: *generate_matrix
 
-  tier1:
+  tier1-artifacts:
     needs: generate-tier1
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        include: ${{ fromJson(needs.generate-tier1.outputs.matrix) }}
+        include: ${{ fromJson(needs.generate-tier1.outputs.artifact_matrix) }}
     steps: &buildsteps
       - name: Process matrix downloads
         if: ${{ matrix.downloads_steps }}
@@ -168,9 +174,9 @@ jobs:
           git config --global http.timeout 600
           git clone https://github.com/graalvm/mx
           if [[ "${RUNNER_OS}" == "Windows" ]]; then
-            ./mx/mx.cmd fetch-jdk -A --jdk-id labsjdk-ce-latest
+            ./mx/mx.cmd fetch-jdk -A --configuration main/ci/graal/common.json --jdk-id labsjdk-ce-latest
           else
-            ./mx/mx fetch-jdk -A --jdk-id labsjdk-ce-latest
+            ./mx/mx fetch-jdk -A --configuration main/ci/graal/common.json --jdk-id labsjdk-ce-latest
           fi
           git -C mx checkout ${{ matrix.mx_version }}
 
@@ -200,7 +206,6 @@ jobs:
         with:
           pattern: ${{ matrix.require_artifact[0] }}
           merge-multiple: true
-        continue-on-error: true
 
       - name: Export artifact paths Linux
         if: ${{ matrix.require_artifact }}
@@ -288,20 +293,50 @@ jobs:
           path: /tmp/test-report-*.json
           retention-days: 1
 
-  tier2:
-    if: ${{ success() || inputs.jobs_to_run }}
+  tier1:
+    if: ${{ always() && needs.generate-tier1.result == 'success' && needs.tier1-artifacts.result == 'success' }}
+    needs: [generate-tier1, tier1-artifacts]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.generate-tier1.outputs.matrix) }}
+    steps: *buildsteps
+
+  tier2-artifacts:
+    if: ${{ always() && needs.generate-tier2.result == 'success' && (needs.tier1.result == 'success' || inputs.jobs_to_run) }}
     needs: [generate-tier2, tier1]
     runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.generate-tier2.outputs.artifact_matrix) }}
+    steps: *buildsteps
+
+  tier2:
+    if: ${{ always() && needs.generate-tier2.result == 'success' && needs.tier2-artifacts.result == 'success' }}
+    needs: [generate-tier2, tier2-artifacts]
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         include: ${{ fromJson(needs.generate-tier2.outputs.matrix) }}
     steps: *buildsteps
 
-  tier3:
-    if: ${{ success() || inputs.jobs_to_run }}
+  tier3-artifacts:
+    if: ${{ always() && needs.generate-tier3.result == 'success' && (needs.tier2.result == 'success' || inputs.jobs_to_run) }}
     needs: [generate-tier3, tier2]
     runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.generate-tier3.outputs.artifact_matrix) }}
+    steps: *buildsteps
+
+  tier3:
+    if: ${{ always() && needs.generate-tier3.result == 'success' && needs.tier3-artifacts.result == 'success' }}
+    needs: [generate-tier3, tier3-artifacts]
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:

graalpython/com.oracle.graal.python.test/src/tests/test_subprocess.py

@@ -230,47 +230,52 @@ def test_graal_python_args(self):
         if sys.implementation.name == "graalpy":
             import subprocess
 
-            env = {"GRAAL_PYTHON_ARGS": "-c 12"}
+            def env_with_graal_python_args(args):
+                env = os.environ.copy()
+                env["GRAAL_PYTHON_ARGS"] = args
+                return env
+
+            env = env_with_graal_python_args("-c 12")
             result = subprocess.run([sys.executable], env=env)
             self.assertEqual(0, result.returncode)
 
-            env = {"GRAAL_PYTHON_ARGS": "-c 'print(12)'"}
+            env = env_with_graal_python_args("-c 'print(12)'")
             result = subprocess.check_output([sys.executable], env=env, text=True)
             self.assertEqual('12\n', result)
 
-            env = {"GRAAL_PYTHON_ARGS": """-c 'print("Hello world")'"""}
+            env = env_with_graal_python_args("""-c 'print("Hello world")'""")
             result = subprocess.check_output([sys.executable], env=env, text=True)
             self.assertEqual('Hello world\n', result)
 
-            env = {"GRAAL_PYTHON_ARGS": """-c ""'print("Hello world")'"""""}
+            env = env_with_graal_python_args("""-c ""'print("Hello world")'""""")
             result = subprocess.check_output([sys.executable], env=env, text=True)
             self.assertEqual('Hello world\n', result)
 
-            env = {"GRAAL_PYTHON_ARGS": r"""-c 'print(\'"Hello world"\')'"""""}
+            env = env_with_graal_python_args(r"""-c 'print(\'"Hello world"\')'""")
             result = subprocess.check_output([sys.executable], env=env, text=True)
             self.assertEqual('"Hello world"\n', result)
 
-            env = {"GRAAL_PYTHON_ARGS": """\v-c\vprint('"Hello world"')"""}
+            env = env_with_graal_python_args("""\v-c\vprint('"Hello world"')""")
             result = subprocess.check_output([sys.executable], env=env, text=True)
             self.assertEqual('"Hello world"\n', result)
 
-            env = {"GRAAL_PYTHON_ARGS": """\v-c\vprint('Hello', "world")"""}
+            env = env_with_graal_python_args("""\v-c\vprint('Hello', "world")""")
             result = subprocess.check_output([sys.executable], env=env, text=True)
             self.assertEqual('Hello world\n', result)
 
             # check that the subprocess receives the args and thus it should fail because it recurses
             args = """\v-c\vimport os\nprint(os.environ.get("GRAAL_PYTHON_ARGS"))"""
-            env = {"GRAAL_PYTHON_ARGS": args}
+            env = env_with_graal_python_args(args)
             result = subprocess.check_output([sys.executable], env=env, text=True)
             self.assertEqual(f"{args}\n", result)
 
             # check that the subprocess does not receive the args when we end with \v
-            env = {"GRAAL_PYTHON_ARGS": """\v-c\vimport os\nprint(os.environ.get("GRAAL_PYTHON_ARGS"))\v"""}
+            env = env_with_graal_python_args("""\v-c\vimport os\nprint(os.environ.get("GRAAL_PYTHON_ARGS"))\v""")
             result = subprocess.check_output([sys.executable], env=env, text=True)
             self.assertEqual('None\n', result)
 
             # check that the subprocess receives an empty arg
             args = """\v-c\vimport sys\nprint(repr(sys.argv))\va1\v\va3"""
-            env = {"GRAAL_PYTHON_ARGS": args}
+            env = env_with_graal_python_args(args)
             result = subprocess.check_output([sys.executable], env=env, text=True)
             self.assertEqual("['-c', 'a1', '', 'a3']\n", result)

graalpython/com.oracle.graal.python.test/src/tests/test_venv.py

@@ -74,16 +74,21 @@ def test_venv_launcher(self):
             with open(tmpfile, "ab") as f:
                 sz = f.write(launcher_command.encode("utf-16le"))
                 assert f.write(struct.pack("@I", sz)) == 4
+            env = os.environ.copy()
+            env["PYLAUNCHER_DEBUG"] = "1"
             try:
                 out = subprocess.check_output([tmpfile, "-c", """if True:
                 import sys, os
                 x = os
                 print("Hello", sys.executable)
                 print("Base", sys._base_executable)
                 print("Original", __graalpython__.venvlauncher_command)
-                """], env={"PYLAUNCHER_DEBUG": "1"}, text=True)
+                """], env=env, stderr=subprocess.STDOUT, text=True)
             except subprocess.CalledProcessError as err:
-                out = err.output.decode(errors="replace") if err.output else ""
+                if isinstance(err.output, str):
+                    out = err.output
+                else:
+                    out = err.output.decode(errors="replace") if err.output else ""
             print("out=", out, sep="\n")
             assert f"Hello {tmpfile}" in out, out
             assert f"Base {os.path.realpath(sys.executable)}" in out, out

mx.graalpython/mx_graalpython.py

@@ -3116,6 +3116,10 @@ def apply(self, config):
     def processDeps(self, deps):
         if PythonMxUnittestConfig.useResources:
             deps.add(mx.distribution('GRAALPYTHON_RESOURCES', fatalIfMissing=True))
+        if mx.suite('compiler', fatalIfMissing=False):
+            # punittest enables compiler-failure checks when /compiler is imported;
+            # the corresponding engine options are registered by the optimized Truffle runtime.
+            deps.add(mx.distribution('truffle:TRUFFLE_RUNTIME', fatalIfMissing=True))
 
 
 mx_unittest.register_unittest_config(PythonMxUnittestConfig('python-internal'))