From 5a3fe7819105a050402a92aba1196ea2fc3c360a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 30 Jul 2026 19:31:39 +0200 Subject: [PATCH 1/2] fix(parity): support Windows harness runs --- .github/workflows/test.yml | 69 +++++------ run_parity_tests.sh | 193 ++++++++++++++++++++++--------- scripts/gap_snapshot.py | 10 +- scripts/parity_known_failures.py | 174 ++++++++++++++++++++++++++++ scripts/parity_matrix_trend.py | 41 ++++++- scripts/run_gap_tests.sh | 50 +++++++- test-parity/README.md | 28 +++++ test-parity/known_failures.json | 3 +- tests/test_parity_build_reuse.sh | 43 +++++++ 9 files changed, 506 insertions(+), 105 deletions(-) create mode 100644 scripts/parity_known_failures.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2e92495b2a..ad2ae1cd52 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -184,6 +184,9 @@ jobs: - name: Gap snapshot checker self-test run: python3 scripts/gap_snapshot.py --self-test + - name: Platform-aware parity allowlist self-test + run: python3 scripts/parity_known_failures.py --self-test + # --------------------------------------------------------------------------- # Clippy — enforces the deny-level lints in [workspace.lints] (root # Cargo.toml). `cargo clippy` exits nonzero only on `deny` lints, so @@ -811,6 +814,11 @@ jobs: - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable + - name: Setup Node.js + uses: actions/setup-node@v7 + with: + node-version-file: .node-version + - uses: Swatinem/rust-cache@v2 with: shared-key: "${{ runner.os }}-perry" @@ -819,10 +827,25 @@ jobs: # perry-dev (see Cargo.toml [profile.perry-dev]) trades peak optimization # for build speed. The package set covers the compiler binary (link # gate), the runtime/stdlib pair (the usual source of cfg(windows) - # externs), and both Windows UI crates — the ones cargo-test's ubuntu + # externs), their static-library wrappers (used by the parity smoke + # below), and both Windows UI crates — the ones cargo-test's ubuntu # runner must exclude and therefore never compiles. - name: Build compiler + runtime + Windows UI crates (perry-dev) - run: cargo build --profile perry-dev -p perry -p perry-runtime -p perry-stdlib -p perry-ui-windows -p perry-ui-windows-winui + run: cargo build --profile perry-dev -p perry -p perry-runtime -p perry-stdlib -p perry-runtime-static -p perry-stdlib-static -p perry-ui-windows -p perry-ui-windows-winui + + # Small deterministic subset: verifies the Git Bash driver itself, + # `.exe`/`.lib` discovery, native TEMP paths, compilation, execution, + # and Node/Perry comparison on a real Windows host. + - name: Windows parity harness smoke + shell: bash + run: | + PERRY_SKIP_BUILD=1 \ + PERRY_BIN="$PWD/target/perry-dev/perry.exe" \ + PERRY_RUNTIME_DIR="$PWD/target/perry-dev" \ + ./run_parity_tests.sh \ + --suite node-suite \ + --module process \ + --filter process/env/access # --------------------------------------------------------------------------- # GC write-barrier stress (optional / non-blocking) @@ -1320,44 +1343,10 @@ jobs: exit "$status" - name: Check for new failures - run: | - REPORT="test-parity/reports/latest.json" - KNOWN="test-parity/known_failures.json" - - # Filter empty strings — `run_parity_tests.sh` emits `compile: [""]` - # when there are zero compile failures (a printf+sed quirk in the - # JSON generator), and that empty entry would propagate to a - # spurious "NEW FAILURES: - " line and fail this gate. - jq -r '(.failures.parity // []) + (.failures.compile // []) | .[] | select(. != "")' "$REPORT" | sort -u > /tmp/all_fails.txt - if [[ -f "$KNOWN" ]]; then - # Issue #797 — known_failures.json moved from flat strings to - # structured records. Keep the audit metadata (the `_schema` - # key at the top of the file) out of the test-name set so - # CI doesn't try to match a real test against it. - jq -r 'keys[] | select(. != "_schema")' "$KNOWN" | sort -u > /tmp/known.txt - - # Schema sanity check — every non-_schema entry must be an - # object with non-empty `category` and `reason`. Fails the - # build on malformed entries so the format doesn't silently - # drift back to the legacy flat-string shape. - bad="$(jq -r 'to_entries | map(select(.key != "_schema")) | .[] | select((.value | type) != "object" or (.value.category // "") == "" or (.value.reason // "") == "") | .key' "$KNOWN")" - if [[ -n "$bad" ]]; then - echo "Malformed known_failures.json entries (missing category/reason or not an object):" - echo "$bad" | sed 's/^/ - /' - exit 1 - fi - else - : > /tmp/known.txt - fi - - TOTAL=$(wc -l < /tmp/all_fails.txt | tr -d ' ') - comm -23 /tmp/all_fails.txt /tmp/known.txt > /tmp/new.txt - if [[ -s /tmp/new.txt ]]; then - echo "NEW FAILURES (not in known_failures.json):" - sed 's/^/ - /' /tmp/new.txt - exit 1 - fi - echo "All ${TOTAL} failures are known/triaged." + run: >- + python3 scripts/parity_known_failures.py + --report test-parity/reports/latest.json + --known test-parity/known_failures.json - name: Generate parity matrix trend run: | diff --git a/run_parity_tests.sh b/run_parity_tests.sh index 8dd7b536be..a67514e261 100755 --- a/run_parity_tests.sh +++ b/run_parity_tests.sh @@ -7,11 +7,55 @@ TEST_DIR="$SCRIPT_DIR/test-files" NODE_SUITE_DIR="$SCRIPT_DIR/test-parity/node-suite" OUTPUT_DIR="$SCRIPT_DIR/test-parity/output" REPORT_DIR="$SCRIPT_DIR/test-parity/reports" + +# Normalize the host once. `uname` under Git Bash reports MINGW/MSYS rather +# than Windows, while CI metadata and the parity allowlists use stable, +# lowercase platform names. PERRY_HOST_PLATFORM is also a deliberate test hook +# for exercising the Windows shell path on non-Windows hosts. +if [[ -n "${PERRY_HOST_PLATFORM:-}" ]]; then + HOST_PLATFORM="$PERRY_HOST_PLATFORM" +else + case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) HOST_PLATFORM="windows" ;; + Darwin) HOST_PLATFORM="macos" ;; + Linux) HOST_PLATFORM="linux" ;; + *) HOST_PLATFORM="other" ;; + esac +fi +case "$HOST_PLATFORM" in + windows|macos|linux|other) ;; + *) + echo "Invalid PERRY_HOST_PLATFORM '$HOST_PLATFORM' (want windows, macos, linux, or other)" >&2 + exit 1 + ;; +esac + +# Git Bash exposes the native TEMP/TMP directories even when TMPDIR is unset. +# Prefer those before the Unix fallback, and translate a native `C:\...` path +# into the POSIX spelling expected by Bash utilities. +TEMP_ROOT="${TMPDIR:-${TEMP:-${TMP:-/tmp}}}" +if [[ "$HOST_PLATFORM" == "windows" ]] && command -v cygpath &>/dev/null; then + TEMP_ROOT="$(cygpath -u "$TEMP_ROOT")" +fi +mkdir -p "$TEMP_ROOT" + +PYTHON_CMD="" +if command -v python3 &>/dev/null; then + PYTHON_CMD="python3" +elif command -v python &>/dev/null; then + # GitHub's Windows image exposes the setup-python shim as `python` on + # some revisions and `python3` on others. + PYTHON_CMD="python" +fi +if [[ -z "$PYTHON_CMD" ]]; then + echo "Python 3 is required by the parity output normalizer" >&2 + exit 1 +fi # Per-run scratch dir for compiled test binaries (2026-07-02 audit): the old # fixed /tmp/perry_parity_ paths meant two concurrent suite runs # (two agents / two worktrees on one machine) executed EACH OTHER'S compiler # output — cross-contaminated pass/fail attributed to the wrong build. -PARITY_TMP="$(mktemp -d "${TMPDIR:-/tmp}/perry-parity.XXXXXX")" +PARITY_TMP="$(mktemp -d "$TEMP_ROOT/perry-parity.XXXXXX")" trap 'rm -rf "$PARITY_TMP"' EXIT # LLVM is the only backend post-Phase K hard cutover. The --llvm / @@ -130,7 +174,7 @@ wait_for_tcp_port() { local port=$2 local attempts=$3 local delay=${4:-0.1} - python3 - "$host" "$port" "$attempts" "$delay" <<'PY' + "$PYTHON_CMD" - "$host" "$port" "$attempts" "$delay" <<'PY' import socket import sys import time @@ -160,15 +204,15 @@ TLS_UPGRADE_SERVER_PID="" start_tls_upgrade_server() { local server_script="$SCRIPT_DIR/test-files/test_net_upgrade_tls_server.py" - if ! command -v python3 &>/dev/null; then - echo -e "${YELLOW}WARN${NC} python3 not found — test_net_upgrade_tls will fail parity" >&2 + if [[ -z "$PYTHON_CMD" ]]; then + echo -e "${YELLOW}WARN${NC} python not found — test_net_upgrade_tls will fail parity" >&2 return 1 fi if [[ ! -f "$server_script" ]]; then echo -e "${YELLOW}WARN${NC} $server_script not found — test_net_upgrade_tls will fail parity" >&2 return 1 fi - python3 "$server_script" --port 17892 & + "$PYTHON_CMD" "$server_script" --port 17892 & TLS_UPGRADE_SERVER_PID=$! # Wait up to 3 s for the port to open. wait_for_tcp_port 127.0.0.1 17892 30 0.1 && return 0 @@ -314,15 +358,15 @@ normalize_output() { # while-read loop with `decoded+="$line"\n` per iteration. That's # O(n²) on input size: 5.7M lines × 2.85M-char-average tail ≈ 16T # bytes of string concatenation, which burned ~3 hours on CI before - # the runner was killed. Replaced with a single python3 pass — + # the runner was killed. Replaced with a single Python 3 pass — # linear time, decodes `` to its UTF-8 bytes in - # one walk. python3 is preinstalled on every ubuntu/macos runner. + # one walk. Python 3 is preinstalled on the hosted CI runners. # # The decode is bytes-faithful: invalid UTF-8 sequences become U+FFFD # via `errors="replace"`, matching the pre-fix `xxd -r -p` behavior # for arbitrary binary content. local decoded - decoded=$(printf '%s' "$input" | python3 -c ' + decoded=$(printf '%s' "$input" | "$PYTHON_CMD" -c ' import sys for raw in sys.stdin: line = raw.rstrip("\n").rstrip("\r") @@ -417,6 +461,13 @@ echo "" # release binary the prior step had just produced, and (b) adds cargo's own # per-invocation overhead × ~150 tests. TARGET_DIR="${CARGO_TARGET_DIR:-$SCRIPT_DIR/target}" +if [[ "$HOST_PLATFORM" == "windows" ]] && command -v cygpath &>/dev/null; then + TARGET_DIR="$(cygpath -u "$TARGET_DIR")" +fi +PERRY_EXE_SUFFIX="" +if [[ "$HOST_PLATFORM" == "windows" ]]; then + PERRY_EXE_SUFFIX=".exe" +fi PERRY_SKIP_BUILD="${PERRY_SKIP_BUILD:-0}" case "$PERRY_SKIP_BUILD" in 0|1) ;; @@ -427,25 +478,40 @@ case "$PERRY_SKIP_BUILD" in esac if [[ "$PERRY_SKIP_BUILD" == "1" ]]; then - PERRY_BIN="${PERRY_BIN:-$TARGET_DIR/release/perry}" + PERRY_BIN="${PERRY_BIN:-$TARGET_DIR/release/perry$PERRY_EXE_SUFFIX}" + if [[ "$HOST_PLATFORM" == "windows" ]] && command -v cygpath &>/dev/null; then + PERRY_BIN="$(cygpath -u "$PERRY_BIN")" + fi if [[ ! -x "$PERRY_BIN" ]]; then echo -e "${RED}PERRY_BIN is not executable: $PERRY_BIN${NC}" >&2 exit 1 fi PERRY_RUNTIME_DIR="${PERRY_RUNTIME_DIR:-$(cd "$(dirname "$PERRY_BIN")" && pwd)}" - case "$(uname -s)" in - MINGW*|MSYS*|CYGWIN*) runtime_lib=perry_runtime.lib; stdlib_lib=perry_stdlib.lib ;; - *) runtime_lib=libperry_runtime.a; stdlib_lib=libperry_stdlib.a ;; - esac + if [[ "$HOST_PLATFORM" == "windows" ]] && command -v cygpath &>/dev/null; then + PERRY_RUNTIME_DIR="$(cygpath -u "$PERRY_RUNTIME_DIR")" + fi + if [[ "$HOST_PLATFORM" == "windows" ]]; then + runtime_lib=perry_runtime.lib + stdlib_lib=perry_stdlib.lib + else + runtime_lib=libperry_runtime.a + stdlib_lib=libperry_stdlib.a + fi if [[ ! -f "$PERRY_RUNTIME_DIR/$runtime_lib" || ! -f "$PERRY_RUNTIME_DIR/$stdlib_lib" ]]; then echo -e "${RED}PERRY_RUNTIME_DIR must contain $runtime_lib and $stdlib_lib: $PERRY_RUNTIME_DIR${NC}" >&2 exit 1 fi + PERRY_RUNTIME_DIR_SHELL="$PERRY_RUNTIME_DIR" + if [[ "$HOST_PLATFORM" == "windows" ]] && command -v cygpath &>/dev/null; then + # Native Windows processes do not receive Git Bash's argv path + # conversion for arbitrary environment variables. + PERRY_RUNTIME_DIR="$(cygpath -w "$PERRY_RUNTIME_DIR")" + fi export PERRY_RUNTIME_DIR PERRY_NO_AUTO_OPTIMIZE=1 echo "Using prebuilt compiler: $PERRY_BIN" - echo "Using prebuilt runtime archives: $PERRY_RUNTIME_DIR" + echo "Using prebuilt runtime archives: $PERRY_RUNTIME_DIR_SHELL" else - PERRY_BIN="$TARGET_DIR/release/perry" + PERRY_BIN="$TARGET_DIR/release/perry$PERRY_EXE_SUFFIX" echo "Building compiler (release)..." fi BUILD_PACKAGES=(-p perry -p perry-runtime -p perry-stdlib -p perry-runtime-static -p perry-stdlib-static) @@ -574,15 +640,15 @@ ECHO_SERVER_PID="" ECHO_SERVER_SCRIPT="$SCRIPT_DIR/test-files/test_net_echo_server.py" start_echo_server() { - if ! command -v python3 &>/dev/null; then - echo "Warning: python3 not found — test_net_min / test_net_socket will fail parity" + if [[ -z "$PYTHON_CMD" ]]; then + echo "Warning: python not found — test_net_min / test_net_socket will fail parity" return fi if [[ ! -f "$ECHO_SERVER_SCRIPT" ]]; then echo "Warning: $ECHO_SERVER_SCRIPT not found — test_net_min / test_net_socket will fail parity" return fi - python3 "$ECHO_SERVER_SCRIPT" & + "$PYTHON_CMD" "$ECHO_SERVER_SCRIPT" & ECHO_SERVER_PID=$! # Poll up to 5 s (50 × 100 ms) for the server to accept connections. local ready=0 @@ -604,7 +670,12 @@ stop_echo_server() { fi } -trap stop_echo_server EXIT +cleanup_parity_run() { + stop_echo_server + rm -rf "$PARITY_TMP" +} + +trap cleanup_parity_run EXIT # The granular node-suite starts with deterministic module cases (path, url, # etc.) that do not need the legacy top-level net echo server. Future net @@ -697,7 +768,7 @@ for test_file in "${TEST_FILES[@]}"; do safe_test_id="${test_id//\//__}" node_output_file="$OUTPUT_DIR/node/${safe_test_id}.txt" perry_output_file="$OUTPUT_DIR/perry/${safe_test_id}.txt" - perry_binary="$PARITY_TMP/perry_parity_$safe_test_id" + perry_binary="$PARITY_TMP/perry_parity_$safe_test_id$PERRY_EXE_SUFFIX" parity_test_file="$test_file" perry_compile_command=() parity_argv_line=$(sed -n -E 's|^[[:space:]]*//[[:space:]]*parity-argv:[[:space:]]*(.*)$|\1|p' "$test_file" | head -1) @@ -738,7 +809,7 @@ for test_file in "${TEST_FILES[@]}"; do # memory and DOS the runner. PIPESTATUS doesn't propagate across # `$(...)`, so capturing the exit code requires the file detour # rather than a `cmd | cap_output` pipeline. - node_tmp=$(mktemp) + node_tmp="$PARITY_TMP/${safe_test_id}.node-output" run_with_timeout 10 env FORCE_COLOR=0 NO_COLOR=1 NODE_DISABLE_COLORS=1 \ node --experimental-strip-types "${node_argv[@]}" "$test_file" "${test_argv[@]}" > "$node_tmp" 2>&1 node_exit=$? @@ -825,7 +896,7 @@ for test_file in "${TEST_FILES[@]}"; do # Perry's `[perry] warning: ... is a stub` lines. They'd otherwise show # up on the 2>&1-captured stream for any test that exercises a flagged # API (dns/dgram loopback, v8 heap snapshot, …) and diff against Node. - perry_tmp=$(mktemp) + perry_tmp="$PARITY_TMP/${safe_test_id}.perry-output" # Cap the test binary's fork budget at current-user-tasks + 50: legit # multi-process tests (cluster/child_process) fit easily, while a # fork-bombing test (cluster re-exec loop, 2026-07-22) stalls at ~50 @@ -837,35 +908,48 @@ for test_file in "${TEST_FILES[@]}"; do # so a concurrent heavy job can transiently eat the +50 headroom; recomputing # per test keeps that window small, and the worst case is one test # classified as crash rather than a wedged machine. - run_with_timeout 10 "$BASH" -c ' - if [ "$(uname -s)" = "Linux" ]; then - proc_list=$(ps -u "$(id -u)" -L -o lwp= 2>/dev/null) - else - proc_list=$(ps -u "$(id -u)" -o pid= 2>/dev/null) - fi || { - echo "failed to measure the current user process count" >&2 - exit 125 - } - nproc_now=$(printf "%s\n" "$proc_list" | awk "NF { count++ } END { print count + 0 }") || { - echo "failed to count the current user processes" >&2 - exit 125 - } - if ! [ "$nproc_now" -gt 0 ] 2>/dev/null; then - echo "invalid current user process count: $nproc_now" >&2 - exit 125 - fi - if ! ulimit -u $(( nproc_now + 50 )) 2>/dev/null; then - echo "failed to apply the per-test process limit" >&2 - exit 125 - fi - exec "$@"' _ env PERRY_STUB_DIAG=off "${parity_env[@]}" "$perry_binary" "${test_argv[@]}" > "$perry_tmp" 2>&1 + if [[ "$HOST_PLATFORM" == "windows" ]]; then + # Git Bash has no enforceable RLIMIT_NPROC (`ulimit -u`) for native + # Windows processes. The outer timeout still bounds the direct test; + # Windows CI should additionally use the job-level Actions timeout. + run_with_timeout 10 env PERRY_STUB_DIAG=off "${parity_env[@]}" \ + "$perry_binary" "${test_argv[@]}" > "$perry_tmp" 2>&1 + else + run_with_timeout 10 "$BASH" -c ' + if [ "$(uname -s)" = "Linux" ]; then + proc_list=$(ps -u "$(id -u)" -L -o lwp= 2>/dev/null) + else + proc_list=$(ps -u "$(id -u)" -o pid= 2>/dev/null) + fi || { + echo "failed to measure the current user process count" >&2 + exit 125 + } + nproc_now=$(printf "%s\n" "$proc_list" | awk "NF { count++ } END { print count + 0 }") || { + echo "failed to count the current user processes" >&2 + exit 125 + } + if ! [ "$nproc_now" -gt 0 ] 2>/dev/null; then + echo "invalid current user process count: $nproc_now" >&2 + exit 125 + fi + if ! ulimit -u $(( nproc_now + 50 )) 2>/dev/null; then + echo "failed to apply the per-test process limit" >&2 + exit 125 + fi + exec "$@"' _ env PERRY_STUB_DIAG=off "${parity_env[@]}" "$perry_binary" "${test_argv[@]}" > "$perry_tmp" 2>&1 + fi perry_exit=$? - # Reap orphaned children of the test binary (cluster tests fork workers - # that survive the timeout kill of the direct child and can fork-bomb the - # machine — 2026-07-22). Scoped to this run's tmp dir, so concurrent - # suite runs are unaffected. pkill -f treats the pattern as a regex, so - # escape the path's metacharacters (the mktemp dir contains a dot). - pkill -9 -f "$(printf '%s/' "$PARITY_TMP" | sed 's/[][\.*^$()+?{|]/\\&/g')" 2>/dev/null + # Reap orphaned Unix children of the test binary (cluster tests fork + # workers that survive the timeout kill of the direct child and can + # fork-bomb the machine — 2026-07-22). Scoped to this run's tmp dir, so + # concurrent suite runs are unaffected. Git Bash does not provide a + # process-tree primitive equivalent to a Windows Job Object, so its + # supported CI subset relies on the job timeout documented above. + if [[ "$HOST_PLATFORM" != "windows" ]] && command -v pkill &>/dev/null; then + # pkill -f treats the pattern as a regex; escape the path's + # metacharacters (the mktemp dir contains a dot). + pkill -9 -f "$(printf '%s/' "$PARITY_TMP" | sed 's/[][\.*^$()+?{|]/\\&/g')" 2>/dev/null + fi perry_output=$(cap_output < "$perry_tmp") rm -f "$perry_tmp" @@ -951,7 +1035,8 @@ done # Calculate parity percentage TOTAL_RUN=$((PARITY_PASS + PARITY_FAIL + CRASH_FAIL)) if [[ $TOTAL_RUN -gt 0 ]]; then - PARITY_PCT=$(echo "scale=1; $PARITY_PASS * 100 / $TOTAL_RUN" | bc) + PARITY_TENTHS=$((PARITY_PASS * 1000 / TOTAL_RUN)) + PARITY_PCT="$((PARITY_TENTHS / 10)).$((PARITY_TENTHS % 10))" else PARITY_PCT="0.0" fi @@ -1003,6 +1088,7 @@ RESULTS_JSON=$(printf '%s\n' "${TEST_RESULTS[@]}" | paste -sd, -) cat > "$REPORT_FILE" << EOF { "generated_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)", + "platform": "$HOST_PLATFORM", "summary": { "parity_pass": $PARITY_PASS, "parity_fail": $PARITY_FAIL, @@ -1039,8 +1125,9 @@ if [[ -n "${PERRY_TEST_SUMMARY_OUT:-}" ]]; then EOF fi -# Exit with error if parity is below threshold (80%) -if (( $(echo "$PARITY_PCT < 80" | bc -l) )); then +# Exit with error if parity is below threshold (80%). Integer arithmetic keeps +# the Git Bash path independent of `bc`, which Git for Windows does not ship. +if (( TOTAL_RUN == 0 || PARITY_PASS * 100 < TOTAL_RUN * 80 )); then echo -e "${RED}Parity below 80% threshold${NC}" exit 1 fi diff --git a/scripts/gap_snapshot.py b/scripts/gap_snapshot.py index 9823c5bc97..5f5b9560ff 100755 --- a/scripts/gap_snapshot.py +++ b/scripts/gap_snapshot.py @@ -160,7 +160,9 @@ def merge( return {test_id: entry for test_id, entry in merged.items() if exists(test_id)} -def report_diffs(diffs: list[tuple[str, str, str, str]]) -> None: +def report_diffs( + diffs: list[tuple[str, str, str, str]], snapshot_path: Path = DEFAULT_SNAPSHOT +) -> None: order = {"regression": 0, "changed": 1, "improvement": 2} headers = { "regression": "REGRESSIONS — these were expected to pass:", @@ -176,7 +178,7 @@ def report_diffs(diffs: list[tuple[str, str, str, str]]) -> None: print( "\nFix the regressions, then accept the rest with:\n" " UPDATE_SNAPSHOT=1 ./scripts/run_gap_tests.sh\n" - "and commit test-parity/gap_snapshot.json.", + f"and commit {snapshot_path}.", file=sys.stderr, ) @@ -257,10 +259,10 @@ def main() -> int: diffs = diff(report, tests) if diffs: - report_diffs(diffs) + report_diffs(diffs, args.snapshot) return 1 print( - f"Gap snapshot OK — {len(report)} tests match test-parity/gap_snapshot.json " + f"Gap snapshot OK — {len(report)} tests match {args.snapshot} " f"({len(tests)} known non-passing)." ) return 0 diff --git a/scripts/parity_known_failures.py b/scripts/parity_known_failures.py new file mode 100644 index 0000000000..3ef7c37e58 --- /dev/null +++ b/scripts/parity_known_failures.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Check a parity report against the platform-aware known-failure allowlist.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_REPORT = ROOT / "test-parity" / "reports" / "latest.json" +DEFAULT_KNOWN = ROOT / "test-parity" / "known_failures.json" +PLATFORMS = frozenset({"linux", "macos", "windows", "other"}) + + +def normalize_platform(value: str) -> str: + folded = value.strip().lower() + aliases = { + "darwin": "macos", + "macos": "macos", + "linux": "linux", + "win32": "windows", + "windows": "windows", + "cygwin": "windows", + "msys": "windows", + "mingw": "windows", + } + return aliases.get(folded, "other") + + +def load_json(path: Path) -> dict: + with path.open(encoding="utf-8") as handle: + data = json.load(handle) + if not isinstance(data, dict): + raise ValueError(f"{path} must contain a JSON object") + return data + + +def report_failures(report: dict) -> set[str]: + failures = report.get("failures") + if not isinstance(failures, dict): + raise ValueError("parity report must contain a failures object") + result: set[str] = set() + for category in ("parity", "compile"): + values = failures.get(category) or [] + if not isinstance(values, list) or not all(isinstance(item, str) for item in values): + raise ValueError(f"failures.{category} must be a string array") + result.update(item for item in values if item) + return result + + +def known_for_platform(known: dict, platform: str) -> tuple[set[str], list[str]]: + selected: set[str] = set() + problems: list[str] = [] + for test_id, record in known.items(): + if test_id == "_schema": + continue + if not isinstance(record, dict): + problems.append(f"{test_id}: entry must be an object") + continue + if not isinstance(record.get("category"), str) or not record["category"]: + problems.append(f"{test_id}: category must be a non-empty string") + if not isinstance(record.get("reason"), str) or not record["reason"]: + problems.append(f"{test_id}: reason must be a non-empty string") + + platforms = record.get("platforms") + if platforms is None: + selected.add(test_id) + continue + if ( + not isinstance(platforms, list) + or not platforms + or not all(isinstance(item, str) for item in platforms) + ): + problems.append(f"{test_id}: platforms must be a non-empty string array") + continue + normalized = [normalize_platform(item) for item in platforms] + unknown = sorted({item for item in platforms if item.strip().lower() not in PLATFORMS}) + if unknown: + problems.append(f"{test_id}: unknown platforms: {', '.join(unknown)}") + continue + if len(normalized) != len(set(normalized)): + problems.append(f"{test_id}: platforms must not contain duplicates") + continue + if platform in normalized: + selected.add(test_id) + return selected, problems + + +def check(report: dict, known: dict, platform_override: str | None = None) -> tuple[str, list[str], list[str]]: + platform_value = platform_override or report.get("platform") or sys.platform + if not isinstance(platform_value, str): + raise ValueError("report platform must be a string") + platform = normalize_platform(platform_value) + failures = report_failures(report) + allowed, schema_problems = known_for_platform(known, platform) + return platform, sorted(failures - allowed), schema_problems + + +def self_test() -> int: + report = { + "platform": "windows", + "failures": {"parity": ["all_hosts", "windows_only", "linux_only"], "compile": [""]}, + } + known = { + "_schema": {}, + "all_hosts": {"category": "bug-open", "reason": "all"}, + "windows_only": { + "category": "bug-open", + "reason": "win", + "platforms": ["windows"], + }, + "linux_only": { + "category": "bug-open", + "reason": "linux", + "platforms": ["linux"], + }, + } + platform, new, problems = check(report, known) + assert platform == "windows" + assert new == ["linux_only"] + assert problems == [] + + malformed = { + "bad": {"category": "", "reason": "", "platforms": ["plan9"]}, + } + _, _, problems = check(report, malformed) + assert len(problems) == 3, problems + print("parity_known_failures self-test OK") + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--report", type=Path, default=DEFAULT_REPORT) + parser.add_argument("--known", type=Path, default=DEFAULT_KNOWN) + parser.add_argument("--platform") + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args(argv) + + if args.self_test: + return self_test() + + try: + report = load_json(args.report) + known = load_json(args.known) if args.known.exists() else {} + platform, new_failures, schema_problems = check(report, known, args.platform) + except (OSError, ValueError, json.JSONDecodeError) as error: + print(f"known-failure check error: {error}", file=sys.stderr) + return 2 + + if schema_problems: + print("Malformed known_failures.json entries:", file=sys.stderr) + for problem in schema_problems: + print(f" - {problem}", file=sys.stderr) + return 2 + if new_failures: + print( + f"NEW FAILURES on {platform} (not allowed for this platform):", + file=sys.stderr, + ) + for test_id in new_failures: + print(f" - {test_id}", file=sys.stderr) + return 1 + + total = len(report_failures(report)) + print(f"All {total} failures are known/triaged for {platform}.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/parity_matrix_trend.py b/scripts/parity_matrix_trend.py index ddd054e8d1..770d9f56ab 100755 --- a/scripts/parity_matrix_trend.py +++ b/scripts/parity_matrix_trend.py @@ -82,11 +82,40 @@ def diff_line_count(node_lines: list[str] | None, perry_lines: list[str] | None) return count -def known_failures(path: Path) -> set[str]: +def normalize_platform(value: str) -> str: + folded = value.strip().lower() + if folded in {"win32", "windows", "cygwin", "msys", "mingw"}: + return "windows" + if folded in {"darwin", "macos"}: + return "macos" + if folded == "linux": + return "linux" + return "other" + + +def known_failures(path: Path, platform: str | None = None) -> set[str]: if not path.exists(): return set() data = load_json(path) - return {key for key in data if key != "_schema"} + selected_platform = normalize_platform(platform or sys.platform) + selected = set() + for key, record in data.items(): + if key == "_schema": + continue + if not isinstance(record, dict): + continue + platforms = record.get("platforms") + if platforms is None or ( + isinstance(platforms, list) + and selected_platform + in { + normalize_platform(item) + for item in platforms + if isinstance(item, str) + } + ): + selected.add(key) + return selected def report_results(report: dict) -> list[dict[str, str]]: @@ -245,11 +274,17 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR) parser.add_argument("--output-json", type=Path, default=DEFAULT_JSON) parser.add_argument("--output-md", type=Path, default=DEFAULT_MARKDOWN) + parser.add_argument( + "--platform", + help="Override the report platform (linux, macos, windows, or other).", + ) parser.add_argument("--check", action="store_true") args = parser.parse_args(argv) report = load_json(args.report) - known = known_failures(args.known) + report_platform = report.get("platform") + platform = args.platform or (report_platform if isinstance(report_platform, str) else None) + known = known_failures(args.known, platform) baseline = load_baseline(args.baseline) records = build_records(report, known, args.output_dir) problems = check_records(records, baseline) if args.check else [] diff --git a/scripts/run_gap_tests.sh b/scripts/run_gap_tests.sh index 1c1d11b144..cd9b37513a 100755 --- a/scripts/run_gap_tests.sh +++ b/scripts/run_gap_tests.sh @@ -26,7 +26,7 @@ # Requirements: # - a Rust toolchain (the wrapped run_parity_tests.sh builds target/release/perry) # - node with --experimental-strip-types, at the .node-version pin -# - jq, python3 +# - jq, Python 3 (`python3` or `python`) # # Usage: scripts/run_gap_tests.sh [--shard N/M] # UPDATE_SNAPSHOT=1 scripts/run_gap_tests.sh # accept current state @@ -36,10 +36,52 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" cd "$ROOT" +if [[ -n "${PERRY_HOST_PLATFORM:-}" ]]; then + host_platform="$PERRY_HOST_PLATFORM" +else + case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) host_platform=windows ;; + Darwin) host_platform=macos ;; + Linux) host_platform=linux ;; + *) host_platform=other ;; + esac +fi +case "$host_platform" in + windows|macos|linux|other) ;; + *) + echo "Invalid PERRY_HOST_PLATFORM '$host_platform'" >&2 + exit 2 + ;; +esac +if command -v python3 &>/dev/null; then + PYTHON_CMD=python3 +elif command -v python &>/dev/null; then + PYTHON_CMD=python +else + echo "Python 3 is required by the gap snapshot checker" >&2 + exit 2 +fi +TEMP_ROOT="${TMPDIR:-${TEMP:-${TMP:-/tmp}}}" +if [[ "$host_platform" == "windows" ]] && command -v cygpath &>/dev/null; then + TEMP_ROOT="$(cygpath -u "$TEMP_ROOT")" +fi +mkdir -p "$TEMP_ROOT" + +# The committed legacy snapshot is the Linux baseline used by required CI. +# Other hosts get their own file so accepting a Windows-only divergence never +# changes Linux's ratchet (and vice versa). The first run on a new platform is +# expected to request UPDATE_SNAPSHOT=1 to establish that platform's baseline. +if [[ "$host_platform" == "linux" ]]; then + default_snapshot="test-parity/gap_snapshot.json" +else + default_snapshot="test-parity/gap_snapshot.${host_platform}.json" +fi +GAP_SNAPSHOT="${GAP_SNAPSHOT:-$default_snapshot}" + # Run-scoped temp dir — fixed /tmp names would let concurrent runs (a second # PR, local + CI on the same box, or the future node-suite-guard alongside) # clobber each other's failure lists and produce a false gate result. -WORK="$(mktemp -d "${TMPDIR:-/tmp}/perry-gap.XXXXXX")" +WORK="$(mktemp -d "$TEMP_ROOT/perry-gap.XXXXXX")" trap 'rm -rf "$WORK"' EXIT echo "==> Running gap suite (test-files/test_gap_*.ts) via run_parity_tests.sh --filter test_gap_" @@ -85,6 +127,6 @@ fi # Snapshot ratchet. UPDATE_SNAPSHOT=1 accepts the current state instead of # gating on it; commit the resulting test-parity/gap_snapshot.json diff. if [[ "${UPDATE_SNAPSHOT:-0}" == "1" ]]; then - exec python3 scripts/gap_snapshot.py update --report "$REPORT" + exec "$PYTHON_CMD" scripts/gap_snapshot.py update --report "$REPORT" --snapshot "$GAP_SNAPSHOT" fi -exec python3 scripts/gap_snapshot.py check --report "$REPORT" +exec "$PYTHON_CMD" scripts/gap_snapshot.py check --report "$REPORT" --snapshot "$GAP_SNAPSHOT" diff --git a/test-parity/README.md b/test-parity/README.md index a431686b94..7d66397212 100644 --- a/test-parity/README.md +++ b/test-parity/README.md @@ -24,6 +24,14 @@ UPDATE_SNAPSHOT=1 ./scripts/run_gap_tests.sh then commit the diff. New entries land as `category: untriaged` with a null issue — fill those in, that is the triage step the gate is asking for. +The required CI baseline remains `gap_snapshot.json` on Linux. The gap wrapper +selects `gap_snapshot.windows.json`, `gap_snapshot.macos.json`, or +`gap_snapshot.other.json` on other hosts, so platform-only results do not +pollute Linux's ratchet. Bootstrap a missing platform file on that host with +the same `UPDATE_SNAPSHOT=1` command, triage its generated entries, and commit +it. `GAP_SNAPSHOT=/path/to/file` can override the selection for an explicit +comparison. + The snapshot records `node_fail` and `skipped` explicitly instead of dropping them. A test the oracle stopped covering is a visible diff, not a silent hole: CI sat on Node 22 while the suite grew Node 24/26 features and hid 14 tests @@ -40,6 +48,26 @@ full-suite baseline from a tag run and is a follow-up; until then its gap-suite entries are redundant with `gap_snapshot.json` and are kept only so the tag-gated job keeps passing. +Entries apply on every host by default. A failure that is specific to one or +more operating systems can add a `platforms` array containing `linux`, `macos`, +`windows`, or `other`; `scripts/parity_known_failures.py` compares it with the +`platform` recorded by `run_parity_tests.sh`. For example: + +```json +"test_windows_only_gap": { + "issue": "1234", + "added": "2026-07-30", + "category": "bug-open", + "reason": "Fails only with the Windows runtime.", + "platforms": ["windows"] +} +``` + +The parity runner also supports Git Bash on Windows. It uses the native +`TEMP`/`TMP` directory, selects `.exe`/`.lib` artifacts, and does not require +Unix-only `bc` or process-limit support. The remaining documented prerequisites +(`cargo`, Node.js, Python, and the Windows linker toolchain) must be on `PATH`. + ## Category definitions Used by both files. diff --git a/test-parity/known_failures.json b/test-parity/known_failures.json index e9736c832b..8609ce6486 100644 --- a/test-parity/known_failures.json +++ b/test-parity/known_failures.json @@ -5,7 +5,8 @@ "issue": "Open GitHub issue number tracking this failure (e.g. \"793\"), or null when the failure is environmental / pending triage. Closed issues must be re-evaluated — flag the entry as `category: bug-stale` until a new tracking issue is filed.", "added": "ISO date (YYYY-MM-DD) when the test was first skip-listed. Use 2026-05-15 for entries inherited from the pre-audit format where the historical date is unknown.", "category": "ci-env | module-inventory | bug-open | bug-stale | gap-categorical | gap-bisect — see test-parity/README.md for definitions.", - "reason": "Free-text explanation. Keep the most-actionable signal first — what changes the verdict (a fixture, a Perry fix, a Node spec change)." + "reason": "Free-text explanation. Keep the most-actionable signal first — what changes the verdict (a fixture, a Perry fix, a Node spec change).", + "platforms": "Optional non-empty array of linux, macos, windows, or other. Omit when the failure is accepted on every platform." }, "scope": "Consumed by the TAG-GATED `parity` job (full test-files/*.ts suite) only. The per-PR gap-suite gate moved to test-parity/gap_snapshot.json, which is generated and checked in BOTH directions (see test-parity/README.md); the test_gap_* entries below are redundant with it and kept only so the tag-gated job keeps passing. Migrating this file to a generated snapshot needs a full-suite baseline from a tag run — follow-up to #797." }, diff --git a/tests/test_parity_build_reuse.sh b/tests/test_parity_build_reuse.sh index 5d9a28c5d1..7d079b084f 100755 --- a/tests/test_parity_build_reuse.sh +++ b/tests/test_parity_build_reuse.sh @@ -6,6 +6,9 @@ WORK="$(mktemp -d)" trap 'rm -rf "$WORK"' EXIT cp "$ROOT/run_parity_tests.sh" "$WORK/run_parity_tests.sh" +mkdir -p "$WORK/scripts" +cp "$ROOT/scripts/run_gap_tests.sh" "$WORK/scripts/run_gap_tests.sh" +cp "$ROOT/scripts/gap_snapshot.py" "$WORK/scripts/gap_snapshot.py" mkdir -p "$WORK/bin" "$WORK/test-files" "$WORK/test-parity/node-suite/reuse" \ "$WORK/test-parity/output/node" "$WORK/test-parity/output/perry" "$WORK/test-parity/reports" cat > "$WORK/bin/cargo" <<'EOF' @@ -38,6 +41,7 @@ cat > "$WORK/bin/ps" <<'EOF' echo 1 EOF touch "$WORK/test-parity/node-suite/reuse/basic.ts" +touch "$WORK/test-files/test_gap_reuse.ts" chmod +x "$WORK/bin/cargo" "$WORK/bin/node" "$WORK/bin/ps" "$WORK/perry" export PATH="$WORK/bin:$PATH" CARGO_LOG="$WORK/cargo.log" PERRY_LOG="$WORK/perry.log" @@ -75,4 +79,43 @@ grep -F "Using prebuilt runtime archives: $WORK" "$WORK/output" >/dev/null grep -F "$WORK/perry|$WORK|1" "$PERRY_LOG" >/dev/null [[ ! -s "$CARGO_LOG" ]] +# Exercise the Git Bash branch without requiring a Windows host. The mock +# compiler receives and creates an `.exe`, `.lib` archives satisfy the +# prebuilt-artifact check, and TEMP supplies the run-scoped scratch root when +# TMPDIR is absent. +cp "$WORK/perry" "$WORK/perry.exe" +chmod +x "$WORK/perry.exe" +touch "$WORK/perry_runtime.lib" "$WORK/perry_stdlib.lib" +mkdir -p "$WORK/windows-temp" +: > "$PERRY_LOG" +set +e +env -u TMPDIR -u TMP \ + PERRY_HOST_PLATFORM=windows \ + TEMP="$WORK/windows-temp" \ + PERRY_SKIP_BUILD=1 \ + PERRY_BIN="$WORK/perry.exe" \ + PERRY_RUNTIME_DIR="$WORK" \ + "$WORK/run_parity_tests.sh" --suite node-suite --module reuse >"$WORK/windows-output" 2>&1 +status=$? +set -e +[[ "$status" -eq 0 ]] +grep -F "Using prebuilt compiler: $WORK/perry.exe" "$WORK/windows-output" >/dev/null +grep -F "$WORK/perry.exe|$WORK|1" "$PERRY_LOG" >/dev/null +grep -F '"platform": "windows"' "$WORK/test-parity/reports/latest.json" >/dev/null +if find "$WORK/windows-temp" -maxdepth 1 -name 'perry-parity.*' | grep -q .; then + echo "Windows scratch directory was not cleaned" >&2 + exit 1 +fi + +# The gap wrapper must select an independent Windows snapshot instead of +# comparing the Windows result with the committed Linux baseline. +env -u TMPDIR -u TMP \ + PERRY_HOST_PLATFORM=windows \ + TEMP="$WORK/windows-temp" \ + PERRY_SKIP_BUILD=1 \ + PERRY_BIN="$WORK/perry.exe" \ + PERRY_RUNTIME_DIR="$WORK" \ + "$WORK/scripts/run_gap_tests.sh" --filter test_gap_reuse >"$WORK/gap-output" 2>&1 +grep -F "test-parity/gap_snapshot.windows.json" "$WORK/gap-output" >/dev/null + echo "PASS" From 13222491924577c4038c439ceaaf9e6358a61d52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 30 Jul 2026 19:32:13 +0200 Subject: [PATCH 2/2] docs: add changelog for PR 7082 --- changelog.d/7082-windows-parity-harness.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/7082-windows-parity-harness.md diff --git a/changelog.d/7082-windows-parity-harness.md b/changelog.d/7082-windows-parity-harness.md new file mode 100644 index 0000000000..b2a6dafc16 --- /dev/null +++ b/changelog.d/7082-windows-parity-harness.md @@ -0,0 +1 @@ +fix(parity): make the parity and gap harnesses runnable and platform-aware on Windows