From 1cddbcaf0fbff724f28a784cf74e922a718403f5 Mon Sep 17 00:00:00 2001 From: comphead Date: Sat, 1 Aug 2026 10:31:57 -0700 Subject: [PATCH 1/2] chore: run `Lint Java` once --- .github/workflows/pr_build_linux.yml | 45 +++++++++++----------------- 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index ff69a2cc68..1ae958f89c 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -48,10 +48,13 @@ jobs: # Fast syntactic-only scalafix check. Parses sources without compiling, so it # surfaces version-independent style issues (e.g. RedundantSyntax) in seconds, - # long before the lint-java matrix finishes its ~3.5 min build. It also scans - # the spark-4.1 / spark-4.2 sources that lint-java skips (those profiles can't - # run -Psemanticdb yet), so it is the only gate covering them. The full rule - # set, including the semantic rules, still runs in lint-java. + # long before lint-java finishes its ~3.5 min build. It is also the only gate + # covering the Spark profiles lint-java skips: 3.4, 3.5, 4.1, and 4.2. Spark + # 4.1 / 4.2 skip lint-java because -Psemanticdb can't run against their Scala + # versions (see the note on lint-java below); the 3.x profiles are skipped to + # keep the lint matrix to a single row pinned to the latest supported Spark. + # The full rule set, including the semantic rules, runs in lint-java against + # that single profile. scalafix-syntactic: name: Lint Scala (syntactic) runs-on: ubuntu-24.04 @@ -72,34 +75,20 @@ jobs: --exclude '**/target/**' \ spark + # Semantic scalafix + prettier + working-tree check, pinned to a single + # Spark 4.0 / JDK 21 row. 4.0 is the latest Spark that can currently run + # -Psemanticdb: semanticdb-scalac for the Scala 2.13.17 / 2.13.18 patch + # versions used by spark-4.1 / spark-4.2 has not been published yet. JDK 21 + # is the newest JDK in the supported test matrix. Cross-version syntactic + # coverage is provided by scalafix-syntactic above. lint-java: needs: lint - name: Lint Java (${{ matrix.profile.name }}) + name: Lint Java (Spark 4.0, JDK 21) runs-on: ubuntu-24.04 container: image: amd64/rust env: - JAVA_TOOL_OPTIONS: ${{ (matrix.profile.java_version == '17' || matrix.profile.java_version == '21') && '--add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED' || '' }} - strategy: - matrix: - profile: - - name: "Spark 3.4, JDK 11, Scala 2.12" - java_version: "11" - maven_opts: "-Pspark-3.4 -Pscala-2.12" - - name: "Spark 3.5, JDK 17, Scala 2.12" - java_version: "17" - maven_opts: "-Pspark-3.5 -Pscala-2.12" - - name: "Spark 4.0, JDK 17" - java_version: "17" - maven_opts: "-Pspark-4.0" - - name: "Spark 4.0, JDK 21" - java_version: "21" - maven_opts: "-Pspark-4.0" - # Spark 4.1 and 4.2 are intentionally absent: the lint job invokes -Psemanticdb, - # but semanticdb-scalac for those Scala patch versions (2.13.17 / 2.13.18) is not - # yet published, so we cannot currently run scalafix against the spark-4.1 or - # spark-4.2 profiles. - fail-fast: false + JAVA_TOOL_OPTIONS: "--add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED" steps: - uses: actions/checkout@v7 @@ -107,7 +96,7 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: ${{ env.RUST_VERSION }} - jdk-version: ${{ matrix.profile.java_version }} + jdk-version: "21" - name: Cache Maven dependencies uses: actions/cache@v6 @@ -121,7 +110,7 @@ jobs: - name: Run scalafix check run: | - ./mvnw -B package -DskipTests scalafix:scalafix -Dscalafix.mode=CHECK -Psemanticdb ${{ matrix.profile.maven_opts }} + ./mvnw -B package -DskipTests scalafix:scalafix -Dscalafix.mode=CHECK -Psemanticdb -Pspark-4.0 - name: Setup Node.js uses: actions/setup-node@v7 From fabe70acb592f9624e21e9d51eedc4bc215a6299 Mon Sep 17 00:00:00 2001 From: comphead Date: Sat, 1 Aug 2026 22:16:02 -0700 Subject: [PATCH 2/2] chore: optimize run `Lint Java` --- .github/workflows/pr_build_linux.yml | 41 ++++++++++++++++++---------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index 1ae958f89c..212b022200 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -49,12 +49,8 @@ jobs: # Fast syntactic-only scalafix check. Parses sources without compiling, so it # surfaces version-independent style issues (e.g. RedundantSyntax) in seconds, # long before lint-java finishes its ~3.5 min build. It is also the only gate - # covering the Spark profiles lint-java skips: 3.4, 3.5, 4.1, and 4.2. Spark - # 4.1 / 4.2 skip lint-java because -Psemanticdb can't run against their Scala - # versions (see the note on lint-java below); the 3.x profiles are skipped to - # keep the lint matrix to a single row pinned to the latest supported Spark. - # The full rule set, including the semantic rules, runs in lint-java against - # that single profile. + # covering the Spark profiles lint-java skips (3.4, 4.1, 4.2); see the + # lint-java header below for the rationale on those skips. scalafix-syntactic: name: Lint Scala (syntactic) runs-on: ubuntu-24.04 @@ -75,20 +71,32 @@ jobs: --exclude '**/target/**' \ spark - # Semantic scalafix + prettier + working-tree check, pinned to a single - # Spark 4.0 / JDK 21 row. 4.0 is the latest Spark that can currently run + # Semantic scalafix + prettier + working-tree check. The matrix is kept small: + # one row on the latest supported Spark/JDK (4.0 / JDK 21), plus one row on + # Spark 3.5 so the version-specific spark-3.x / spark-3.5 shim source dirs + # get semantic coverage too. 4.0 is the latest Spark that can currently run # -Psemanticdb: semanticdb-scalac for the Scala 2.13.17 / 2.13.18 patch # versions used by spark-4.1 / spark-4.2 has not been published yet. JDK 21 # is the newest JDK in the supported test matrix. Cross-version syntactic # coverage is provided by scalafix-syntactic above. lint-java: needs: lint - name: Lint Java (Spark 4.0, JDK 21) + name: Lint Java (${{ matrix.profile.name }}) runs-on: ubuntu-24.04 container: image: amd64/rust env: JAVA_TOOL_OPTIONS: "--add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED" + strategy: + matrix: + profile: + - name: "Spark 3.5, JDK 17, Scala 2.12" + java_version: "17" + maven_opts: "-Pspark-3.5 -Pscala-2.12" + - name: "Spark 4.0, JDK 21" + java_version: "21" + maven_opts: "-Pspark-4.0" + fail-fast: false steps: - uses: actions/checkout@v7 @@ -96,7 +104,7 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: ${{ env.RUST_VERSION }} - jdk-version: "21" + jdk-version: ${{ matrix.profile.java_version }} - name: Cache Maven dependencies uses: actions/cache@v6 @@ -104,39 +112,44 @@ jobs: path: | ~/.m2/repository /root/.m2/repository - key: ${{ runner.os }}-java-maven-${{ hashFiles('**/pom.xml') }}-lint + key: ${{ runner.os }}-java-maven-${{ hashFiles('**/pom.xml') }}-lint-${{ matrix.profile.java_version }} restore-keys: | + ${{ runner.os }}-java-maven-${{ hashFiles('**/pom.xml') }}-lint- ${{ runner.os }}-java-maven- - name: Run scalafix check run: | - ./mvnw -B package -DskipTests scalafix:scalafix -Dscalafix.mode=CHECK -Psemanticdb -Pspark-4.0 + ./mvnw -B package -DskipTests scalafix:scalafix -Dscalafix.mode=CHECK -Psemanticdb ${{ matrix.profile.maven_opts }} - name: Setup Node.js + if: matrix.profile.java_version == '21' uses: actions/setup-node@v7 with: node-version: '24' - name: Install prettier + if: matrix.profile.java_version == '21' run: | npm install -g prettier - name: Run prettier + if: matrix.profile.java_version == '21' run: | npx prettier "**/*.md" --write - name: Mark workspace as safe for git + if: matrix.profile.java_version == '21' run: | git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Check for any local git changes (such as generated docs) + if: matrix.profile.java_version == '21' run: | ./dev/ci/check-working-tree-clean.sh # Compile-only verification for Spark 4.1. Tests are intentionally skipped: the spark-4.1 # profile is currently a build target only, and several runtime/test failures are tracked - # in follow-up PRs. Excluded from lint-java because semanticdb-scalac_2.13.17 is not yet - # published and the lint job activates -Psemanticdb. + # in follow-up PRs. See lint-java above for why Spark 4.1 is not linted. build-spark-4-1: needs: lint name: Build Spark 4.1, JDK 17