Build Java SDK e2e example bundles natively in CI

.github/workflows/additional-prod-image-tests.yml

@@ -56,6 +56,10 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to run Java SDK e2e tests (true/false)"
         required: true
         type: string
+      java-sdk-version:
+        description: "JDK version used to build the Java SDK bundles natively in the e2e tests"
+        required: true
+        type: string
       run-go-sdk-e2e-tests:
         description: "Whether to run Go SDK e2e tests (true/false)"
         required: true
@@ -320,6 +324,7 @@ jobs:
       default-python-version: "${{ inputs.default-python-version }}"
       use-uv: ${{ inputs.use-uv }}
       e2e_test_mode: "java_sdk"
+      java-sdk-version: "${{ inputs.java-sdk-version }}"
     if: inputs.canary-run == 'true' || inputs.run-java-sdk-e2e-tests == 'true'
   test-e2e-integration-tests-go-sdk:
     name: "Go SDK e2e tests with PROD image"

.github/workflows/airflow-e2e-tests.yml

@@ -52,6 +52,10 @@ on:  # yamllint disable-line rule:truthy
         description: "Test mode - basic, remote_log, remote_log_elasticsearch, remote_log_opensearch, xcom_object_storage, event_driven, java_sdk, or go_sdk"  # yamllint disable-line rule:line-length
         type: string
         default: "basic"
+      java-sdk-version:
+        description: "JDK version used to build the Java SDK bundles natively (tracks the JAVA_SDK_VERSION breeze constant)"  # yamllint disable-line rule:line-length
+        type: string
+        default: "17"
 
   workflow_call:
     inputs:
@@ -83,6 +87,10 @@ on:  # yamllint disable-line rule:truthy
         description: "Test mode - basic, remote_log, remote_log_elasticsearch, remote_log_opensearch, xcom_object_storage, event_driven, java_sdk, or go_sdk"  # yamllint disable-line rule:line-length
         type: string
         default: "basic"
+      java-sdk-version:
+        description: "JDK version used to build the Java SDK bundles natively (tracks the JAVA_SDK_VERSION breeze constant)"  # yamllint disable-line rule:line-length
+        type: string
+        default: "17"
 
 jobs:
   test-e2e-integration-tests:
@@ -113,42 +121,54 @@ jobs:
           use-uv: ${{ inputs.use-uv }}
           make-mnt-writeable-and-cleanup: true
         id: breeze
+      # Only the java_sdk mode runs the Gradle builds; every other e2e mode skips the
+      # steps below. LANG_SDK_NATIVE_TOOLCHAIN=true (the same switch the lang-SDK k8s
+      # job uses) makes the e2e conftest build with the host toolchain provisioned
+      # here instead of ephemeral eclipse-temurin containers, skipping the
+      # toolchain-image pull; local runs keep the containerized build. Without the
+      # cache each run re-downloads the example bundles' full dependency tree (Spark
+      # alone is ~1GB) into a fresh ~/.gradle. The key carries a ``-vN-`` salt (bump
+      # to force-invalidate a poisoned cache) and runner.arch keys the cache per
+      # architecture: ~/.gradle/jdks holds arch-specific auto-provisioned JDKs (the
+      # example pins a Java 11 toolchain) and this job runs on amd64 and arm64.
+      - name: "Setup Java for the Java SDK build"
+        if: inputs.e2e_test_mode == 'java_sdk'
+        uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520  # v5.4.0
+        with:
+          distribution: 'temurin'
+          java-version: ${{ inputs.java-sdk-version }}
       - name: "Restore Java SDK Gradle dependency cache"
-        # Only the java_sdk mode runs the Gradle builds; every other e2e mode skips
-        # this. Without it each java_sdk run re-downloads the example bundles' full
-        # dependency tree (Spark alone is ~1GB) into a fresh GRADLE_USER_HOME.
-        # runner.arch keys the cache per architecture: .gradle/jdks holds
-        # arch-specific auto-provisioned JDKs and this job runs on amd64 and arm64.
         if: inputs.e2e_test_mode == 'java_sdk'
         id: gradle-cache
         uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
         with:
           path: |
-            java-sdk/.gradle/caches
-            java-sdk/.gradle/wrapper
-            java-sdk/.gradle/jdks
-            !java-sdk/.gradle/caches/*.lock
-            !java-sdk/.gradle/caches/journal-1
-          key: java-sdk-gradle-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('java-sdk/**/*.gradle*', 'java-sdk/**/gradle.properties', 'java-sdk/gradle/wrapper/gradle-wrapper.properties', 'java-sdk/gradle/libs.versions.toml') }}  # yamllint disable-line rule:line-length
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+            ~/.gradle/jdks
+            !~/.gradle/caches/*.lock
+            !~/.gradle/caches/journal-1
+          key: e2e-java-sdk-gradle-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('java-sdk/**/*.gradle*', 'java-sdk/**/gradle.properties', 'java-sdk/gradle/wrapper/gradle-wrapper.properties', 'java-sdk/gradle/libs.versions.toml') }}  # yamllint disable-line rule:line-length
           restore-keys: |
-            java-sdk-gradle-${{ runner.os }}-${{ runner.arch }}-
+            e2e-java-sdk-gradle-v1-${{ runner.os }}-${{ runner.arch }}-
       - name: "Test e2e integration tests"
         run: breeze testing airflow-e2e-tests
         env:
           DOCKER_IMAGE: "${{ inputs.docker-image-tag }}"
           E2E_TEST_MODE: "${{ inputs.e2e_test_mode }}"
+          LANG_SDK_NATIVE_TOOLCHAIN: "${{ inputs.e2e_test_mode == 'java_sdk' && 'true' || 'false' }}"
       - name: "Save Java SDK Gradle dependency cache"
         # Saved even when the e2e tests fail: the Gradle warm-up is independent of
         # test outcome, and actions/cache's post step would drop it on every red run.
         if: always() && inputs.e2e_test_mode == 'java_sdk' && steps.gradle-cache.outputs.cache-hit != 'true'
         uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
         with:
           path: |
-            java-sdk/.gradle/caches
-            java-sdk/.gradle/wrapper
-            java-sdk/.gradle/jdks
-            !java-sdk/.gradle/caches/*.lock
-            !java-sdk/.gradle/caches/journal-1
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+            ~/.gradle/jdks
+            !~/.gradle/caches/*.lock
+            !~/.gradle/caches/journal-1
           key: ${{ steps.gradle-cache.outputs.cache-primary-key }}
       - name: Zip logs
         run: |

.github/workflows/ci-amd.yml

@@ -920,6 +920,7 @@ jobs:
       run-remote-logging-s3-e2e-tests: ${{ needs.build-info.outputs.run-remote-logging-s3-e2e-tests }}
       run-event-driven-e2e-tests: ${{ needs.build-info.outputs.run-event-driven-e2e-tests }}
       run-java-sdk-e2e-tests: ${{ needs.build-info.outputs.run-java-sdk-e2e-tests }}
+      java-sdk-version: ${{ needs.build-info.outputs.java-sdk-version }}
       run-go-sdk-e2e-tests: ${{ needs.build-info.outputs.run-go-sdk-e2e-tests }}
       run-openlineage-e2e-tests: ${{ needs.build-info.outputs.run-openlineage-e2e-tests }}
       run-openlineage-e2e-compat-tests: ${{ needs.build-info.outputs.run-openlineage-e2e-compat-tests }}

.github/workflows/ci-arm.yml

@@ -909,6 +909,7 @@ jobs:
       run-remote-logging-s3-e2e-tests: ${{ needs.build-info.outputs.run-remote-logging-s3-e2e-tests }}
       run-event-driven-e2e-tests: ${{ needs.build-info.outputs.run-event-driven-e2e-tests }}
       run-java-sdk-e2e-tests: ${{ needs.build-info.outputs.run-java-sdk-e2e-tests }}
+      java-sdk-version: ${{ needs.build-info.outputs.java-sdk-version }}
       run-go-sdk-e2e-tests: ${{ needs.build-info.outputs.run-go-sdk-e2e-tests }}
       run-openlineage-e2e-tests: ${{ needs.build-info.outputs.run-openlineage-e2e-tests }}
       run-openlineage-e2e-compat-tests: ${{ needs.build-info.outputs.run-openlineage-e2e-compat-tests }}

airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py

@@ -49,7 +49,9 @@
     JAVA_SDK_EXAMPLE_DAGS_PATH,
     JAVA_SDK_EXAMPLE_LIBS_PATH,
     JAVA_SDK_MAVEN_CACHE_PATH,
+    JAVA_SDK_ROOT_PATH,
     KAFKA_DIR_PATH,
+    LANG_SDK_NATIVE_TOOLCHAIN,
     LOCALSTACK_PATH,
     LOGS_FOLDER,
     OPENLINEAGE_COMPOSE_PATH,
@@ -281,8 +283,16 @@ def _setup_xcom_object_storage_integration(dot_env_file, tmp_dir):
 ]
 
 
-def _run_java_sdk_gradle_container(workdir, *gradle_argv, capture_output=False):
-    """Run the Java SDK Gradle wrapper inside the pinned JDK container.
+def _run_java_sdk_gradle(workdir, *gradle_argv, capture_output=False, native=False):
+    """Run the Java SDK Gradle wrapper natively or inside the pinned JDK container.
+
+    In ``native`` mode (used in CI, where the host already has a cached JDK +
+    Gradle cache via ``actions/setup-java``) it invokes the host ``./gradlew``
+    directly, skipping the toolchain-image pull and the container workarounds
+    below while reusing the runner's ~/.gradle cache.
+
+    The containerized path stays the default for local runs so a dev host needs
+    no JDK installed:
 
     * --user keeps build outputs owned by the current user (not root).
     * --network=host shares one loopback across concurrent builds so Gradle's
@@ -300,8 +310,18 @@ def _run_java_sdk_gradle_container(workdir, *gradle_argv, capture_output=False):
     * files/m2 is mounted directly as ~/.m2 so publishToMavenLocal writes
       there without nesting, and its contents are visible on the host.
     """
-    return subprocess.run(
-        [
+    if native:
+        cwd = workdir
+        argv = [
+            str(JAVA_SDK_ROOT_PATH / "gradlew"),
+            "--no-daemon",
+            "--console=plain",
+            *gradle_argv,
+        ]
+    else:
+        JAVA_SDK_MAVEN_CACHE_PATH.mkdir(parents=True, exist_ok=True)
+        cwd = None
+        argv = [
             "docker",
             "run",
             "--rm",
@@ -317,22 +337,19 @@ def _run_java_sdk_gradle_container(workdir, *gradle_argv, capture_output=False):
             "-v",
             f"{AIRFLOW_ROOT_PATH}:/repo",
             "-w",
-            workdir,
+            f"/repo/{workdir.relative_to(AIRFLOW_ROOT_PATH)}",
             "eclipse-temurin:17-jdk",
             "/repo/java-sdk/gradlew",
             "--no-daemon",
             *gradle_argv,
-        ],
-        check=True,
-        capture_output=capture_output,
-        text=True,
-    )
+        ]
+    return subprocess.run(argv, cwd=cwd, check=True, capture_output=capture_output, text=True)
 
 
-def _build_example_bundle(workdir):
+def _build_example_bundle(workdir, *, native=False):
     """Build one example bundle, capturing output so concurrent builds don't interleave."""
     try:
-        completed = _run_java_sdk_gradle_container(workdir, "bundle", capture_output=True)
+        completed = _run_java_sdk_gradle(workdir, "bundle", capture_output=True, native=native)
     except subprocess.CalledProcessError as e:
         console.print(f"[red]Bundle build failed in {workdir}:")
         console.print(e.stdout, e.stderr, sep="\n", markup=False, soft_wrap=True)
@@ -348,14 +365,15 @@ def _setup_java_sdk_integration(dot_env_file, tmp_dir):
     then builds a Java-capable Airflow worker image, copies the JARs into the
     temp directory, and writes the coordinator configuration.
     """
+    native = LANG_SDK_NATIVE_TOOLCHAIN
     console.print("[yellow]Publishing Java SDK artifacts to local Maven repository...")
-    JAVA_SDK_MAVEN_CACHE_PATH.mkdir(parents=True, exist_ok=True)
-    _run_java_sdk_gradle_container("/repo/java-sdk", "publishToMavenLocal", "-PskipSigning=true")
+    _run_java_sdk_gradle(JAVA_SDK_ROOT_PATH, "publishToMavenLocal", "-PskipSigning=true", native=native)
 
     # The example and scala_spark_example are independent Gradle builds that both
     # consume the SDK artifact published above, so build them concurrently. Sharing
-    # a writable Gradle user home between concurrent builds is safe only because
-    # --network=host lets each build ping the other's lock-owner port (see the
+    # a writable Gradle user home between concurrent builds is safe because each
+    # build can ping the other's lock-owner port over one shared loopback - the
+    # host's own in native mode, --network=host in the container path (see the
     # helper's docstring); publishToMavenLocal has already unpacked the shared
     # wrapper distribution, so neither build races to fetch it.
     #
@@ -365,15 +383,16 @@ def _setup_java_sdk_integration(dot_env_file, tmp_dir):
     # start each bundle from an empty directory.
     rmtree(JAVA_SDK_EXAMPLE_LIBS_PATH, ignore_errors=True)
     rmtree(SCALA_SPARK_EXAMPLE_LIBS_PATH, ignore_errors=True)
-    console.print(
-        "[yellow]Building Java SDK and Scala Spark example bundles concurrently (eclipse-temurin:17-jdk)..."
-    )
+    toolchain = "host toolchain" if native else "eclipse-temurin:17-jdk"
+    console.print(f"[yellow]Building Java SDK and Scala Spark example bundles concurrently ({toolchain})...")
     example_bundle_workdirs = [
-        "/repo/java-sdk/example",
-        "/repo/java-sdk/scala_spark_example",
+        JAVA_SDK_ROOT_PATH / "example",
+        JAVA_SDK_ROOT_PATH / "scala_spark_example",
     ]
     with ThreadPoolExecutor(max_workers=len(example_bundle_workdirs)) as pool:
-        bundle_builds = [pool.submit(_build_example_bundle, workdir) for workdir in example_bundle_workdirs]
+        bundle_builds = [
+            pool.submit(_build_example_bundle, workdir, native=native) for workdir in example_bundle_workdirs
+        ]
         for build in bundle_builds:
             build.result()
 

airflow-e2e-tests/tests/airflow_e2e_tests/constants.py

@@ -54,6 +54,10 @@
 # openlineage_tests/prepare_dags.py; the overlay carries the OpenLineage-specific env + dag_doc mount.
 OPENLINEAGE_COMPOSE_PATH = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" / "docker" / "openlineage.yml"
 
+# CI sets this (the same switch the lang-SDK k8s job uses) to build the lang-SDK
+# artifacts with the host toolchain instead of ephemeral toolchain containers.
+LANG_SDK_NATIVE_TOOLCHAIN = os.environ.get("LANG_SDK_NATIVE_TOOLCHAIN", "").lower() == "true"
+
 # Java SDK E2E test paths
 JAVA_SDK_ROOT_PATH = AIRFLOW_ROOT_PATH / "java-sdk"
 JAVA_SDK_EXAMPLE_DAGS_PATH = JAVA_SDK_ROOT_PATH / "example" / "src" / "resources" / "dags"