diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml index e81bfb7256610..99a3a4c872858 100644 --- a/.github/workflows/additional-prod-image-tests.yml +++ b/.github/workflows/additional-prod-image-tests.yml @@ -56,6 +56,10 @@ on: # yamllint disable-line rule:truthy description: "Whether to run Java SDK e2e tests (true/false)" required: true type: string + java-sdk-version: + description: "JDK version used to build the Java SDK bundles natively in the e2e tests" + required: true + type: string run-go-sdk-e2e-tests: description: "Whether to run Go SDK e2e tests (true/false)" required: true @@ -320,6 +324,7 @@ jobs: default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} e2e_test_mode: "java_sdk" + java-sdk-version: "${{ inputs.java-sdk-version }}" if: inputs.canary-run == 'true' || inputs.run-java-sdk-e2e-tests == 'true' test-e2e-integration-tests-go-sdk: name: "Go SDK e2e tests with PROD image" diff --git a/.github/workflows/airflow-e2e-tests.yml b/.github/workflows/airflow-e2e-tests.yml index 4fb1fcf8d8438..a13a5cccf2d92 100644 --- a/.github/workflows/airflow-e2e-tests.yml +++ b/.github/workflows/airflow-e2e-tests.yml @@ -52,6 +52,10 @@ on: # yamllint disable-line rule:truthy description: "Test mode - basic, remote_log, remote_log_elasticsearch, remote_log_opensearch, xcom_object_storage, event_driven, java_sdk, or go_sdk" # yamllint disable-line rule:line-length type: string default: "basic" + java-sdk-version: + description: "JDK version used to build the Java SDK bundles natively (tracks the JAVA_SDK_VERSION breeze constant)" # yamllint disable-line rule:line-length + type: string + default: "17" workflow_call: inputs: @@ -83,6 +87,10 @@ on: # yamllint disable-line rule:truthy description: "Test mode - basic, remote_log, remote_log_elasticsearch, remote_log_opensearch, xcom_object_storage, event_driven, java_sdk, or go_sdk" # yamllint disable-line rule:line-length type: string default: "basic" + java-sdk-version: + description: "JDK version used to build the Java SDK bundles natively (tracks the JAVA_SDK_VERSION breeze constant)" # yamllint disable-line rule:line-length + type: string + default: "17" jobs: test-e2e-integration-tests: @@ -113,30 +121,42 @@ jobs: use-uv: ${{ inputs.use-uv }} make-mnt-writeable-and-cleanup: true id: breeze + # Only the java_sdk mode runs the Gradle builds; every other e2e mode skips the + # steps below. LANG_SDK_NATIVE_TOOLCHAIN=true (the same switch the lang-SDK k8s + # job uses) makes the e2e conftest build with the host toolchain provisioned + # here instead of ephemeral eclipse-temurin containers, skipping the + # toolchain-image pull; local runs keep the containerized build. Without the + # cache each run re-downloads the example bundles' full dependency tree (Spark + # alone is ~1GB) into a fresh ~/.gradle. The key carries a ``-vN-`` salt (bump + # to force-invalidate a poisoned cache) and runner.arch keys the cache per + # architecture: ~/.gradle/jdks holds arch-specific auto-provisioned JDKs (the + # example pins a Java 11 toolchain) and this job runs on amd64 and arm64. + - name: "Setup Java for the Java SDK build" + if: inputs.e2e_test_mode == 'java_sdk' + uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0 + with: + distribution: 'temurin' + java-version: ${{ inputs.java-sdk-version }} - name: "Restore Java SDK Gradle dependency cache" - # Only the java_sdk mode runs the Gradle builds; every other e2e mode skips - # this. Without it each java_sdk run re-downloads the example bundles' full - # dependency tree (Spark alone is ~1GB) into a fresh GRADLE_USER_HOME. - # runner.arch keys the cache per architecture: .gradle/jdks holds - # arch-specific auto-provisioned JDKs and this job runs on amd64 and arm64. if: inputs.e2e_test_mode == 'java_sdk' id: gradle-cache uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: | - java-sdk/.gradle/caches - java-sdk/.gradle/wrapper - java-sdk/.gradle/jdks - !java-sdk/.gradle/caches/*.lock - !java-sdk/.gradle/caches/journal-1 - key: java-sdk-gradle-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('java-sdk/**/*.gradle*', 'java-sdk/**/gradle.properties', 'java-sdk/gradle/wrapper/gradle-wrapper.properties', 'java-sdk/gradle/libs.versions.toml') }} # yamllint disable-line rule:line-length + ~/.gradle/caches + ~/.gradle/wrapper + ~/.gradle/jdks + !~/.gradle/caches/*.lock + !~/.gradle/caches/journal-1 + key: e2e-java-sdk-gradle-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('java-sdk/**/*.gradle*', 'java-sdk/**/gradle.properties', 'java-sdk/gradle/wrapper/gradle-wrapper.properties', 'java-sdk/gradle/libs.versions.toml') }} # yamllint disable-line rule:line-length restore-keys: | - java-sdk-gradle-${{ runner.os }}-${{ runner.arch }}- + e2e-java-sdk-gradle-v1-${{ runner.os }}-${{ runner.arch }}- - name: "Test e2e integration tests" run: breeze testing airflow-e2e-tests env: DOCKER_IMAGE: "${{ inputs.docker-image-tag }}" E2E_TEST_MODE: "${{ inputs.e2e_test_mode }}" + LANG_SDK_NATIVE_TOOLCHAIN: "${{ inputs.e2e_test_mode == 'java_sdk' && 'true' || 'false' }}" - name: "Save Java SDK Gradle dependency cache" # Saved even when the e2e tests fail: the Gradle warm-up is independent of # test outcome, and actions/cache's post step would drop it on every red run. @@ -144,11 +164,11 @@ jobs: uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: | - java-sdk/.gradle/caches - java-sdk/.gradle/wrapper - java-sdk/.gradle/jdks - !java-sdk/.gradle/caches/*.lock - !java-sdk/.gradle/caches/journal-1 + ~/.gradle/caches + ~/.gradle/wrapper + ~/.gradle/jdks + !~/.gradle/caches/*.lock + !~/.gradle/caches/journal-1 key: ${{ steps.gradle-cache.outputs.cache-primary-key }} - name: Zip logs run: | diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index ad0b70da87130..fd2892027ffe0 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -920,6 +920,7 @@ jobs: run-remote-logging-s3-e2e-tests: ${{ needs.build-info.outputs.run-remote-logging-s3-e2e-tests }} run-event-driven-e2e-tests: ${{ needs.build-info.outputs.run-event-driven-e2e-tests }} run-java-sdk-e2e-tests: ${{ needs.build-info.outputs.run-java-sdk-e2e-tests }} + java-sdk-version: ${{ needs.build-info.outputs.java-sdk-version }} run-go-sdk-e2e-tests: ${{ needs.build-info.outputs.run-go-sdk-e2e-tests }} run-openlineage-e2e-tests: ${{ needs.build-info.outputs.run-openlineage-e2e-tests }} run-openlineage-e2e-compat-tests: ${{ needs.build-info.outputs.run-openlineage-e2e-compat-tests }} diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index ee7ffdd916dae..ed2da6cf4346b 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -909,6 +909,7 @@ jobs: run-remote-logging-s3-e2e-tests: ${{ needs.build-info.outputs.run-remote-logging-s3-e2e-tests }} run-event-driven-e2e-tests: ${{ needs.build-info.outputs.run-event-driven-e2e-tests }} run-java-sdk-e2e-tests: ${{ needs.build-info.outputs.run-java-sdk-e2e-tests }} + java-sdk-version: ${{ needs.build-info.outputs.java-sdk-version }} run-go-sdk-e2e-tests: ${{ needs.build-info.outputs.run-go-sdk-e2e-tests }} run-openlineage-e2e-tests: ${{ needs.build-info.outputs.run-openlineage-e2e-tests }} run-openlineage-e2e-compat-tests: ${{ needs.build-info.outputs.run-openlineage-e2e-compat-tests }} diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py index 5b5883980c576..56b51992bb4cb 100644 --- a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py @@ -49,7 +49,9 @@ JAVA_SDK_EXAMPLE_DAGS_PATH, JAVA_SDK_EXAMPLE_LIBS_PATH, JAVA_SDK_MAVEN_CACHE_PATH, + JAVA_SDK_ROOT_PATH, KAFKA_DIR_PATH, + LANG_SDK_NATIVE_TOOLCHAIN, LOCALSTACK_PATH, LOGS_FOLDER, OPENLINEAGE_COMPOSE_PATH, @@ -281,8 +283,16 @@ def _setup_xcom_object_storage_integration(dot_env_file, tmp_dir): ] -def _run_java_sdk_gradle_container(workdir, *gradle_argv, capture_output=False): - """Run the Java SDK Gradle wrapper inside the pinned JDK container. +def _run_java_sdk_gradle(workdir, *gradle_argv, capture_output=False, native=False): + """Run the Java SDK Gradle wrapper natively or inside the pinned JDK container. + + In ``native`` mode (used in CI, where the host already has a cached JDK + + Gradle cache via ``actions/setup-java``) it invokes the host ``./gradlew`` + directly, skipping the toolchain-image pull and the container workarounds + below while reusing the runner's ~/.gradle cache. + + The containerized path stays the default for local runs so a dev host needs + no JDK installed: * --user keeps build outputs owned by the current user (not root). * --network=host shares one loopback across concurrent builds so Gradle's @@ -300,8 +310,18 @@ def _run_java_sdk_gradle_container(workdir, *gradle_argv, capture_output=False): * files/m2 is mounted directly as ~/.m2 so publishToMavenLocal writes there without nesting, and its contents are visible on the host. """ - return subprocess.run( - [ + if native: + cwd = workdir + argv = [ + str(JAVA_SDK_ROOT_PATH / "gradlew"), + "--no-daemon", + "--console=plain", + *gradle_argv, + ] + else: + JAVA_SDK_MAVEN_CACHE_PATH.mkdir(parents=True, exist_ok=True) + cwd = None + argv = [ "docker", "run", "--rm", @@ -317,22 +337,19 @@ def _run_java_sdk_gradle_container(workdir, *gradle_argv, capture_output=False): "-v", f"{AIRFLOW_ROOT_PATH}:/repo", "-w", - workdir, + f"/repo/{workdir.relative_to(AIRFLOW_ROOT_PATH)}", "eclipse-temurin:17-jdk", "/repo/java-sdk/gradlew", "--no-daemon", *gradle_argv, - ], - check=True, - capture_output=capture_output, - text=True, - ) + ] + return subprocess.run(argv, cwd=cwd, check=True, capture_output=capture_output, text=True) -def _build_example_bundle(workdir): +def _build_example_bundle(workdir, *, native=False): """Build one example bundle, capturing output so concurrent builds don't interleave.""" try: - completed = _run_java_sdk_gradle_container(workdir, "bundle", capture_output=True) + completed = _run_java_sdk_gradle(workdir, "bundle", capture_output=True, native=native) except subprocess.CalledProcessError as e: console.print(f"[red]Bundle build failed in {workdir}:") console.print(e.stdout, e.stderr, sep="\n", markup=False, soft_wrap=True) @@ -348,14 +365,15 @@ def _setup_java_sdk_integration(dot_env_file, tmp_dir): then builds a Java-capable Airflow worker image, copies the JARs into the temp directory, and writes the coordinator configuration. """ + native = LANG_SDK_NATIVE_TOOLCHAIN console.print("[yellow]Publishing Java SDK artifacts to local Maven repository...") - JAVA_SDK_MAVEN_CACHE_PATH.mkdir(parents=True, exist_ok=True) - _run_java_sdk_gradle_container("/repo/java-sdk", "publishToMavenLocal", "-PskipSigning=true") + _run_java_sdk_gradle(JAVA_SDK_ROOT_PATH, "publishToMavenLocal", "-PskipSigning=true", native=native) # The example and scala_spark_example are independent Gradle builds that both # consume the SDK artifact published above, so build them concurrently. Sharing - # a writable Gradle user home between concurrent builds is safe only because - # --network=host lets each build ping the other's lock-owner port (see the + # a writable Gradle user home between concurrent builds is safe because each + # build can ping the other's lock-owner port over one shared loopback - the + # host's own in native mode, --network=host in the container path (see the # helper's docstring); publishToMavenLocal has already unpacked the shared # wrapper distribution, so neither build races to fetch it. # @@ -365,15 +383,16 @@ def _setup_java_sdk_integration(dot_env_file, tmp_dir): # start each bundle from an empty directory. rmtree(JAVA_SDK_EXAMPLE_LIBS_PATH, ignore_errors=True) rmtree(SCALA_SPARK_EXAMPLE_LIBS_PATH, ignore_errors=True) - console.print( - "[yellow]Building Java SDK and Scala Spark example bundles concurrently (eclipse-temurin:17-jdk)..." - ) + toolchain = "host toolchain" if native else "eclipse-temurin:17-jdk" + console.print(f"[yellow]Building Java SDK and Scala Spark example bundles concurrently ({toolchain})...") example_bundle_workdirs = [ - "/repo/java-sdk/example", - "/repo/java-sdk/scala_spark_example", + JAVA_SDK_ROOT_PATH / "example", + JAVA_SDK_ROOT_PATH / "scala_spark_example", ] with ThreadPoolExecutor(max_workers=len(example_bundle_workdirs)) as pool: - bundle_builds = [pool.submit(_build_example_bundle, workdir) for workdir in example_bundle_workdirs] + bundle_builds = [ + pool.submit(_build_example_bundle, workdir, native=native) for workdir in example_bundle_workdirs + ] for build in bundle_builds: build.result() diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py b/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py index b0392d96a8a7c..ec1732ac75a8b 100644 --- a/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py @@ -54,6 +54,10 @@ # openlineage_tests/prepare_dags.py; the overlay carries the OpenLineage-specific env + dag_doc mount. OPENLINEAGE_COMPOSE_PATH = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" / "docker" / "openlineage.yml" +# CI sets this (the same switch the lang-SDK k8s job uses) to build the lang-SDK +# artifacts with the host toolchain instead of ephemeral toolchain containers. +LANG_SDK_NATIVE_TOOLCHAIN = os.environ.get("LANG_SDK_NATIVE_TOOLCHAIN", "").lower() == "true" + # Java SDK E2E test paths JAVA_SDK_ROOT_PATH = AIRFLOW_ROOT_PATH / "java-sdk" JAVA_SDK_EXAMPLE_DAGS_PATH = JAVA_SDK_ROOT_PATH / "example" / "src" / "resources" / "dags"