Add integration tests for coordinator-mode TypeScript tasks

.github/workflows/additional-prod-image-tests.yml

@@ -72,6 +72,10 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to run the OpenLineage older-Airflow compat matrix (true/false)"
         required: true
         type: string
+      run-ts-sdk-e2e-tests:
+        description: "Whether to run TypeScript SDK e2e tests (true/false)"
+        required: true
+        type: string
       providers-compatibility-tests-matrix:
         description: "JSON matrix of released Airflow versions for the OpenLineage e2e compat tests."
         required: true
@@ -338,6 +342,18 @@ jobs:
       e2e_test_mode: "go_sdk"
     if: inputs.canary-run == 'true' || inputs.run-go-sdk-e2e-tests == 'true'
 
+  test-e2e-integration-tests-ts-sdk:
+    name: "TypeScript SDK e2e tests with PROD image"
+    uses: ./.github/workflows/airflow-e2e-tests.yml
+    with:
+      workflow-name: "TypeScript SDK e2e test"
+      runners: ${{ inputs.runners }}
+      platform: ${{ inputs.platform }}
+      default-python-version: "${{ inputs.default-python-version }}"
+      use-uv: ${{ inputs.use-uv }}
+      e2e_test_mode: "ts_sdk"
+    if: inputs.canary-run == 'true' || inputs.run-ts-sdk-e2e-tests == 'true'
+
   test-e2e-integration-tests-openlineage:
     name: "OpenLineage e2e tests with PROD image"
     uses: ./.github/workflows/airflow-e2e-tests.yml

.github/workflows/airflow-e2e-tests.yml

@@ -49,7 +49,7 @@ on:  # yamllint disable-line rule:truthy
         type: string
         required: true
       e2e_test_mode:
-        description: "Test mode - basic, remote_log, remote_log_elasticsearch, remote_log_opensearch, xcom_object_storage, event_driven, java_sdk, or go_sdk"  # yamllint disable-line rule:line-length
+        description: "Test mode - basic, remote_log, remote_log_elasticsearch, remote_log_opensearch, xcom_object_storage, event_driven, java_sdk, go_sdk, or ts_sdk"  # yamllint disable-line rule:line-length
         type: string
         default: "basic"
       java-sdk-version:
@@ -84,7 +84,7 @@ on:  # yamllint disable-line rule:truthy
         type: string
         default: ""
       e2e_test_mode:
-        description: "Test mode - basic, remote_log, remote_log_elasticsearch, remote_log_opensearch, xcom_object_storage, event_driven, java_sdk, or go_sdk"  # yamllint disable-line rule:line-length
+        description: "Test mode - basic, remote_log, remote_log_elasticsearch, remote_log_opensearch, xcom_object_storage, event_driven, java_sdk, go_sdk, or ts_sdk"  # yamllint disable-line rule:line-length
         type: string
         default: "basic"
       java-sdk-version:
@@ -121,8 +121,9 @@ jobs:
           use-uv: ${{ inputs.use-uv }}
           make-mnt-writeable-and-cleanup: true
         id: breeze
-      # The java_sdk mode runs the Gradle builds and the go_sdk mode runs the Go
-      # build; every other e2e mode skips the toolchain steps below.
+      # The java_sdk mode runs the Gradle builds, the go_sdk mode runs the Go
+      # build and the ts_sdk mode runs the pnpm build; every other e2e mode
+      # skips the toolchain steps below.
       # LANG_SDK_NATIVE_TOOLCHAIN=true (the same switch the lang-SDK k8s job uses)
       # makes the e2e conftest build with the host toolchain provisioned here
       # instead of an ephemeral toolchain container, skipping the image pull; local
@@ -172,12 +173,25 @@ jobs:
           key: e2e-go-sdk-go-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('go-sdk/go.mod', 'go-sdk/go.sum') }}  # yamllint disable-line rule:line-length
           restore-keys: |
             e2e-go-sdk-go-v1-${{ runner.os }}-${{ runner.arch }}-
+      - name: "Setup pnpm for the TypeScript SDK build"
+        if: inputs.e2e_test_mode == 'ts_sdk'
+        uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271  # v6.0.9
+        with:
+          package_json_file: ts-sdk/package.json
+          run_install: false
+      - name: "Setup Node for the TypeScript SDK build"
+        if: inputs.e2e_test_mode == 'ts_sdk'
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020  # v7.0.0
+        with:
+          node-version: 22
+          cache: 'pnpm'
+          cache-dependency-path: 'ts-sdk/pnpm-lock.yaml'
       - name: "Test e2e integration tests"
         run: breeze testing airflow-e2e-tests
         env:
           DOCKER_IMAGE: "${{ inputs.docker-image-tag }}"
           E2E_TEST_MODE: "${{ inputs.e2e_test_mode }}"
-          LANG_SDK_NATIVE_TOOLCHAIN: "${{ (inputs.e2e_test_mode == 'java_sdk' || inputs.e2e_test_mode == 'go_sdk') && 'true' || 'false' }}"  # yamllint disable-line rule:line-length
+          LANG_SDK_NATIVE_TOOLCHAIN: "${{ (inputs.e2e_test_mode == 'java_sdk' || inputs.e2e_test_mode == 'go_sdk' || inputs.e2e_test_mode == 'ts_sdk') && 'true' || 'false' }}"  # yamllint disable-line rule:line-length
       - name: "Save Java SDK Gradle dependency cache"
         # Saved even when the e2e tests fail: the Gradle warm-up is independent of
         # test outcome, and actions/cache's post step would drop it on every red run.

.github/workflows/ci-amd.yml

@@ -149,6 +149,7 @@ jobs:
       run-go-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-go-sdk-e2e-tests }}
       run-openlineage-e2e-tests: ${{ steps.selective-checks.outputs.run-openlineage-e2e-tests }}
       run-openlineage-e2e-compat-tests: ${{ steps.selective-checks.outputs.run-openlineage-e2e-compat-tests }}
+      run-ts-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-ts-sdk-e2e-tests }}
       run-system-tests: ${{ steps.selective-checks.outputs.run-system-tests }}
       run-task-sdk-tests: ${{ steps.selective-checks.outputs.run-task-sdk-tests }}
       run-task-sdk-integration-tests: ${{ steps.selective-checks.outputs.run-task-sdk-integration-tests }}
@@ -930,6 +931,7 @@ jobs:
       # yamllint disable rule:line-length
       providers-compatibility-tests-matrix: >
         ${{ needs.build-info.outputs.providers-compatibility-tests-matrix }}
+      run-ts-sdk-e2e-tests: ${{ needs.build-info.outputs.run-ts-sdk-e2e-tests }}
       use-uv: ${{ needs.build-info.outputs.use-uv }}
       run-ui-e2e-tests: ${{ needs.build-info.outputs.run-ui-e2e-tests }}
       run-airflow-ctl-integration-tests: ${{ needs.build-info.outputs.run-airflow-ctl-integration-tests }}

.github/workflows/ci-arm.yml

@@ -138,6 +138,7 @@ jobs:
       run-go-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-go-sdk-e2e-tests }}
       run-openlineage-e2e-tests: ${{ steps.selective-checks.outputs.run-openlineage-e2e-tests }}
       run-openlineage-e2e-compat-tests: ${{ steps.selective-checks.outputs.run-openlineage-e2e-compat-tests }}
+      run-ts-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-ts-sdk-e2e-tests }}
       run-system-tests: ${{ steps.selective-checks.outputs.run-system-tests }}
       run-task-sdk-tests: ${{ steps.selective-checks.outputs.run-task-sdk-tests }}
       run-task-sdk-integration-tests: ${{ steps.selective-checks.outputs.run-task-sdk-integration-tests }}
@@ -919,6 +920,7 @@ jobs:
       # yamllint disable rule:line-length
       providers-compatibility-tests-matrix: >
         ${{ needs.build-info.outputs.providers-compatibility-tests-matrix }}
+      run-ts-sdk-e2e-tests: ${{ needs.build-info.outputs.run-ts-sdk-e2e-tests }}
       use-uv: ${{ needs.build-info.outputs.use-uv }}
       run-ui-e2e-tests: ${{ needs.build-info.outputs.run-ui-e2e-tests }}
       run-airflow-ctl-integration-tests: ${{ needs.build-info.outputs.run-airflow-ctl-integration-tests }}

airflow-e2e-tests/docker/ts.yml

@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Docker Compose override for ts_sdk E2E test mode.
+#
+# The stock worker image ships no Node.js runtime, so node-provider copies the
+# node binary from the same image conftest builds the bundle with into a
+# shared volume. The bundle is bind-mounted where NodeCoordinator scans, and
+# the worker consumes the "typescript" queue where @task.stub tasks are routed.
+---
+services:
+  node-provider:
+    image: ${NODE_IMAGE:-node:22-slim}
+    command: ["cp", "/usr/local/bin/node", "/opt/nodejs/node"]
+    volumes:
+      - nodejs-bin:/opt/nodejs
+
+  airflow-worker:
+    volumes:
+      - ./ts-bundles:/opt/airflow/ts-bundles:ro
+      - nodejs-bin:/opt/nodejs:ro
+    command: celery worker -q typescript,default
+    depends_on:
+      node-provider:
+        condition: service_completed_successfully
+
+volumes:
+  nodejs-bin:

airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py

@@ -55,13 +55,18 @@
     LANG_SDK_NATIVE_TOOLCHAIN,
     LOCALSTACK_PATH,
     LOGS_FOLDER,
+    NODE_IMAGE,
     OPENLINEAGE_COMPOSE_PATH,
     OPENSEARCH_PATH,
     PROVIDERS_MOUNT_CONTAINER_PATH,
     PROVIDERS_ROOT_PATH,
     SCALA_SPARK_EXAMPLE_DAGS_PATH,
     SCALA_SPARK_EXAMPLE_LIBS_PATH,
     TEST_REPORT_FILE,
+    TS_COMPOSE_PATH,
+    TS_SDK_BUILD_HOME_PATH,
+    TS_SDK_EXAMPLE_PATH,
+    TS_SDK_ROOT_PATH,
     XCOM_BUCKET,
 )
 
@@ -640,6 +645,92 @@ def _setup_openlineage_integration(dot_env_file, tmp_dir):
     copyfile(OPENLINEAGE_COMPOSE_PATH, tmp_dir / "openlineage.yml")
 
 
+def _build_ts_sdk_example_bundle(*, native=False):
+    build_commands = (
+        "pnpm install --frozen-lockfile && pnpm run build && cd example && pnpm install && pnpm run build"
+    )
+    if native:
+        console.print("[yellow]Building TypeScript SDK example bundle (host toolchain)...")
+        subprocess.run(["bash", "-c", build_commands], cwd=TS_SDK_ROOT_PATH, check=True)
+        return
+    # --user keeps build outputs owned by the current user; HOME is a
+    # writable, gitignored dir so pnpm/corepack caches persist between runs.
+    TS_SDK_BUILD_HOME_PATH.mkdir(parents=True, exist_ok=True)
+    # corepack shims go in $HOME/bin (on PATH) because the container user
+    # cannot write to /usr/local/bin.
+    build_script = (
+        'export PATH="$HOME/bin:$PATH"'
+        ' && mkdir -p "$HOME/bin"'
+        ' && corepack enable --install-directory "$HOME/bin"'
+        " && cd /repo/ts-sdk"
+        f" && {build_commands}"
+    )
+    console.print(f"[yellow]Building TypeScript SDK example bundle ({NODE_IMAGE})...")
+    subprocess.run(
+        [
+            "docker",
+            "run",
+            "--rm",
+            "--user",
+            f"{os.getuid()}:{os.getgid()}",
+            "-e",
+            "HOME=/repo/files/pnpm-home",
+            "-e",
+            "COREPACK_ENABLE_DOWNLOAD_PROMPT=0",
+            "-e",
+            "CI=true",
+            "-v",
+            f"{AIRFLOW_ROOT_PATH}:/repo",
+            NODE_IMAGE,
+            "bash",
+            "-c",
+            build_script,
+        ],
+        check=True,
+    )
+
+
+def _setup_ts_sdk_integration(dot_env_file, tmp_dir):
+    """Set up the ts_sdk E2E test mode."""
+    _build_ts_sdk_example_bundle(native=LANG_SDK_NATIVE_TOOLCHAIN)
+
+    copyfile(TS_COMPOSE_PATH, tmp_dir / "ts.yml")
+
+    # Deliberately no metadata sidecar: the coordinator must resolve the schema
+    # version from the metadata airflow-ts-pack embedded in the bundle.
+    ts_bundles_dir = tmp_dir / "ts-bundles"
+    ts_bundles_dir.mkdir()
+    copyfile(TS_SDK_EXAMPLE_PATH / "dist" / "bundle.mjs", ts_bundles_dir / "bundle.mjs")
+
+    copyfile(
+        TS_SDK_EXAMPLE_PATH / "dags" / "typescript_example.py", tmp_dir / "dags" / "typescript_example.py"
+    )
+
+    coordinator_config = json.dumps(
+        {
+            "ts": {
+                "classpath": "airflow.sdk.coordinators.node.NodeCoordinator",
+                "kwargs": {
+                    "bundles_root": ["/opt/airflow/ts-bundles"],
+                    "node_executable": "/opt/nodejs/node",
+                },
+            }
+        }
+    )
+    queue_to_coordinator = json.dumps({"typescript": "ts"})
+
+    dot_env_file.write_text(
+        f"AIRFLOW_UID={os.getuid()}\n"
+        f"NODE_IMAGE={NODE_IMAGE}\n"
+        # single-quoted so Docker Compose reads the JSON literally
+        f"AIRFLOW__SDK__COORDINATORS='{coordinator_config}'\n"
+        f"AIRFLOW__SDK__QUEUE_TO_COORDINATOR='{queue_to_coordinator}'\n"
+        "AIRFLOW_CONN_TYPESCRIPT_EXAMPLE_HTTP=http://user:pass@example.com/\n"
+        "AIRFLOW_VAR_TYPESCRIPT_EXAMPLE_GREETING=greetings from e2e\n"
+    )
+    os.environ["ENV_FILE_PATH"] = str(dot_env_file)
+
+
 def spin_up_airflow_environment(tmp_path_factory: pytest.TempPathFactory):
     tmp_dir = tmp_path_factory.mktemp("breeze-airflow-e2e-tests")
 
@@ -694,6 +785,9 @@ def spin_up_airflow_environment(tmp_path_factory: pytest.TempPathFactory):
     elif E2E_TEST_MODE == "openlineage":
         compose_file_names.append("openlineage.yml")
         _setup_openlineage_integration(dot_env_file, tmp_dir)
+    elif E2E_TEST_MODE == "ts_sdk":
+        compose_file_names.append("ts.yml")
+        _setup_ts_sdk_integration(dot_env_file, tmp_dir)
 
     #
     # Please Do not use this Fernet key in any deployments! Please generate your own key.

airflow-e2e-tests/tests/airflow_e2e_tests/constants.py

@@ -90,6 +90,16 @@
 # and module fetches go through the HTTPS proxy (no git/gcc needed).
 GO_BUILDER_IMAGE = os.environ.get("GO_BUILDER_IMAGE", "golang:1.25-alpine")
 
+# TypeScript SDK E2E test paths
+TS_SDK_ROOT_PATH = AIRFLOW_ROOT_PATH / "ts-sdk"
+TS_SDK_EXAMPLE_PATH = TS_SDK_ROOT_PATH / "example"
+TS_COMPOSE_PATH = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" / "docker" / "ts.yml"
+# Builds the bundle and provides the worker's node binary (ts.yml), so the
+# bundle runs on the runtime it was built for.
+NODE_IMAGE = os.environ.get("NODE_IMAGE", "node:22-slim")
+# Writable HOME for the containerized pnpm build; gitignored, caches persist.
+TS_SDK_BUILD_HOME_PATH = AIRFLOW_ROOT_PATH / "files" / "pnpm-home"
+
 # Local provider sources are mounted into the airflow containers under this directory so
 # ``_PIP_ADDITIONAL_REQUIREMENTS`` can install the in-tree (latest, possibly unreleased)
 # provider rather than the published one from PyPI.

airflow-e2e-tests/tests/airflow_e2e_tests/ts_sdk_tests/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.