Update

.github/labeler.yml

@@ -18,9 +18,50 @@
 #
 
 #
-# Pull Request Labeler Github Action Configuration: https://github.com/marketplace/actions/labeler
+# Pull Request Labeler GitHub Action Configuration: https://github.com/marketplace/actions/labeler
 
-"Area - Documentation":
-  - "docs/**/*"
-  - "website/**"
-  - "examples/quickstart/jupyter-notebooks/**"
+'Area - Batch Ingestion':
+  - 'indexing-hadoop/**'
+  - 'extensions-core/multi-stage-query/**'
+
+'Area - Dependencies':
+  - '**/pom.xml'
+  - 'licenses.yaml'
+
+'Area - Documentation':
+  - 'docs/**/*'
+  - 'website/**'
+  - 'examples/quickstart/jupyter-notebooks/**'
+
+'Area - Ingestion':
+  - 'indexing-service/**'
+
+'Area - Lookups':
+  - 'extensions-core/lookups-cached-global/**'
+  - 'extensions-core/lookups-cached-single/**'
+  - 'extensions-core/kafka-extraction-namespace/**'
+
+'Area - Metrics/Event Emitting':
+  - 'processing/src/main/java/org/apache/druid/java/util/metrics/**'
+  - 'processing/src/main/java/org/apache/druid/java/util/emitter/**'
+  - 'extensions-contrib/*-emitter/**'
+
+'Area - MSQ':
+  - 'extensions-core/multi-stage-query/**'
+
+'Area - Querying':
+  - 'sql/**'
+  - 'extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/**'
+
+'Area - Segment Format and Ser/De':
+  - 'processing/src/main/java/org/apache/druid/segment/**'
+
+'Area - Streaming Ingestion':
+  - 'extensions-core/kafka-indexing-service/**'
+  - 'extensions-core/kinesis-indexing-service/**'
+
+'Area - Web Console':
+  - 'web-console/**'
+
+'Kubernetes':
+  - 'extensions-contrib/kubernetes-overlord-extensions/**'

.github/workflows/codeql.yml

@@ -29,6 +29,12 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@v3
 
+    - uses: actions/setup-java@v3
+      with:
+        distribution: 'zulu'
+        java-version: '8'
+        cache: 'maven'
+
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v2
@@ -42,7 +48,6 @@ jobs:
         # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
         queries: +security-and-quality
 
-
     - run: |
         echo "Building using custom commands"
         mvn clean package -f "pom.xml" -B -V -e -Dfindbugs.skip -Dcheckstyle.skip -Dpmd.skip=true -Denforcer.skip -Dmaven.javadoc.skip -DskipTests -Dmaven.test.skip.exec -Dlicense.skip=true -Dweb.console.skip=true -Dcyclonedx.skip=true

.github/workflows/cron-job-its.yml

@@ -34,8 +34,11 @@ jobs:
       - name: Checkout branch
         uses: actions/checkout@v3
 
-      - name: Setup java
-        run: export JAVA_HOME=$JAVA_HOME_8_X64
+      - name: setup java
+        uses: actions/setup-java@v3
+        with:
+          java-version: '8'
+          distribution: 'zulu'
 
       - name: Cache Maven m2 repository
         id: maven

.github/workflows/reusable-revised-its.yml

@@ -77,9 +77,11 @@ jobs:
       - name: Checkout branch
         uses: actions/checkout@v3
 
-      - name: Setup java
-        run: |
-          echo "JAVA_HOME=$JAVA_HOME_${{ inputs.build_jdk }}_X64" >> $GITHUB_ENV
+      - name: setup java
+        uses: actions/setup-java@v3
+        with:
+          java-version: ${{ inputs.build_jdk }}
+          distribution: 'zulu'
 
       - name: Restore Maven repository
         id: maven-restore
@@ -132,4 +134,37 @@ jobs:
           docker images
 
       - name: Run IT
+        id: run-it
         run: ${{ inputs.script }}
+
+      - name: Collect docker logs on failure
+        if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
+        run: |
+          mkdir docker-logs
+          for c in $(docker ps -a --format="{{.Names}}")
+          do
+            docker logs $c > ./docker-logs/$c.log
+          done
+
+      - name: Tar docker logs
+        if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
+        run: tar cvzf ./docker-logs.tgz ./docker-logs
+
+      - name: Upload docker logs to GitHub
+        if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
+        uses: actions/upload-artifact@master
+        with:
+          name: IT-${{ inputs.it }} docker logs (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }}, Indexer=${{ inputs.use_indexer }}, Mysql=${{ inputs.mysql_driver }})
+          path: docker-logs.tgz
+
+      - name: Collect service logs on failure
+        if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
+        run: |
+          tar cvzf ./service-logs.tgz integration-tests-ex/cases/target/${{ inputs.it }}/logs
+
+      - name: Upload Druid service logs to GitHub
+        if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
+        uses: actions/upload-artifact@master
+        with:
+          name: IT-${{ inputs.it }} service logs (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }}, Indexer=${{ inputs.use_indexer }}, Mysql=${{ inputs.mysql_driver }})
+          path: service-logs.tgz

.github/workflows/reusable-standard-its.yml

@@ -62,9 +62,11 @@ jobs:
       - name: Checkout branch
         uses: actions/checkout@v3
 
-      - name: Setup java
-        run: |
-          echo "JAVA_HOME=$JAVA_HOME_${{ inputs.runtime_jdk }}_X64" >> $GITHUB_ENV
+      - name: setup java
+        uses: actions/setup-java@v3
+        with:
+          java-version: ${{ inputs.runtime_jdk }}
+          distribution: 'zulu'
 
       - name: Restore Maven repository
         id: maven-restore
@@ -88,13 +90,34 @@ jobs:
           echo "${MVN} verify -pl integration-tests -P integration-tests ${{ inputs.testing_groups }} -Djvm.runtime=${{ inputs.runtime_jdk }} -Dit.indexer=${{ inputs.use_indexer }} ${MAVEN_SKIP} -Doverride.config.path=${{ inputs.override_config_path }}"
           ${MVN} verify -pl integration-tests -P integration-tests ${{ inputs.testing_groups }} -Djvm.runtime=${{ inputs.runtime_jdk }} -Dit.indexer=${{ inputs.use_indexer }} ${MAVEN_SKIP} -Doverride.config.path=${{ inputs.override_config_path }}
 
-      - name: Debug IT
+      - name: Collect docker logs on failure
         if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
         run: |
-          for v in broker router ${{ inputs.use_indexer }} historical coordinator overlord; do
-            echo "=======================druid-"$v"========================";
-            echo "-----------------------docker logs-----------------------";
-            sudo docker logs druid-"$v" 2>&1 | tail -1000 ||:;
-            echo "-----------------------service logs----------------------";
-            sudo docker exec druid-"$v" tail -1000 /shared/logs/"$v".log 2>&1 ||:;
+          mkdir docker-logs
+          for c in $(docker ps -a --format="{{.Names}}")
+          do
+            docker logs $c > ./docker-logs/$c.log
           done
+
+      - name: Tar docker logs
+        if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
+        run: tar cvzf ./docker-logs.tgz ./docker-logs
+
+      - name: Upload docker logs to GitHub
+        if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
+        uses: actions/upload-artifact@master
+        with:
+          name: IT-${{ inputs.group }} docker logs (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }}, Indexer=${{ inputs.use_indexer }}, Mysql=${{ inputs.mysql_driver }})
+          path: docker-logs.tgz
+
+      - name: Collect service logs on failure
+        if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
+        run: |
+          tar cvzf ./service-logs.tgz ~/shared/logs
+
+      - name: Upload Druid service logs to GitHub
+        if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
+        uses: actions/upload-artifact@master
+        with:
+          name: IT-${{ inputs.group }} service logs (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }}, Indexer=${{ inputs.use_indexer }}, Mysql=${{ inputs.mysql_driver }})
+          path: service-logs.tgz

.github/workflows/reusable-unit-tests.yml

@@ -59,9 +59,15 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: setup jdk${{ inputs.jdk }}
-        run: echo "JAVA_HOME=$JAVA_HOME_${{ inputs.jdk }}_X64" >> $GITHUB_ENV
+      # skip the "cache: maven" step from setup-java. We explicitly use a
+      # different cache key since we cannot reuse it across commits.
+      - uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: ${{ inputs.jdk }}
 
+      # the build step produces SNAPSHOT artifacts into the local maven repository,
+      # we include github.sha in the cache key to make it specific to that build/jdk
       - name: Restore Maven repository
         id: maven-restore
         uses: actions/cache/restore@v3

.github/workflows/standard-its.yml

@@ -77,7 +77,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        testing_group: [query, query-retry, query-error, security, high-availability]
+        testing_group: [query, query-retry, query-error, security, high-availability, centralized-table-schema]
     uses: ./.github/workflows/reusable-standard-its.yml
     if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
     with:
@@ -93,7 +93,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        jdk: [8, 17]
+        jdk: [8, 17, 21]
     uses: ./.github/workflows/reusable-standard-its.yml
     if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
     with:
@@ -150,15 +150,21 @@ jobs:
       - name: Checkout branch
         uses: actions/checkout@v3
 
-      - name: Setup java
-        run: export JAVA_HOME=$JAVA_HOME_8_X64
+      - name: setup java
+        uses: actions/setup-java@v3
+        with:
+          java-version: '8'
+          distribution: 'zulu'
 
+      # the build step produces SNAPSHOT artifacts into the local maven repository,
+      # we include github.sha in the cache key to make it specific to that build/jdk
       - name: Restore Maven repository
         id: maven-restore
         uses: actions/cache/restore@v3
         with:
           path: ~/.m2/repository
           key: maven-${{ runner.os }}-8-${{ github.sha }}
+          restore-keys: setup-java-Linux-maven-${{ hashFiles('**/pom.xml') }}
 
       - name: Maven build
         if: steps.maven-restore.outputs.cache-hit != 'true'
@@ -189,6 +195,6 @@ jobs:
     with:
       build_jdk: 8
       runtime_jdk: 8
-      testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties
+      testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties,centralized-table-schema
       use_indexer: ${{ matrix.indexer }}
       group: other

.github/workflows/static-checks.yml

@@ -41,20 +41,17 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        java: [ 'jdk8', 'jdk11', 'jdk17' ]
+        java: [ '8', '11', '17', '21' ]
     runs-on: ubuntu-latest
     steps:
       - name: checkout branch
         uses: actions/checkout@v3
 
-      - name: set java version
-        run: |
-          export jdk=${{ matrix.java }}
-          echo "java_version=${jdk:3}" >> $GITHUB_ENV
-
-      - name: setup ${{ matrix.java }}
-        run: |
-          echo "JAVA_HOME=$JAVA_HOME_${{ env.java_version }}_X64" >> $GITHUB_ENV
+      - uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: ${{ matrix.java }}
+          cache: 'maven'
 
       - name: packaging check
         run: |
@@ -66,53 +63,53 @@ jobs:
 
       - name: script checks
         # who watches the watchers?
-        if: ${{ matrix.java == 'jdk8' }}
+        if: ${{ matrix.java == '8' }}
         run: ./check_test_suite_test.py
 
       - name: (openjdk17) strict compilation
-        if: ${{ matrix.java == 'jdk17' }}
+        if: ${{ matrix.java == '17' }}
         # errorprone requires JDK 11+
         # Strict compilation requires more than 2 GB
         run: ${MVN} clean -DstrictCompile compile test-compile --fail-at-end ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
 
       - name: maven install
-        if: ${{ matrix.java == 'jdk8' }}
+        if: ${{ matrix.java == '8' }}
         run: |
           echo 'Running Maven install...' &&
           ${MVN} clean install -q -ff -pl '!distribution,!:druid-it-image,!:druid-it-cases' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C &&
           ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
 
       - name: checkstyle
-        if: ${{ matrix.java == 'jdk8' }}
+        if: ${{ matrix.java == '8' }}
         run: ${MVN} checkstyle:checkstyle --fail-at-end
 
       - name: license checks
-        if: ${{ matrix.java == 'jdk8' }}
+        if: ${{ matrix.java == '8' }}
         run: ./.github/scripts/license_checks_script.sh
 
       - name: analyze dependencies
-        if: ${{ matrix.java == 'jdk8' }}
+        if: ${{ matrix.java == '8' }}
         run: |
           ./.github/scripts/analyze_dependencies_script.sh
 
       - name: animal sniffer checks
-        if: ${{ matrix.java == 'jdk8' }}
+        if: ${{ matrix.java == '8' }}
         run: ${MVN} animal-sniffer:check --fail-at-end
 
       - name: enforcer checks
-        if: ${{ matrix.java == 'jdk8' }}
+        if: ${{ matrix.java == '8' }}
         run: ${MVN} enforcer:enforce --fail-at-end
 
       - name: forbidden api checks
-        if: ${{ matrix.java == 'jdk8' }}
+        if: ${{ matrix.java == '8' }}
         run: ${MVN} forbiddenapis:check forbiddenapis:testCheck --fail-at-end
 
       - name: pmd checks
-        if: ${{ matrix.java == 'jdk8' }}
+        if: ${{ matrix.java == '8' }}
         run: ${MVN} pmd:check --fail-at-end  # TODO: consider adding pmd:cpd-check
 
       - name: spotbugs checks
-        if: ${{ matrix.java == 'jdk8' }}
+        if: ${{ matrix.java == '8' }}
         run: ${MVN} spotbugs:check --fail-at-end -pl '!benchmarks'
 
   intellij-inspections:
@@ -123,9 +120,11 @@ jobs:
       - name: checkout branch
         uses: actions/checkout@v3
 
-      - name: setup JDK8
-        run: |
-          echo "JAVA_HOME=$JAVA_HOME_8_X64" >> $GITHUB_ENV
+      - uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: '8'
+          cache: 'maven'
 
       - name: maven install
         run: |
@@ -153,9 +152,11 @@ jobs:
       - name: checkout branch
         uses: actions/checkout@v3
 
-      - name: setup JDK17
-        run: |
-          echo "JAVA_HOME=$JAVA_HOME_17_X64" >> $GITHUB_ENV
+      - uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: '17'
+          cache: 'maven'
 
       - name: setup node
         uses: actions/setup-node@v3