Patch for the ITs for MSQ

.travis.yml

@@ -46,7 +46,7 @@ addons:
 # Add various options to make 'mvn install' fast and skip javascript compile (-Ddruid.console.skip=true) since it is not
 # needed. Depending on network speeds, "mvn -q install" may take longer than the default 10 minute timeout to print any
 # output.  To compensate, use travis_wait to extend the timeout.
-install: ./check_test_suite.py && travis_terminate 0  || echo 'Running Maven install...' && MAVEN_OPTS='-Xmx3000m' travis_wait 15 ${MVN} clean install -q -ff -pl '!distribution,!:it-tools,!:it-image' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C && ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
+install: ./check_test_suite.py && travis_terminate 0  || echo 'Running Maven install...' && MAVEN_OPTS='-Xmx3000m' travis_wait 15 ${MVN} clean install -q -ff -pl '!distribution,!:druid-it-tools,!:druid-it-image,!:druid-it-cases' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C && ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
 
 # There are 3 stages of tests
 # 1. Tests - phase 1
@@ -73,19 +73,6 @@ jobs:
       stage: Tests - phase 1
       script: ${MVN} animal-sniffer:check --fail-at-end
 
-    # Experimental run of the revised ITs. Done early to debug issues
-    # Disabled for now. Integrating this into the build will come in a later PR.
-    - name: "experimental docker tests"
-      stage: Tests - phase 1
-      # Uses the install defined above. Then, builds the test tools and docker image,
-      # and run the various IT tests. If tests fail, echos log lines of any of
-      # the Druid services that did not exit normally.
-      # Run though install to ensure the test tools are installed, and the docker
-      # image is built. The tests only need verify.
-      script: ${MVN} install -P dist,test-image,docker-tests,IT-HighAvailability -rf :distribution ${MAVEN_SKIP} -DskipUTs=true
-      #after_failure:
-      #  - docker-tests/check-results.sh 
-
     - name: "checkstyle"
       script: ${MVN} checkstyle:checkstyle --fail-at-end
 
@@ -470,9 +457,9 @@ jobs:
           docker exec -it druid-$v sh -c 'dmesg | tail -3' ;
           done
 
-    - <<: *integration_batch_index
-      name: "(Compile=openjdk8, Run=openjdk8) batch index integration test with Indexer"
-      env: TESTNG_GROUPS='-Dgroups=batch-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer'
+    #- <<: *integration_batch_index
+    #  name: "(Compile=openjdk8, Run=openjdk8) batch index integration test with Indexer"
+    #  env: TESTNG_GROUPS='-Dgroups=batch-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer'
 
     - &integration_input_format
       name: "(Compile=openjdk8, Run=openjdk8) input format integration test"
@@ -679,16 +666,33 @@ jobs:
       name: "(Compile=openjdk8, Run=openjdk8) other integration tests with Indexer"
       env: TESTNG_GROUPS='-DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer'
 
-    - <<: *integration_tests
-      name: "(Compile=openjdk8, Run=openjdk8) leadership and high availability integration tests"
-      jdk: openjdk8
-      env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' OVERRIDE_CONFIG_PATH='./environment-configs/test-groups/prepopulated-data'
+    #- <<: *integration_tests
+    #  name: "(Compile=openjdk8, Run=openjdk8) leadership and high availability integration tests"
+    #  jdk: openjdk8
+    #  env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' OVERRIDE_CONFIG_PATH='./environment-configs/test-groups/prepopulated-data'
 
     - <<: *integration_query
       name: "(Compile=openjdk8, Run=openjdk8) query integration test (mariaDB)"
       jdk: openjdk8
       env: TESTNG_GROUPS='-Dgroups=query' USE_INDEXER='middleManager' MYSQL_DRIVER_CLASSNAME='org.mariadb.jdbc.Driver' OVERRIDE_CONFIG_PATH='./environment-configs/test-groups/prepopulated-data'
 
+    # Revised ITs.
+    - &integration_tests_ex
+      name: "(Compile=openjdk8, Run=openjdk8) leadership and high availability integration tests (new)"
+      stage: Tests - phase 2
+      jdk: openjdk8
+      services: *integration_test_services
+      env: JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
+      # Uses the install defined above. Then, builds the test tools and docker image,
+      # and runs one IT. If tests fail, echos log lines of any of
+      # the Druid services that did not exit normally.
+      script: ./it.sh travis HighAvailability
+
+    - <<: *integration_tests_ex
+      name: "(Compile=openjdk8, Run=openjdk8) batch index integration test with Indexer (new)"
+      env: JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer'
+      script: ./it.sh travis BatchIndex
+
     # END - Integration tests for Compile with Java 8 and Run with Java 8
 
     # START - Integration tests for Compile with Java 8 and Run with Java 11
@@ -769,21 +773,6 @@ jobs:
 
     # END - Integration tests for Compile with Java 8 and Run with Java 11
 
-    # BEGIN - Revised integration tests
-
-    # Experimental build of the revised integration test Docker image.
-    # Actual tests will come later.
-    - name: "experimental docker tests"
-      stage: Tests - phase 2
-      # Uses the install defined above. Then, builds the test tools and docker image,
-      # and run the various IT tests. If tests fail, echos log lines of any of
-      # the Druid services that did not exit normally.
-      # Run though install to ensure the test tools are installed, and the docker
-      # image is built. The tests only need verify.
-      script: ${MVN} install -P dist,test-image -rf :distribution ${MAVEN_SKIP} -DskipUTs=true
-
-    # END - Revised integration tests
-
     - &integration_batch_index_k8s
       name: "(Compile=openjdk8, Run=openjdk8, Cluster Build On K8s) ITNestedQueryPushDownTest integration test"
       stage: Tests - phase 2

core/src/main/java/org/apache/druid/guice/PolyBind.java

@@ -43,8 +43,8 @@
  * Provides the ability to create "polymorphic" bindings where the polymorphism is actually just making a decision
  * based on a value in Properties.
  * <p>
- * The workflow is that you first create a choice by calling {@link #createChoice()}. Then you create options using
- * the binder returned by the {@link #optionBinder()} method. Multiple different modules can call
+ * The workflow is that you first create a choice by calling {@code createChoice()}. Then you create options using
+ * the binder returned by the {@code optionBinder()} method. Multiple different modules can call
  * {@code optionBinder()} and all options will be reflected at injection time as long as equivalent interface
  * {@code Key} objects are passed into the various methods.
  */

core/src/test/java/org/apache/druid/data/input/impl/prefetch/PrefetchableTextFilesFirehoseFactoryTest.java

@@ -41,6 +41,7 @@
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
+import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -249,6 +250,7 @@ public void testWithCacheAndFetch() throws IOException
   }
 
   @Test
+  @Ignore("See issue #12638")
   public void testWithLargeCacheAndSmallFetch() throws IOException
   {
     final TestPrefetchableTextFilesFirehoseFactory factory =
@@ -336,6 +338,7 @@ public void testTimeout() throws IOException
   }
 
   @Test
+  @Ignore("See issue #12638")
   public void testReconnectWithCacheAndPrefetch() throws IOException
   {
     final TestPrefetchableTextFilesFirehoseFactory factory =

core/src/test/java/org/apache/druid/guice/PolyBindTest.java

@@ -112,17 +112,17 @@ public void configure(Binder binder)
     }
     catch (Exception e) {
       Assert.assertTrue(e instanceof ProvisionException);
-      Assert.assertTrue(e.getMessage().contains("Unknown provider[c] of Key[type=org.apache.druid.guice.PolyBindTest$Gogo"));
+      Assert.assertTrue(e.getMessage().contains("Unknown provider [c] of Key[type=org.apache.druid.guice.PolyBindTest$Gogo"));
     }
     try {
       Assert.assertEquals("B", injector.getInstance(Key.get(Gogo.class, Names.named("reverse"))).go());
       Assert.fail(); // should never be reached
     }
     catch (Exception e) {
       Assert.assertTrue(e instanceof ProvisionException);
-      Assert.assertTrue(e.getMessage().contains("Unknown provider[c] of Key[type=org.apache.druid.guice.PolyBindTest$Gogo"));
+      Assert.assertTrue(e.getMessage().contains("Unknown provider [c] of Key[type=org.apache.druid.guice.PolyBindTest$Gogo"));
     }
-    
+
     // test default property value
     Assert.assertEquals("B", injector.getInstance(GogoSally.class).go());
     props.setProperty("sally", "a");
@@ -136,7 +136,7 @@ public void configure(Binder binder)
     }
     catch (Exception e) {
       Assert.assertTrue(e instanceof ProvisionException);
-      Assert.assertTrue(e.getMessage().contains("Unknown provider[c] of Key[type=org.apache.druid.guice.PolyBindTest$GogoSally"));
+      Assert.assertTrue(e.getMessage().contains("Unknown provider [c] of Key[type=org.apache.druid.guice.PolyBindTest$GogoSally"));
     }
   }
 

extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/report/MSQStagesReport.java

@@ -22,7 +22,6 @@
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonValue;
 import com.google.common.base.Preconditions;
 import org.apache.druid.msq.kernel.QueryDefinition;
 import org.apache.druid.msq.kernel.StageDefinition;
@@ -84,7 +83,7 @@ public static MSQStagesReport create(
     return new MSQStagesReport(stages);
   }
 
-  @JsonValue
+  @JsonProperty("stages")
   public List<Stage> getStages()
   {
     return stages;

extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/report/MSQTaskReport.java

@@ -55,6 +55,13 @@ public String getReportKey()
     return REPORT_KEY;
   }
 
+  @JsonProperty("type")
+  private String getType()
+  {
+    return REPORT_KEY;
+  }
+
+
   @Override
   @JsonProperty
   public Object getPayload()

integration-tests-ex/README.md

@@ -32,21 +32,26 @@ an explanation.
 
 ### Build Druid
 
+To make the text a bit simpler, define a variable for the standard settings:
+
+```bash
+export MAVEN_IGNORE=-P skip-static-checks,skip-tests -Dmaven.javadoc.skip=true
+
 ```bash
-mvn clean package -P dist,skip-static-checks,skip-tests -Dmaven.javadoc.skip=true -T1.0C
+mvn clean package -P dist $MAVEN_IGNORE -T1.0C
 ```
 
 ### Build the Test Image
 
 ```bash
 cd $DRUID_DEV/integration-tests-ex/image
-mvn -P test-image install
+mvn install -P test-image $MAVEN_IGNORE
 ```
 
 ### Run an IT from the Command Line
 
 ```bash
-mvn install -P IT-<category> -pl :druid-it-cases
+mvn verify -P IT-<category> -pl :druid-it-cases $MAVEN_IGNORE
 ```
 
 Where `<category>` is one of the test categories.
@@ -56,7 +61,8 @@ Or
 ```bash
 cd $DRUID_DEV/integration-tests-ex/cases
 mvn verify -P skip-static-checks,docker-tests,IT-<category> \
-    -Dmaven.javadoc.skip=true -DskipUTs=true
+    -Dmaven.javadoc.skip=true -DskipUTs=true \
+    -pl :druid-it-cases
 ```
 
 ### Run an IT from the IDE
@@ -75,6 +81,7 @@ test as a JUnit test.
 
 * [Goals](#Goals)
 * [Quickstart](docs/quickstart.md)
+* [Create a new test](docs/guide.md)
 * [Maven configuration](docs/maven.md)
 * [Travis integration](docs/travis.md)
 * [Docker image](docs/docker.md)

integration-tests-ex/cases/.gitignore

@@ -0,0 +1 @@
+/bin/