From ac14060243f06d578358e04b55e0946d3bc2de6c Mon Sep 17 00:00:00 2001 From: Paul Rogers Date: Tue, 16 Aug 2022 23:45:09 -0700 Subject: [PATCH 01/11] Refine the IT framework Support for all of the IntegrationTestingConfig properties Wrapper script for IT operations Additional documentation Build fix --- .travis.yml | 2 +- integration-tests-ex/README.md | 14 +- .../druid/testsEx/config/ClusterConfig.java | 14 + .../testsEx/config/ClusterConfigTest.java | 27 +- .../druid/testsEx/config/Initializer.java | 21 +- .../config/IntegrationTestingConfigEx.java | 421 +++++++++++++++++ .../druid/testsEx/config/ResolvedConfig.java | 443 +++--------------- .../testsEx/config/ResolvedMetastore.java | 18 +- .../druid/testsEx/config/TestConfigs.java | 18 + .../src/test/resources/config-test/test.yaml | 10 + integration-tests-ex/docs/conversion.md | 13 + integration-tests-ex/docs/quickstart.md | 102 ++-- integration-tests-ex/docs/test-config.md | 135 +++++- .../testing/ConfigFileConfigProvider.java | 3 +- .../IntegrationTestingConfigProvider.java | 1 + .../druid/testing/guice/DruidTestModule.java | 2 +- it.sh | 77 +++ pom.xml | 2 +- 18 files changed, 874 insertions(+), 449 deletions(-) create mode 100644 integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/IntegrationTestingConfigEx.java create mode 100755 it.sh diff --git a/.travis.yml b/.travis.yml index c245e0264732..e2351a6ee2f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,7 +46,7 @@ addons: # Add various options to make 'mvn install' fast and skip javascript compile (-Ddruid.console.skip=true) since it is not # needed. Depending on network speeds, "mvn -q install" may take longer than the default 10 minute timeout to print any # output. To compensate, use travis_wait to extend the timeout. -install: ./check_test_suite.py && travis_terminate 0 || echo 'Running Maven install...' && MAVEN_OPTS='-Xmx3000m' travis_wait 15 ${MVN} clean install -q -ff -pl '!distribution,!:it-tools,!:it-image' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C && ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} +install: ./check_test_suite.py && travis_terminate 0 || echo 'Running Maven install...' && MAVEN_OPTS='-Xmx3000m' travis_wait 15 ${MVN} clean install -q -ff -pl '!distribution,!:druid-it-tools,!:druid-it-image' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C && ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} # There are 3 stages of tests # 1. Tests - phase 1 diff --git a/integration-tests-ex/README.md b/integration-tests-ex/README.md index 1e0a85d9c3e6..a11c97414973 100644 --- a/integration-tests-ex/README.md +++ b/integration-tests-ex/README.md @@ -32,21 +32,26 @@ an explanation. ### Build Druid +To make the text a bit simpler, define a variable for the standard settings: + +```bash +export MAVEN_IGNORE=-P skip-static-checks,skip-tests -Dmaven.javadoc.skip=true + ```bash -mvn clean package -P dist,skip-static-checks,skip-tests -Dmaven.javadoc.skip=true -T1.0C +mvn clean package -P dist $MAVEN_IGNORE -T1.0C ``` ### Build the Test Image ```bash cd $DRUID_DEV/integration-tests-ex/image -mvn -P test-image install +mvn install -P test-image $MAVEN_IGNORE ``` ### Run an IT from the Command Line ```bash -mvn install -P IT- -pl :druid-it-cases +mvn verify -P IT- -pl :druid-it-cases $MAVEN_IGNORE ``` Where `` is one of the test categories. @@ -56,7 +61,8 @@ Or ```bash cd $DRUID_DEV/integration-tests-ex/cases mvn verify -P skip-static-checks,docker-tests,IT- \ - -Dmaven.javadoc.skip=true -DskipUTs=true + -Dmaven.javadoc.skip=true -DskipUTs=true \ + -pl :druid-it-cases ``` ### Run an IT from the IDE diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ClusterConfig.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ClusterConfig.java index 732746e18a25..a98072196a4b 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ClusterConfig.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ClusterConfig.java @@ -73,6 +73,8 @@ public enum ClusterType private KafkaConfig kafka; @JsonProperty("druid") private Map druidServices; + @JsonProperty("settings") + private Map settings; @JsonProperty("properties") private Map properties; @JsonProperty("metastoreInit") @@ -252,6 +254,13 @@ public Map druid() return druidServices; } + @JsonProperty("settings") + @JsonInclude(Include.NON_NULL) + public Map settings() + { + return settings; + } + @JsonProperty("properties") @JsonInclude(Include.NON_NULL) public Map properties() @@ -310,6 +319,11 @@ public ClusterConfig merge(ClusterConfig overrides) } else if (overrides.druidServices != null) { merged.druidServices.putAll(overrides.druidServices); } + if (merged.settings == null) { + merged.settings = overrides.settings; + } else if (overrides.settings != null) { + merged.settings.putAll(overrides.settings); + } if (merged.properties == null) { merged.properties = overrides.properties; } else if (overrides.properties != null) { diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ClusterConfigTest.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ClusterConfigTest.java index e71bfd0584b5..912f8d91c0f3 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ClusterConfigTest.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ClusterConfigTest.java @@ -19,10 +19,14 @@ package org.apache.druid.testsEx.config; +import org.apache.druid.testing.IntegrationTestingConfig; import org.apache.druid.testsEx.config.ClusterConfig.ClusterType; import org.apache.druid.testsEx.config.ResolvedService.ResolvedZk; import org.junit.Test; +import java.util.Map; +import java.util.Properties; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -43,7 +47,7 @@ public void testYaml() assertEquals(ClusterType.docker, resolved.type()); assertEquals(ResolvedConfig.DEFAULT_READY_TIMEOUT_SEC, resolved.readyTimeoutSec()); assertEquals(ResolvedConfig.DEFAULT_READY_POLL_MS, resolved.readyPollMs()); - assertEquals(1, resolved.properties().size()); + assertEquals(3, resolved.properties().size()); ResolvedZk zk = resolved.zk(); assertNotNull(zk); @@ -74,5 +78,26 @@ public void testYaml() assertEquals("router", service.service()); assertEquals("http://localhost:8888", service.clientUrl()); assertEquals("http://localhost:8888", resolved.routerUrl()); + + Map props = resolved.toProperties(); + // Added from ZK section + assertEquals("localhost:2181", props.get("druid.zk.service.zkHosts")); + // Generic property + assertEquals("howdy", props.get("my.test.property")); + // Mapped from settings + assertEquals("myBucket", props.get("druid.test.config.cloudBucket")); + assertEquals("myPath", props.get("druid.test.config.cloudPath")); + assertEquals("secret", props.get("druid.test.config.s3AccessKey")); + // From settings, overridden in properties + assertEquals("myRegion", props.get("druid.test.config.cloudRegion")); + + // Test plumbing through the test config + Properties properties = new Properties(); + properties.putAll(props); + IntegrationTestingConfig testingConfig = new IntegrationTestingConfigEx(resolved, properties); + assertEquals("myBucket", testingConfig.getCloudBucket()); + assertEquals("myPath", testingConfig.getCloudPath()); + // From settings, overridden in properties + assertEquals("myRegion", testingConfig.getCloudRegion()); } } diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/Initializer.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/Initializer.java index bcf7ba7666af..f61d9501b9e5 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/Initializer.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/Initializer.java @@ -70,6 +70,7 @@ import org.apache.druid.metadata.storage.mysql.MySQLMetadataStorageModule; import org.apache.druid.server.DruidNode; import org.apache.druid.testing.IntegrationTestingConfig; +import org.apache.druid.testing.IntegrationTestingConfigProvider; import org.apache.druid.testing.guice.TestClient; import org.apache.druid.testsEx.cluster.DruidClusterClient; import org.apache.druid.testsEx.cluster.MetastoreClient; @@ -132,7 +133,8 @@ public void configure(Binder binder) .toInstance(config); binder .bind(IntegrationTestingConfig.class) - .toInstance(config.toIntegrationTestingConfig()); + .to(IntegrationTestingConfigEx.class) + .in(LazySingleton.class); binder .bind(MetastoreClient.class) .in(LazySingleton.class); @@ -265,15 +267,24 @@ public Builder(String clusterName) property("druid.client.https.keyManagerPassword", "druid123"); property("druid.client.https.keyStorePassword", "druid123"); + // More env var bindings for properties formerly passed in via + // a generated config file. + final String base = IntegrationTestingConfigProvider.PROPERTY_BASE + "."; + propertyEnvVarBinding(base + "cloudBucket", "DRUID_CLOUD_BUCKET"); + propertyEnvVarBinding(base + "cloudPath", "DRUID_CLOUD_PATH"); + propertyEnvVarBinding(base + "s3AccessKey", "AWS_ACCESS_KEY_ID"); + propertyEnvVarBinding(base + "s3SecretKey", "AWS_SECRET_ACCESS_KEY"); + propertyEnvVarBinding(base + "azureContainer", "AZURE_CONTAINER"); + propertyEnvVarBinding(base + "azureAccount", "AZURE_ACCOUNT"); + propertyEnvVarBinding(base + "azureKey", "AZURE_KEY"); + propertyEnvVarBinding(base + "googleBucket", "GOOGLE_BUCKET"); + propertyEnvVarBinding(base + "googlePrefix", "GOOGLE_PREFIX"); + // Other defaults // druid.global.http.numMaxThreads avoids creating 40+ Netty threads. // We only ever use 1. property("druid.global.http.numMaxThreads", 3); property("druid.broker.http.numMaxThreads", 3); - - // druid.test.config.dockerIp is used by some older test code. Remove - // it when that code is updated. - property("druid.test.config.dockerIp", "localhost"); } /** diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/IntegrationTestingConfigEx.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/IntegrationTestingConfigEx.java new file mode 100644 index 000000000000..c14ea745aa81 --- /dev/null +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/IntegrationTestingConfigEx.java @@ -0,0 +1,421 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.testsEx.config; + +import com.google.common.collect.ImmutableMap; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.testing.IntegrationTestingConfig; +import org.apache.druid.testing.IntegrationTestingConfigProvider; + +import javax.inject.Inject; + +import java.util.Map; +import java.util.Properties; + +/** + * Adapter to the "legacy" cluster configuration used by tests. + */ +class IntegrationTestingConfigEx implements IntegrationTestingConfig +{ + private final ResolvedConfig config; + private final Map properties; + + @Inject + public IntegrationTestingConfigEx( + final ResolvedConfig config, + final Properties properties) + { + this.config = config; + ImmutableMap.Builder builder = ImmutableMap.builder(); + for (Map.Entry entry : properties.entrySet()) { + String key = (String) entry.getKey(); + if (key.startsWith(IntegrationTestingConfigProvider.PROPERTY_BASE)) { + key = key.substring(IntegrationTestingConfigProvider.PROPERTY_BASE.length() + 1); + builder.put(key, (String) entry.getValue()); + } + } + this.properties = builder.build(); + } + + @Override + public String getZookeeperHosts() + { + return config.requireZk().clientHosts(); + } + + @Override + public String getKafkaHost() + { + return config.requireKafka().instance().clientHost(); + } + + @Override + public String getKafkaInternalHost() + { + return config.requireKafka().instance().host(); + } + + @Override + public String getBrokerHost() + { + return config.requireBroker().instance().clientHost(); + } + + @Override + public String getBrokerInternalHost() + { + return config.requireBroker().instance().host(); + } + + @Override + public String getRouterHost() + { + return config.requireRouter().instance().clientHost(); + } + + @Override + public String getRouterInternalHost() + { + return config.requireRouter().instance().host(); + } + + @Override + public String getCoordinatorHost() + { + return config.requireCoordinator().tagOrDefault("one").clientHost(); + } + + @Override + public String getCoordinatorInternalHost() + { + return config.requireCoordinator().tagOrDefault("one").host(); + } + + @Override + public String getCoordinatorTwoInternalHost() + { + return config.requireCoordinator().requireInstance("two").host(); + } + + @Override + public String getCoordinatorTwoHost() + { + return config.requireCoordinator().tagOrDefault("one").clientHost(); + } + + @Override + public String getOverlordHost() + { + return config.requireOverlord().tagOrDefault("one").clientHost(); + } + + @Override + public String getOverlordTwoHost() + { + return config.requireOverlord().tagOrDefault("two").clientHost(); + } + + @Override + public String getOverlordInternalHost() + { + return config.requireOverlord().tagOrDefault("one").host(); + } + + @Override + public String getOverlordTwoInternalHost() + { + return config.requireOverlord().requireInstance("two").host(); + } + + @Override + public String getMiddleManagerHost() + { + return config.requireMiddleManager().instance().clientHost(); + } + + @Override + public String getMiddleManagerInternalHost() + { + return config.requireMiddleManager().instance().host(); + } + + @Override + public String getHistoricalHost() + { + return config.requireHistorical().instance().clientHost(); + } + + @Override + public String getHistoricalInternalHost() + { + return config.requireHistorical().instance().host(); + } + + @Override + public String getCoordinatorUrl() + { + ResolvedDruidService serviceConfig = config.requireCoordinator(); + return serviceConfig.resolveUrl(serviceConfig.tagOrDefault("one")); + } + + @Override + public String getCoordinatorTLSUrl() + { + throw new ISE("Not implemented"); + } + + @Override + public String getCoordinatorTwoUrl() + { + ResolvedDruidService serviceConfig = config.requireCoordinator(); + return serviceConfig.resolveUrl(serviceConfig.requireInstance("two")); + } + + @Override + public String getCoordinatorTwoTLSUrl() + { + throw new ISE("Not implemented"); + } + + @Override + public String getOverlordUrl() + { + ResolvedDruidService serviceConfig = config.requireOverlord(); + return serviceConfig.resolveUrl(serviceConfig.tagOrDefault("one")); + } + + @Override + public String getOverlordTLSUrl() + { + throw new ISE("Not implemented"); + } + + @Override + public String getOverlordTwoUrl() + { + ResolvedDruidService serviceConfig = config.requireOverlord(); + return serviceConfig.resolveUrl(serviceConfig.requireInstance("two")); + } + + @Override + public String getOverlordTwoTLSUrl() + { + throw new ISE("Not implemented"); + } + + @Override + public String getIndexerUrl() + { + ResolvedDruidService indexer = config.druidService(ResolvedConfig.INDEXER); + if (indexer == null) { + indexer = config.requireMiddleManager(); + } + return indexer.resolveUrl(indexer.instance()); + } + + @Override + public String getIndexerTLSUrl() + { + throw new ISE("Not implemented"); + } + + @Override + public String getRouterUrl() + { + return config.routerUrl(); + } + + @Override + public String getRouterTLSUrl() + { + ResolvedDruidService serviceConfig = config.requireRouter(); + return serviceConfig.resolveUrl(serviceConfig.tagOrDefault("tls")); + } + + @Override + public String getPermissiveRouterUrl() + { + throw new ISE("Not implemented"); + } + + @Override + public String getPermissiveRouterTLSUrl() + { + throw new ISE("Not implemented"); + } + + @Override + public String getNoClientAuthRouterUrl() + { + throw new ISE("Not implemented"); + } + + @Override + public String getNoClientAuthRouterTLSUrl() + { + throw new ISE("Not implemented"); + } + + @Override + public String getCustomCertCheckRouterUrl() + { + throw new ISE("Not implemented"); + } + + @Override + public String getCustomCertCheckRouterTLSUrl() + { + throw new ISE("Not implemented"); + } + + @Override + public String getBrokerUrl() + { + ResolvedDruidService serviceConfig = config.requireBroker(); + return serviceConfig.resolveUrl(serviceConfig.instance()); + } + + @Override + public String getBrokerTLSUrl() + { + ResolvedDruidService serviceConfig = config.requireBroker(); + return serviceConfig.resolveUrl(serviceConfig.tagOrDefault("tls")); + } + + @Override + public String getHistoricalUrl() + { + return config.requireHistorical().resolveUrl(); + } + + @Override + public String getHistoricalTLSUrl() + { + throw new ISE("Not implemented"); + } + + @Override + public String getProperty(String prop) + { + return properties.get(prop); + } + + @Override + public String getUsername() + { + return getProperty("username"); + } + + @Override + public String getPassword() + { + return getProperty("password"); + } + + @Override + public Map getProperties() + { + return properties; + } + + @Override + public boolean manageKafkaTopic() + { + throw new ISE("Not implemented"); + } + + @Override + public String getExtraDatasourceNameSuffix() + { + return config.datasourceNameSuffix; + } + + @Override + public String getCloudBucket() + { + return getProperty("cloudBucket"); + } + + @Override + public String getCloudPath() + { + return getProperty("cloudPath"); + } + + @Override + public String getCloudRegion() + { + return getProperty("cloudRegion"); + } + + @Override + public String getS3AssumeRoleWithExternalId() + { + return getProperty("s3AssumeRoleWithExternalId"); + } + + @Override + public String getS3AssumeRoleExternalId() + { + return getProperty("s3AssumeRoleExternalId"); + } + + @Override + public String getS3AssumeRoleWithoutExternalId() + { + return getProperty("s3AssumeRoleWithoutExternalId"); + } + + @Override + public String getAzureKey() + { + return getProperty("azureKey"); + } + + @Override + public String getHadoopGcsCredentialsPath() + { + return getProperty("hadoopGcsCredentialsPath"); + } + + @Override + public String getStreamEndpoint() + { + return getProperty("streamEndpoint"); + } + + @Override + public String getSchemaRegistryHost() + { + return getProperty("schemaRegistryHost"); + } + + @Override + public boolean isDocker() + { + return config.isDocker(); + } + + @Override + public String getDockerHost() + { + return config.proxyHost(); + } +} diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedConfig.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedConfig.java index fe7fad091aeb..e90853e570d3 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedConfig.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedConfig.java @@ -24,7 +24,7 @@ import org.apache.druid.curator.CuratorConfig; import org.apache.druid.curator.ExhibitorConfig; import org.apache.druid.java.util.common.ISE; -import org.apache.druid.testing.IntegrationTestingConfig; +import org.apache.druid.testing.IntegrationTestingConfigProvider; import org.apache.druid.testsEx.config.ClusterConfig.ClusterType; import org.apache.druid.testsEx.config.ResolvedService.ResolvedKafka; import org.apache.druid.testsEx.config.ResolvedService.ResolvedZk; @@ -52,8 +52,9 @@ public class ResolvedConfig private final String proxyHost; private final int readyTimeoutSec; private final int readyPollMs; - private final String datasourceNameSuffix; + final String datasourceNameSuffix; private Map properties; + private Map settings; private final ResolvedZk zk; private final ResolvedKafka kafka; @@ -78,6 +79,11 @@ public ResolvedConfig(ClusterConfig config) } else { this.properties = config.properties(); } + if (config.settings() == null) { + this.settings = ImmutableMap.of(); + } else { + this.settings = config.settings(); + } if (config.datasourceSuffix() == null) { this.datasourceNameSuffix = ""; } else { @@ -159,6 +165,11 @@ public ResolvedKafka kafka() return kafka; } + public Map settings() + { + return settings; + } + public Map properties() { return properties; @@ -172,6 +183,14 @@ public Map requireDruid() return druidServices; } + public ResolvedZk requireZk() + { + if (zk == null) { + throw new ISE("Please specify the ZooKeeper configuration"); + } + return zk; + } + public ResolvedMetastore requireMetastore() { if (metastore == null) { @@ -252,6 +271,28 @@ public ExhibitorConfig toExhibitorConfig() return ExhibitorConfig.create(Collections.emptyList()); } + /** + * Map from old-style config file (and settings) name to the + * corresponding property. + */ + private static final Map SETTINGS_MAP = + ImmutableMap.builder() + .put("cloud_bucket", "cloudBucket") + .put("cloud_path", "cloudPath") + .put("cloud_region", "cloudRegion") + .put("s3_assume_role_with_external_id", "s3AssumeRoleWithExternalId") + .put("s3_assume_role_external_id", "s3AssumeRoleExternalId") + .put("s3_assume_role_without_external_id", "s3AssumeRoleWithoutExternalId") + .put("stream_endpoint", "streamEndpoint") + .put("s3_accessKey", "s3AccessKey") + .put("s3_secretKey", "s3SecretKey") + .put("azure_account", "azureAccount") + .put("azure_key", "azureKey") + .put("azure_container", "azureContainer") + .put("google_bucket", "googleBucket") + .put("google_prefix", "googlePrefix") + .build(); + /** * Convert the config in this structure the the properties * used to configure Guice. @@ -259,9 +300,9 @@ public ExhibitorConfig toExhibitorConfig() public Map toProperties() { Map properties = new HashMap<>(); - if (proxyHost != null) { - properties.put("druid.test.config.dockerIp", proxyHost); - } + // druid.test.config.dockerIp is used by some older test code. Remove + // it when that code is updated. + TestConfigs.putProperty(properties, "druid.test.config.dockerIp", proxyHost); // Start with implicit properties from various sections. if (zk != null) { @@ -271,386 +312,24 @@ public Map toProperties() properties.putAll(metastore.toProperties()); } - // Add explicit properties - if (this.properties != null) { - properties.putAll(properties); - } - return properties; - } - - public IntegrationTestingConfig toIntegrationTestingConfig() - { - return new IntegrationTestingConfigShim(); - } - - /** - * Adapter to the "legacy" cluster configuration used by tests. - */ - private class IntegrationTestingConfigShim implements IntegrationTestingConfig - { - @Override - public String getZookeeperHosts() - { - return zk.clientHosts(); - } - - @Override - public String getKafkaHost() - { - return requireKafka().instance().clientHost(); - } - - @Override - public String getKafkaInternalHost() - { - return requireKafka().instance().host(); - } - - @Override - public String getBrokerHost() - { - return requireBroker().instance().clientHost(); - } - - @Override - public String getBrokerInternalHost() - { - return requireBroker().instance().host(); - } - - @Override - public String getRouterHost() - { - return requireRouter().instance().clientHost(); - } - - @Override - public String getRouterInternalHost() - { - return requireRouter().instance().host(); - } - - @Override - public String getCoordinatorHost() - { - return requireCoordinator().tagOrDefault("one").clientHost(); - } - - @Override - public String getCoordinatorInternalHost() - { - return requireCoordinator().tagOrDefault("one").host(); - } - - @Override - public String getCoordinatorTwoInternalHost() - { - return requireCoordinator().requireInstance("two").host(); - } - - @Override - public String getCoordinatorTwoHost() - { - return requireCoordinator().tagOrDefault("one").clientHost(); - } - - @Override - public String getOverlordHost() - { - return requireOverlord().tagOrDefault("one").clientHost(); - } - - @Override - public String getOverlordTwoHost() - { - return requireOverlord().tagOrDefault("two").clientHost(); - } - - @Override - public String getOverlordInternalHost() - { - return requireOverlord().tagOrDefault("one").host(); - } - - @Override - public String getOverlordTwoInternalHost() - { - return requireOverlord().requireInstance("two").host(); - } - - @Override - public String getMiddleManagerHost() - { - return requireMiddleManager().instance().clientHost(); - } - - @Override - public String getMiddleManagerInternalHost() - { - return requireMiddleManager().instance().host(); - } - - @Override - public String getHistoricalHost() - { - return requireHistorical().instance().clientHost(); - } - - @Override - public String getHistoricalInternalHost() - { - return requireHistorical().instance().host(); - } - - @Override - public String getCoordinatorUrl() - { - ResolvedDruidService config = requireCoordinator(); - return config.resolveUrl(config.tagOrDefault("one")); - } - - @Override - public String getCoordinatorTLSUrl() - { - throw new ISE("Not implemented"); - } - - @Override - public String getCoordinatorTwoUrl() - { - ResolvedDruidService config = requireCoordinator(); - return config.resolveUrl(config.requireInstance("two")); - } - - @Override - public String getCoordinatorTwoTLSUrl() - { - throw new ISE("Not implemented"); - } - - @Override - public String getOverlordUrl() - { - ResolvedDruidService config = requireOverlord(); - return config.resolveUrl(config.tagOrDefault("one")); - } - - @Override - public String getOverlordTLSUrl() - { - throw new ISE("Not implemented"); - } - - @Override - public String getOverlordTwoUrl() - { - ResolvedDruidService config = requireOverlord(); - return config.resolveUrl(config.requireInstance("two")); - } - - @Override - public String getOverlordTwoTLSUrl() - { - throw new ISE("Not implemented"); - } - - @Override - public String getIndexerUrl() - { - ResolvedDruidService indexer = druidService(INDEXER); - if (indexer == null) { - indexer = requireMiddleManager(); + // Add settings, converted to properties. Map both old and + // "property-style" settings to the full property path. + // Settings are converted to properties so they can be overridden + // by environment variables and -D command-line settings. + for (Map.Entry entry : settings.entrySet()) { + String key = entry.getKey(); + if (key.startsWith("druid_")) { + key = key.substring("druid_".length()); } - return indexer.resolveUrl(indexer.instance()); - } - - @Override - public String getIndexerTLSUrl() - { - throw new ISE("Not implemented"); - } - - @Override - public String getRouterUrl() - { - return routerUrl(); - } - - @Override - public String getRouterTLSUrl() - { - ResolvedDruidService config = requireRouter(); - return config.resolveUrl(config.tagOrDefault("tls")); - } - - @Override - public String getPermissiveRouterUrl() - { - throw new ISE("Not implemented"); - } - - @Override - public String getPermissiveRouterTLSUrl() - { - throw new ISE("Not implemented"); - } - - @Override - public String getNoClientAuthRouterUrl() - { - throw new ISE("Not implemented"); - } - - @Override - public String getNoClientAuthRouterTLSUrl() - { - throw new ISE("Not implemented"); - } - - @Override - public String getCustomCertCheckRouterUrl() - { - throw new ISE("Not implemented"); - } - - @Override - public String getCustomCertCheckRouterTLSUrl() - { - throw new ISE("Not implemented"); - } - - @Override - public String getBrokerUrl() - { - ResolvedDruidService config = requireBroker(); - return config.resolveUrl(config.instance()); - } - - @Override - public String getBrokerTLSUrl() - { - ResolvedDruidService config = requireBroker(); - return config.resolveUrl(config.tagOrDefault("tls")); - } - - @Override - public String getHistoricalUrl() - { - return requireHistorical().resolveUrl(); - } - - @Override - public String getHistoricalTLSUrl() - { - throw new ISE("Not implemented"); - } - - @Override - public String getProperty(String prop) - { - throw new ISE("Not implemented"); - } - - @Override - public String getUsername() - { - throw new ISE("Not implemented"); - } - - @Override - public String getPassword() - { - throw new ISE("Not implemented"); - } - - @Override - public Map getProperties() - { - throw new ISE("Not implemented"); - } - - @Override - public boolean manageKafkaTopic() - { - throw new ISE("Not implemented"); - } - - @Override - public String getExtraDatasourceNameSuffix() - { - return datasourceNameSuffix; - } - - @Override - public String getCloudBucket() - { - throw new ISE("Not implemented"); - } - - @Override - public String getCloudPath() - { - throw new ISE("Not implemented"); - } - - @Override - public String getCloudRegion() - { - throw new ISE("Not implemented"); - } - - @Override - public String getS3AssumeRoleWithExternalId() - { - throw new ISE("Not implemented"); - } - - @Override - public String getS3AssumeRoleExternalId() - { - throw new ISE("Not implemented"); - } - - @Override - public String getS3AssumeRoleWithoutExternalId() - { - throw new ISE("Not implemented"); - } - - @Override - public String getAzureKey() - { - throw new ISE("Not implemented"); - } - - @Override - public String getHadoopGcsCredentialsPath() - { - throw new ISE("Not implemented"); + String mapped = SETTINGS_MAP.get(key); + key = mapped == null ? key : mapped; + TestConfigs.putProperty(properties, IntegrationTestingConfigProvider.PROPERTY_BASE, key, entry.getValue()); } - @Override - public String getStreamEndpoint() - { - throw new ISE("Not implemented"); - } - - @Override - public String getSchemaRegistryHost() - { - throw new ISE("Not implemented"); - } - - @Override - public boolean isDocker() - { - return ResolvedConfig.this.isDocker(); - } - - @Override - public String getDockerHost() - { - return proxyHost(); + // Add explicit properties + if (this.properties != null) { + properties.putAll(this.properties); } + return properties; } } diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedMetastore.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedMetastore.java index f4112e64714b..4534f30e51dc 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedMetastore.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedMetastore.java @@ -105,18 +105,14 @@ public Map properties() */ public Map toProperties() { - final String base = MetadataStorageConnectorConfig.PROPERTY_BASE + "."; + final String base = MetadataStorageConnectorConfig.PROPERTY_BASE; Map properties = new HashMap<>(); - if (driver != null) { - properties.put("druid.metadata.mysql.driver.driverClassName", driver); - } - properties.put("druid.metadata.storage.type", "mysql"); - properties.put(base + "connectURI", connectURI); - properties.put(base + "user", user); - properties.put(base + "password", password); - if (properties != null) { - properties.put(base + "dbcp", properties); - } + TestConfigs.putProperty(properties, "druid.metadata.mysql.driver.driverClassName", driver); + TestConfigs.putProperty(properties, "druid.metadata.storage.type", "mysql"); + TestConfigs.putProperty(properties, base, "connectURI", connectURI); + TestConfigs.putProperty(properties, base, "user", user); + TestConfigs.putProperty(properties, base, "password", password); + TestConfigs.putProperty(properties, base, "dbcp", this.properties); return properties; } diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/TestConfigs.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/TestConfigs.java index 6eb405cbb8ca..8b909487c23b 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/TestConfigs.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/TestConfigs.java @@ -24,6 +24,8 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; +import java.util.Map; + /** * Utility functions related to test configuration. */ @@ -45,4 +47,20 @@ public static String toYaml(Object obj) return ""; } } + + public static void putProperty(Map properties, String key, Object value) + { + if (value == null) { + return; + } + properties.put(key, value); + } + + public static void putProperty(Map properties, String base, String key, Object value) + { + if (value == null) { + return; + } + properties.put(base + "." + key, value); + } } diff --git a/integration-tests-ex/cases/src/test/resources/config-test/test.yaml b/integration-tests-ex/cases/src/test/resources/config-test/test.yaml index 8a5bb3a6330a..92745caea11e 100644 --- a/integration-tests-ex/cases/src/test/resources/config-test/test.yaml +++ b/integration-tests-ex/cases/src/test/resources/config-test/test.yaml @@ -39,8 +39,18 @@ druid: router: instances: - port: 8888 + properties: druid.test.config.dockerIp: localhost + druid.test.config.cloudRegion: myRegion + my.test.property: howdy + +settings: + cloudBucket: myBucket + cloud_path: myPath + cloud_region: hidden + druid_s3_accessKey: secret + metastoreInit: - sql: | REPLACE INTO druid_segments ( diff --git a/integration-tests-ex/docs/conversion.md b/integration-tests-ex/docs/conversion.md index 737936ea5b5b..b4a8b17d7c8b 100644 --- a/integration-tests-ex/docs/conversion.md +++ b/integration-tests-ex/docs/conversion.md @@ -59,6 +59,8 @@ run in the Docker container. If so, copy those queries into the these setup queries run in the test client, not in each Docker service.) See the former `druid.sh` script to see what SQL was used previously. +### + ## Test Runner ITs require a large amount of setup. All that code is encapsulated in the @@ -156,6 +158,17 @@ folder, copy in the required files, and mount that at `/resources`. Or, if we feel energetic, just change the specs to read their data from `/shared/data`, since `/shared` is already mounted. +### Extensions + +You may see build or other code that passes a list of extensions to an old +integration test. Such configuration represents a misunderstanding of how tests (as +clients) actually work. Tests nave no visibility to a Druid installation directory. +As a result, the "extension" concept does not apply. Instead, tests are run from +Maven, and are subject to the usual Maven process for locating jar files. That +means that any extensions which the test wants to use should be listed as dependencies +in the `pom.xml` file, and will be available on the class path. There is no need for, +or use of, the `druid_extensions_loadList` for tests (or, indeed, for any client.) + ### Starter Test (Optional) An optional step is to ease into the new system by doing a simple diff --git a/integration-tests-ex/docs/quickstart.md b/integration-tests-ex/docs/quickstart.md index 1f29abfc0237..1a3fcb22f7b6 100644 --- a/integration-tests-ex/docs/quickstart.md +++ b/integration-tests-ex/docs/quickstart.md @@ -29,20 +29,6 @@ When first learning the framework, you can try thing out using the `HighAvailability` test. Of the tests converted thus far, it is the one that runs for the shortest time (on the order of a minute or two.) -## Run all Tests - -Run the new ITs with: - -```bash -mvn clean install -P dist,test-image,integration-tests-ex,skip-static-checks \ - -Dmaven.javadoc.skip=true -DskipUTs=true -``` - -This will build Druid, create the distribution tarball, build the -test image and run the two groups of ITs. - -See [this page](maven.md) for details of the Maven build process. - ## Working with Individual Tests To work with tests for development and debugging, you can break the @@ -59,11 +45,24 @@ above all-in-one step into a number of sub-steps. ## Build Druid The integration tests start with a Druid distribution in `distribution/target`, -which you can build using your preferred Maven command line. For example: +which you can build using your preferred Maven command line. Simplest: + +``` +cd $DRUID_DEV +./it.sh dist +``` + +Or, in detail: + +For example: +To make the text a bit simpler, define a variable for the standard settings: ```bash -mvn clean install -P dist,skip-static-checks -Ddruid.console.skip=true \ - -Dmaven.javadoc.skip=true -P skip-tests -T1.0C +export MAVEN_IGNORE=-P skip-static-checks,skip-tests -Dmaven.javadoc.skip=true +``` + +```bash +mvn clean package -P dist $MAVEN_IGNORE -T1.0C ``` Hint: put this into a script somewhere, such as a `~/bin` directory and @@ -90,12 +89,16 @@ You must rebuild the Docker image whenever you rebuild the Druid distribution, since the image includes the distribution. You also will want to rebuild the image if you change the `it-image` project which contains the build scripts. -Assuming `DRUID_DEV` points to your Druid build directory, +```bash +./it.sh image +``` + +In detail, and assuming `DRUID_DEV` points to your Druid build directory, to build the image (only): ```bash -cd $DRUID_DEV/integration-tests-ex/it-image -mvn -P test-image install +cd $DRUID_DEV/integration-tests-ex/image +mvn install -P test-image $MAVEN_IGNORE ``` The above has you `cd` into the project to avoid the need to disable all the @@ -103,6 +106,33 @@ unwanted bits of the Maven build. See [this page](docker.md) for more information. +## Run an IT from the Command Line + +```bash +./it.sh test +``` + +Or, in detail: + +```bash +mvn verify -P docker-tests,IT- -pl :druid-it-cases \ + -P skip-static-checks,skip-tests -Dmaven.javadoc.skip=true -DskipUTs=true +``` + +Where `` is one of the test categories. + +Or + +```bash +cd $DRUID_DEV/integration-tests-ex/cases +mvn verify -P skip-static-checks,docker-tests,IT- \ + -Dmaven.javadoc.skip=true -DskipUTs=true \ + -pl :druid-it-cases +``` + +If the test fails, find the Druid logs in `target/shared/logs` within the +test group project. + ## Start a Cluster The previous generation of tests were organized into TestNG groups. This @@ -113,8 +143,14 @@ So, to start a cluster, you have to pick a group to run. See [this list](maven.md#Modules) for the list of groups. ```bash -cd $DRUID_DEV/integration-tests-ex/ -./cluster.sh up +./it.sh up +``` + +Or, in detail: + +```bash +cd $DRUID_DEV/integration-tests-ex/cases +./cluster.sh up ``` You can use Docker Desktop to monitor the cluster. Give things about 30 seconds @@ -126,19 +162,6 @@ your machine. See [this page](docker.md) for more information. -## Run a Test from the Command Line - -You can run a test group from the command line any number of times against -a test cluster. - -```bash -cd $DRUID_DEV/integration-tests-ex/ -mvn verify -P integration-tests-ex -``` - -If the test fails, find the Druid logs in `target/shared/logs` within the -test group project. - ## Run a Test from an IDE To run an IT in your IDE: @@ -153,9 +176,16 @@ just run them directly. Once you are done with your cluster, you can stop it as follows: + +```bash +./it.sh down +``` + +Or, in detail: + ```bash cd $DRUID_DEV/integration-tests-ex/ -./cluster.sh down +./cluster.sh down ``` ## Clean Up diff --git a/integration-tests-ex/docs/test-config.md b/integration-tests-ex/docs/test-config.md index 5952e22f017c..4a5c7a583f77 100644 --- a/integration-tests-ex/docs/test-config.md +++ b/integration-tests-ex/docs/test-config.md @@ -33,6 +33,11 @@ To create a test, you must supply at least three key components: This section explains the test configuration file which defines the test cluster. +Note that you can create multiple versions of the `docker.yaml` file. For example, +you might want to create one that lists hosts and credentials unique to your +debugging environment. You then use your custom version in place of the standard +one. + ## Cluster Types The integration tests can run in a variety of cluster types, depending @@ -302,15 +307,41 @@ a "service", for example.) Technically, the properties listed here are added to Guice as the one and only `Properties` object. Typically most components work using the default values. Tests are free to change -any of these values for a given test scenario. At present, the properties are -the same for all tests within a Maven module, but we could extend the -`ClusterConfig.Builder` class to allow test-specific settings if needed. +any of these values for a given test scenario. The properties are +the same for all tests within a category. However, they can be changed via environment +variables via the environment variable "binding" mechanism described in +[tests](tests.md). The "JSON configuration" mechanism wants all properties to be strings. YAML will deserialize number-like properties as numbers. To avoid confusion, all properties are converted to strings before being passed to Druid. -When using inheritance, later properties override earlier properties. +When using inheritance, later properties override earlier properties. Environment +variables, if bound, override the defaults specified in this section. Command-line +settings, if provided, have the highest priority. + +A number of test-specific properties are avilable: + +* `druid.test.config.cloudBucket` +* `druid.test.config.cloudPath` + +### `settings` + +The settings section is much like the properties section, and, indeed, are converted +to properties internally. Settings are a fixed set of values that map to the config +files used in the prior tests. Keys include: + +| Setting | Property | Environment Variable | +| `druid_storage_type` | - | - | +| `druid_storage_bucket` | `druid.test.config.cloudBucket` | `DRUID_STORAGE_BUCKET` | +| `druid_storage_baseKey` | `druid.test.config.cloudPath` | `DRUID_STORAGE_BASEKEY` | +| `druid_s3_accessKey` | - | `AWS_ACCESS_KEY_ID` | +| `druid_s3_secretKey` | - | AWS_SECRET_ACCESS_KEY` | + +The above replaces the config file mechanism from the older tests. In general, when a +setting is fixed for a test category, list it in the `docker.yaml` configuration file. +When it varies, pass it in as an environment variable. As a result, the prior configuration +file is not needed. As a result, the prior `override.config.path` property is not supported. ### `metastoreInit` @@ -339,7 +370,7 @@ statements. ### `metastoreInitDelaySec` -``yaml +```yaml metastoreInitDelaySec: ``` @@ -461,3 +492,97 @@ proxyPort: The port number for the service as exposed on the proxy host. Defaults to the same as `port`. You must specify a value if you run multiple instances of the same service. + +## Conversion Guide + +In prior tests, a config file, and the `ConfigFileConfigProvider` class, +provided test configuration. In this version, the file described here +provides configuration. This section presents a mapping from the old to +the new form. + +The `IntegrationTestingConfig` class, which the above class used to provide, +is reimplemented to provide the same information +to tests as before; only the source of the information has changed. + +The new framework assumes that each Druid node is configured either for +plain text or for TLS. (If this assumption is wrong, we'll change the config +file to match.) + +Many of the properties are derived from information in the configuration file. +For example, host names (within Docker) are those given in the `druid` section, +and ports (within the cluster and for the client) are given in `druid..intances.port`, +from which the code computes the URL. + +The old system hard-codes the idea that there are two coordinators or overlords. The +new system allows any number of instances. + +| Method | Old Property | New Format | +| ------ | ------------ | ---------- | +| Router | | | +| `getRouterHost()` | `router_host` | `'router'` | +| `getRouterUrl()` | `router_url` | `'router'` & `instances.port` | +| `getRouterTLSUrl()` | `router_tls_url` | " | +| `getPermissiveRouterUrl()` | `router_permissive_url` | " | +| `getPermissiveRouterTLSUrl()` | `router_permissive_tls_url` | " | +| `getNoClientAuthRouterUrl()` | `router_no_client_auth_url` | " | +| `getNoClientAuthRouterTLSUrl()` | `router_no_client_auth_tls_url` | " | +| `getCustomCertCheckRouterUrl()` | | " | +| `getCustomCertCheckRouterTLSUrl()` | | " | +| Broker | | | +| `getBrokerHost()` | `broker_host` | `'broker'` | +| `getBrokerUrl()` | `broker_url` | `'broker'` & `instances.port` | +| `getBrokerTLSUrl()` | `broker_tls_url` | " | +| Coordinator | | | +| `getCoordinatorHost()` | `coordinator_host` | `'coordinator'` + `tag` | +| `getCoordinatorTwoHost()` | `coordinator_two_host` | " | +| `getCoordinatorUrl()` | `coordinator_url` | host & `instances.port` | +| `getCoordinatorTLSUrl()` | `coordinator_tls_url` | " | +| `getCoordinatorTwoUrl()` | `coordinator_two_url` | " | +| `getCoordinatorTwoTLSUrl()` | `coordinator_two_tls_url` | " | +| Overlord | | | +| `getOverlordUrl()` | ? | `'overlord'` + `tag` | +| `getOverlordTwoHost()` | `overlord_two_host` | " | +| `getOverlordTwoUrl()` | `overlord_two_url` | host & `instances.port` | +| `getOverlordTLSUrl()` | ? | " | +| `getOverlordTwoTLSUrl()` | `overlord_two_tls_url` | " | +| Overlord | | | +| `getHistoricalHost()` | `historical_host` | `historical'` | +| `getHistoricalUrl()` | `historical_url` | `'historical'` & `instances.port` | +| `getHistoricalTLSUrl()` | `historical_tls_url` | " | +| Overlord | | | +| `getMiddleManagerHost()` | `middlemanager_host` | `'middlemanager'` | +| Dependencies | | | +| `getZookeeperHosts()` | `zookeeper_hosts` | `'zk'` | +| `getKafkaHost()` | `kafka_host` | '`kafka`' | +| `getSchemaRegistryHost()` | `schema_registry_host` | ? | +| `getProperty()` | From config file | From `settings` | +| `getProperties()` | " | " | +| `getUsername()` | `username` | Setting | +| `getPassword()` | `password` | Setting | +| `getCloudBucket()` | `cloud_bucket` | Setting | +| `getCloudPath()` | `cloud_path` | Setting | +| `getCloudRegion()` | `cloud_region` | Setting | +| `getS3AssumeRoleWithExternalId()` | `s3_assume_role_with_external_id` | Setting | +| `getS3AssumeRoleExternalId()` | `s3_assume_role_external_id` | Setting | +| `getS3AssumeRoleWithoutExternalId()` | `s3_assume_role_without_external_id` | Setting | +| `getAzureKey()` | `azureKey` | Setting | +| `getHadoopGcsCredentialsPath()` | `hadoopGcsCredentialsPath` | Setting | +| `getStreamEndpoint()` | `stream_endpoint` | Setting | +| `manageKafkaTopic()` | ? | ? | +| `getExtraDatasourceNameSuffix()` | ? | ? | + +Pre-defined environment bindings: + +| Setting | Env. Var. | +| `cloudBucket` | `DRUID_CLOUD_BUCKET` | +| `cloudPath` | `DRUID_CLOUD_PATH` | +| `s3AccessKey` | `AWS_ACCESS_KEY_ID` | +| `s3SecretKey` | `AWS_SECRET_ACCESS_KEY` | +| `azureContainer` | `AZURE_CONTAINER` | +| `azureAccount` | `AZURE_ACCOUNT` | +| `azureKey` | `AZURE_KEY` | +| `googleBucket` | `GOOGLE_BUCKET` | +| `googlePrefix` | `GOOGLE_PREFIX` | + +Others can be added in `Initializer.Builder`. + diff --git a/integration-tests/src/main/java/org/apache/druid/testing/ConfigFileConfigProvider.java b/integration-tests/src/main/java/org/apache/druid/testing/ConfigFileConfigProvider.java index e39c630cf287..8e3c5a83e8d1 100644 --- a/integration-tests/src/main/java/org/apache/druid/testing/ConfigFileConfigProvider.java +++ b/integration-tests/src/main/java/org/apache/druid/testing/ConfigFileConfigProvider.java @@ -221,7 +221,7 @@ private void loadProperties(String configFile) overlordTwoTLSUrl = StringUtils.format("https://%s:%s", overlordTwoHost, props.get("overlord_two_tls_port")); } } - + middleManagerHost = props.get("middlemanager_host"); zookeeperHosts = props.get("zookeeper_hosts"); @@ -259,7 +259,6 @@ public IntegrationTestingConfig get() { return new IntegrationTestingConfig() { - @Override public String getCoordinatorUrl() { diff --git a/integration-tests/src/main/java/org/apache/druid/testing/IntegrationTestingConfigProvider.java b/integration-tests/src/main/java/org/apache/druid/testing/IntegrationTestingConfigProvider.java index 3ce1dd21f68c..a815391fb220 100644 --- a/integration-tests/src/main/java/org/apache/druid/testing/IntegrationTestingConfigProvider.java +++ b/integration-tests/src/main/java/org/apache/druid/testing/IntegrationTestingConfigProvider.java @@ -30,4 +30,5 @@ }) public interface IntegrationTestingConfigProvider extends Provider { + String PROPERTY_BASE = "druid.test.config"; } diff --git a/integration-tests/src/main/java/org/apache/druid/testing/guice/DruidTestModule.java b/integration-tests/src/main/java/org/apache/druid/testing/guice/DruidTestModule.java index 1e239ae57dbe..b0bef0340835 100644 --- a/integration-tests/src/main/java/org/apache/druid/testing/guice/DruidTestModule.java +++ b/integration-tests/src/main/java/org/apache/druid/testing/guice/DruidTestModule.java @@ -51,7 +51,7 @@ public void configure(Binder binder) binder.bind(IntegrationTestingConfig.class) .toProvider(IntegrationTestingConfigProvider.class) .in(ManageLifecycle.class); - JsonConfigProvider.bind(binder, "druid.test.config", IntegrationTestingConfigProvider.class); + JsonConfigProvider.bind(binder, IntegrationTestingConfigProvider.PROPERTY_BASE, IntegrationTestingConfigProvider.class); binder.bind(CuratorConfig.class).to(IntegrationTestingCuratorConfig.class); diff --git a/it.sh b/it.sh new file mode 100755 index 000000000000..39d6daf11f04 --- /dev/null +++ b/it.sh @@ -0,0 +1,77 @@ +#! /bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#-------------------------------------------------------------------- + +# Utility script for running the new integration tests, since the Maven +# commands are unwieldy. + +export DRUID_DEV=$(cd $(dirname $0) && pwd) + +function usage { + echo "Usage: $0 cmd [category]" + echo " dist - build the Druid distribution" + echo " image - build the test image" + echo " up - start the cluster for category" + echo " down - stop the cluster for category" + echo " test - start the cluster, run the test for category, and stop the cluster" +} + +CMD=$1 +shift +MAVEN_IGNORE="-P skip-static-checks,skip-tests -Dmaven.javadoc.skip=true" + +case $CMD in + "dist") + mvn clean package -P dist $MAVEN_IGNORE -T1.0C + ;; + "image") + cd $DRUID_DEV/integration-tests-ex/image + mvn install -P test-image $MAVEN_IGNORE + ;; + "up") + if [ -z "$1" ]; then + usage + exit 1 + fi + cd $DRUID_DEV/integration-tests-ex/cases + ./cluster.sh $1 up + ;; + "down") + if [ -z "$1" ]; then + usage + exit 1 + fi + cd $DRUID_DEV/integration-tests-ex/cases + ./cluster.sh $1 down + ;; + "test") + if [ -z "$1" ]; then + usage + exit 1 + fi + cd $DRUID_DEV/integration-tests-ex/cases + mvn verify -P skip-static-checks,docker-tests,IT-$1 \ + -Dmaven.javadoc.skip=true -DskipUTs=true \ + -pl :druid-it-cases + ;; + "help") + usage + ;; + *) + usage + exit -1 +esac diff --git a/pom.xml b/pom.xml index c161d7dc2039..80c7f6cbe802 100644 --- a/pom.xml +++ b/pom.xml @@ -1875,7 +1875,7 @@ target/** licenses/** **/test/resources/** - **/data/data/** + integration-tests-ex/cases/resources/data/** **/derby.log **/jvm.config **/*.avsc From e034f60d0c00a5304e6b468d56144c013da5958f Mon Sep 17 00:00:00 2001 From: Paul Rogers Date: Wed, 17 Aug 2022 19:11:59 -0700 Subject: [PATCH 02/11] Final revsions Improved env var-based configuration Test createion guide Enable new ITs in travis Parameterized tests --- .travis.yml | 61 ++--- .../java/org/apache/druid/guice/PolyBind.java | 4 +- .../org/apache/druid/guice/PolyBindTest.java | 8 +- integration-tests-ex/README.md | 1 + integration-tests-ex/cases/.gitignore | 1 + integration-tests-ex/cases/cluster.sh | 198 ++++++++------ .../cases/cluster/Common/druid.yaml | 7 + .../Common/environment-configs/common.env | 21 +- integration-tests-ex/cases/pom.xml | 108 ++++++-- .../druid/testsEx/categories/InputFormat.java | 30 --- .../druid/testsEx/config/ClusterConfig.java | 7 +- .../testsEx/config/ClusterConfigTest.java | 24 +- .../druid/testsEx/config/DruidTestRunner.java | 16 +- .../druid/testsEx/config/Initializer.java | 18 +- .../druid/testsEx/config/ResolvedConfig.java | 87 ++++++- .../testsEx/config/ResolvedMetastore.java | 2 +- .../ITLocalInputSourceAllInputFormatTest.java | 103 -------- .../cluster/AzureDeepStorage/docker.yaml | 58 +++++ integration-tests-ex/check-results.sh | 2 + integration-tests-ex/docs/compose.md | 92 ++++++- integration-tests-ex/docs/docker.md | 38 +++ integration-tests-ex/docs/guide.md | 241 ++++++++++++++++++ integration-tests-ex/docs/next-steps.md | 2 +- integration-tests-ex/docs/scripts.md | 14 +- integration-tests-ex/docs/tests.md | 25 +- integration-tests-ex/image/docker/launch.sh | 30 ++- integration-tests-ex/image/pom.xml | 8 +- it.sh | 89 +++++-- pom.xml | 14 +- .../druid/guice/DruidInjectorBuilder.java | 2 - 30 files changed, 963 insertions(+), 348 deletions(-) create mode 100644 integration-tests-ex/cases/.gitignore delete mode 100644 integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/InputFormat.java delete mode 100644 integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/ITLocalInputSourceAllInputFormatTest.java create mode 100644 integration-tests-ex/cases/src/test/resources/cluster/AzureDeepStorage/docker.yaml create mode 100644 integration-tests-ex/docs/guide.md diff --git a/.travis.yml b/.travis.yml index e2351a6ee2f0..fceb60785e52 100644 --- a/.travis.yml +++ b/.travis.yml @@ -73,19 +73,6 @@ jobs: stage: Tests - phase 1 script: ${MVN} animal-sniffer:check --fail-at-end - # Experimental run of the revised ITs. Done early to debug issues - # Disabled for now. Integrating this into the build will come in a later PR. - - name: "experimental docker tests" - stage: Tests - phase 1 - # Uses the install defined above. Then, builds the test tools and docker image, - # and run the various IT tests. If tests fail, echos log lines of any of - # the Druid services that did not exit normally. - # Run though install to ensure the test tools are installed, and the docker - # image is built. The tests only need verify. - script: ${MVN} install -P dist,test-image,docker-tests,IT-HighAvailability -rf :distribution ${MAVEN_SKIP} -DskipUTs=true - #after_failure: - # - docker-tests/check-results.sh - - name: "checkstyle" script: ${MVN} checkstyle:checkstyle --fail-at-end @@ -470,9 +457,9 @@ jobs: docker exec -it druid-$v sh -c 'dmesg | tail -3' ; done - - <<: *integration_batch_index - name: "(Compile=openjdk8, Run=openjdk8) batch index integration test with Indexer" - env: TESTNG_GROUPS='-Dgroups=batch-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer' + #- <<: *integration_batch_index + # name: "(Compile=openjdk8, Run=openjdk8) batch index integration test with Indexer" + # env: TESTNG_GROUPS='-Dgroups=batch-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer' - &integration_input_format name: "(Compile=openjdk8, Run=openjdk8) input format integration test" @@ -679,16 +666,35 @@ jobs: name: "(Compile=openjdk8, Run=openjdk8) other integration tests with Indexer" env: TESTNG_GROUPS='-DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer' - - <<: *integration_tests - name: "(Compile=openjdk8, Run=openjdk8) leadership and high availability integration tests" - jdk: openjdk8 - env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' OVERRIDE_CONFIG_PATH='./environment-configs/test-groups/prepopulated-data' + #- <<: *integration_tests + # name: "(Compile=openjdk8, Run=openjdk8) leadership and high availability integration tests" + # jdk: openjdk8 + # env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' OVERRIDE_CONFIG_PATH='./environment-configs/test-groups/prepopulated-data' - <<: *integration_query name: "(Compile=openjdk8, Run=openjdk8) query integration test (mariaDB)" jdk: openjdk8 env: TESTNG_GROUPS='-Dgroups=query' USE_INDEXER='middleManager' MYSQL_DRIVER_CLASSNAME='org.mariadb.jdbc.Driver' OVERRIDE_CONFIG_PATH='./environment-configs/test-groups/prepopulated-data' + - &integration_tests_ex + stage: Tests - phase 2 + jdk: openjdk8 + services: *integration_test_services + env: JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' + + # Revised ITs. + - <<: *integration_tests_ex + name: "(Compile=openjdk8, Run=openjdk8) leadership and high availability integration tests (new)" + # Uses the install defined above. Then, builds the test tools and docker image, + # and runs one IT. If tests fail, echos log lines of any of + # the Druid services that did not exit normally. + script: it.sh travis HighAvailability + + - <<: *integration_tests_ex + name: "(Compile=openjdk8, Run=openjdk8) batch index integration test with Indexer (new)" + env: JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer' + script: it.sh travis BatchIndex + # END - Integration tests for Compile with Java 8 and Run with Java 8 # START - Integration tests for Compile with Java 8 and Run with Java 11 @@ -769,21 +775,6 @@ jobs: # END - Integration tests for Compile with Java 8 and Run with Java 11 - # BEGIN - Revised integration tests - - # Experimental build of the revised integration test Docker image. - # Actual tests will come later. - - name: "experimental docker tests" - stage: Tests - phase 2 - # Uses the install defined above. Then, builds the test tools and docker image, - # and run the various IT tests. If tests fail, echos log lines of any of - # the Druid services that did not exit normally. - # Run though install to ensure the test tools are installed, and the docker - # image is built. The tests only need verify. - script: ${MVN} install -P dist,test-image -rf :distribution ${MAVEN_SKIP} -DskipUTs=true - - # END - Revised integration tests - - &integration_batch_index_k8s name: "(Compile=openjdk8, Run=openjdk8, Cluster Build On K8s) ITNestedQueryPushDownTest integration test" stage: Tests - phase 2 diff --git a/core/src/main/java/org/apache/druid/guice/PolyBind.java b/core/src/main/java/org/apache/druid/guice/PolyBind.java index 592d578c4b75..19931a6b63e8 100644 --- a/core/src/main/java/org/apache/druid/guice/PolyBind.java +++ b/core/src/main/java/org/apache/druid/guice/PolyBind.java @@ -43,8 +43,8 @@ * Provides the ability to create "polymorphic" bindings where the polymorphism is actually just making a decision * based on a value in Properties. *

- * The workflow is that you first create a choice by calling {@link #createChoice()}. Then you create options using - * the binder returned by the {@link #optionBinder()} method. Multiple different modules can call + * The workflow is that you first create a choice by calling {@code createChoice()}. Then you create options using + * the binder returned by the {@code optionBinder()} method. Multiple different modules can call * {@code optionBinder()} and all options will be reflected at injection time as long as equivalent interface * {@code Key} objects are passed into the various methods. */ diff --git a/core/src/test/java/org/apache/druid/guice/PolyBindTest.java b/core/src/test/java/org/apache/druid/guice/PolyBindTest.java index d6a50d513960..372d428458d6 100644 --- a/core/src/test/java/org/apache/druid/guice/PolyBindTest.java +++ b/core/src/test/java/org/apache/druid/guice/PolyBindTest.java @@ -112,7 +112,7 @@ public void configure(Binder binder) } catch (Exception e) { Assert.assertTrue(e instanceof ProvisionException); - Assert.assertTrue(e.getMessage().contains("Unknown provider[c] of Key[type=org.apache.druid.guice.PolyBindTest$Gogo")); + Assert.assertTrue(e.getMessage().contains("Unknown provider [c] of Key[type=org.apache.druid.guice.PolyBindTest$Gogo")); } try { Assert.assertEquals("B", injector.getInstance(Key.get(Gogo.class, Names.named("reverse"))).go()); @@ -120,9 +120,9 @@ public void configure(Binder binder) } catch (Exception e) { Assert.assertTrue(e instanceof ProvisionException); - Assert.assertTrue(e.getMessage().contains("Unknown provider[c] of Key[type=org.apache.druid.guice.PolyBindTest$Gogo")); + Assert.assertTrue(e.getMessage().contains("Unknown provider [c] of Key[type=org.apache.druid.guice.PolyBindTest$Gogo")); } - + // test default property value Assert.assertEquals("B", injector.getInstance(GogoSally.class).go()); props.setProperty("sally", "a"); @@ -136,7 +136,7 @@ public void configure(Binder binder) } catch (Exception e) { Assert.assertTrue(e instanceof ProvisionException); - Assert.assertTrue(e.getMessage().contains("Unknown provider[c] of Key[type=org.apache.druid.guice.PolyBindTest$GogoSally")); + Assert.assertTrue(e.getMessage().contains("Unknown provider [c] of Key[type=org.apache.druid.guice.PolyBindTest$GogoSally")); } } diff --git a/integration-tests-ex/README.md b/integration-tests-ex/README.md index a11c97414973..9c29ec101fe9 100644 --- a/integration-tests-ex/README.md +++ b/integration-tests-ex/README.md @@ -81,6 +81,7 @@ test as a JUnit test. * [Goals](#Goals) * [Quickstart](docs/quickstart.md) +* [Create a new test](docs/guide.md) * [Maven configuration](docs/maven.md) * [Travis integration](docs/travis.md) * [Docker image](docs/docker.md) diff --git a/integration-tests-ex/cases/.gitignore b/integration-tests-ex/cases/.gitignore new file mode 100644 index 000000000000..ae3c1726048c --- /dev/null +++ b/integration-tests-ex/cases/.gitignore @@ -0,0 +1 @@ +/bin/ diff --git a/integration-tests-ex/cases/cluster.sh b/integration-tests-ex/cases/cluster.sh index 8f4dcde123e8..0b19b478fcb8 100755 --- a/integration-tests-ex/cases/cluster.sh +++ b/integration-tests-ex/cases/cluster.sh @@ -26,67 +26,106 @@ export MODULE_DIR=$(cd $(dirname $0) && pwd) -USAGE="Usage: $0 category [-h|help|up|down|status|compose-cmd]" +function usage { + cat <&2 + usage 1>&2 exit 1 fi -# The untranslated category is used for the local name of the -# shared folder. -CATEGORY=$1 +CMD=$1 shift -# DRUID_INTEGRATION_TEST_GROUP is used in -# docker-compose files and here. Despite the name, it is the -# name of the cluster configuration we want to run, not the -# test category. Multiple categories an map to the same cluster -# definition. - -# Map from category name to shared cluster definition name. -# Add an entry here if you create a new category that shares -# a definition. -case $CATEGORY in - "InputFormat") - export DRUID_INTEGRATION_TEST_GROUP=BatchIndex - ;; - *) - export DRUID_INTEGRATION_TEST_GROUP=$CATEGORY - ;; -esac - -CLUSTER_DIR=$MODULE_DIR/cluster/$DRUID_INTEGRATION_TEST_GROUP -if [ ! -d $CLUSTER_DIR ]; then - echo "Cluster directory $CLUSTER_DIR does not exist." 1>&2 - echo "$USAGE" 1>&2 - exit 1 -fi +function category { + if [ $# -eq 0 ]; then + usage 1>&2 + exit 1 + fi + export CATEGORY=$1 + + # All commands need env vars + ENV_FILE=$MODULE_DIR/../image/target/env.sh + if [ ! -f $ENV_FILE ]; then + echo "Please build the Docker test image before testing" 1>&2 + exit 1 + fi + + source $ENV_FILE + # The untranslated category is used for the local name of the + # shared folder. + + # DRUID_INTEGRATION_TEST_GROUP is used in + # docker-compose files and here. Despite the name, it is the + # name of the cluster configuration we want to run, not the + # test category. Multiple categories an map to the same cluster + # definition. + + # Map from category name to shared cluster definition name. + # Add an entry here if you create a new category that shares + # a definition. + case $CATEGORY in + "InputFormat") + export DRUID_INTEGRATION_TEST_GROUP=BatchIndex + ;; + *) + export DRUID_INTEGRATION_TEST_GROUP=$CATEGORY + ;; + esac + + export CLUSTER_DIR=$MODULE_DIR/cluster/$DRUID_INTEGRATION_TEST_GROUP + if [ ! -d $CLUSTER_DIR ]; then + echo "Cluster directory $CLUSTER_DIR does not exist." 1>&2 + echo "$USAGE" 1>&2 + exit 1 + fi + + export TARGET_DIR=$MODULE_DIR/target + export SHARED_DIR=$TARGET_DIR/$CATEGORY + export ENV_FILE="$TARGET_DIR/${CATEGORY}.env" +} -# 'up' command by default, else whatever is the argument -CMD='up' -if [ $# -ge 1 ]; then - CMD=$1 -fi +function build_override { -# All commands need env vars -ENV_FILE=$MODULE_DIR/../image/target/env.sh -if [ ! -f $ENV_FILE ]; then - echo "Please build the Docker test image before testing" 1>&2 - exit 1 -fi + mkdir -p target + rm -f "$ENV_FILE" + touch "$ENV_FILE" -source $ENV_FILE + # User-local settings? + LOCAL_ENV="$HOME/druid-it/${CATEGORY}.env" + if [ -f "$LOCAL_ENV" ]; then + cat "$LOCAL_ENV" >> "$ENV_FILE" + fi -export TARGET_DIR=$MODULE_DIR/target -export SHARED_DIR=$TARGET_DIR/$CATEGORY + # Provided override file + if [ -n "$OVERRIDE_ENV" ]; then + if [ ! -f "$OVERRIDE_ENV" ]; then + echo "Environment override file (OVERRIDE_ENV) not found: $OVERRIDE_ENV" 1>&2 + exit 1 + fi + cat "$OVERRIDE_ENV" >> "$ENV_FILE" + fi -# Used in docker-compose files -export OVERRIDE_ENV=${OVERRIDE_ENV:-} + # Add all environment variables of the form druid_* + env | grep "^druid_" >> "$ENV_FILE" -# Print environment for debugging -#env + # Reuse the OVERRIDE_ENV variable to pass the full list to Docker compose + export OVERRIDE_ENV="$ENV_FILE" +} # Dump lots of information to debug Docker failures when run inside # of a build environment where we can't inspect Docker directly. @@ -104,46 +143,59 @@ function show_status { echo "====================================" } +function build_shared_dir { + mkdir -p $SHARED_DIR + # Must start with an empty DB to keep MySQL happy + rm -rf $SHARED_DIR/db + mkdir -p $SHARED_DIR/logs + mkdir -p $SHARED_DIR/tasklogs + mkdir -p $SHARED_DIR/db + mkdir -p $SHARED_DIR/kafka + mkdir -p $SHARED_DIR/resources + cp $MODULE_DIR/assets/log4j2.xml $SHARED_DIR/resources + # Permissions in some build setups are screwed up. See above. The user + # which runs Docker does not have permission to write into the /shared + # directory. Force ownership to allow writing. + chmod -R a+rwx $SHARED_DIR +} + +# Print environment for debugging +#env + case $CMD in - "-h" | "-?") - echo "$USAGE" + "-h" ) + usage ;; - "help") - echo "$USAGE" + "help" ) + usage docker-compose help ;; - "up") + "up" ) + category $* echo "Starting cluster $DRUID_INTEGRATION_TEST_GROUP" - mkdir -p $SHARED_DIR - # Must start with an empty DB to keep MySQL happy - rm -rf $SHARED_DIR/db - mkdir -p $SHARED_DIR/logs - mkdir -p $SHARED_DIR/tasklogs - mkdir -p $SHARED_DIR/db - mkdir -p $SHARED_DIR/kafka - mkdir -p $SHARED_DIR/resources - cp $MODULE_DIR/assets/log4j2.xml $SHARED_DIR/resources - # Permissions in some build setups are screwed up. See above. The user - # which runs Docker does not have permission to write into the /shared - # directory. Force ownership to allow writing. - chmod -R a+rwx $SHARED_DIR + build_override + build_shared_dir cd $CLUSTER_DIR docker-compose up -d # Enable the following for debugging - #show_status + show_status ;; - "status") + "status" ) + category $* cd $CLUSTER_DIR show_status ;; - "down") + "down" ) + category $* # Enable the following for debugging - #show_status + show_status cd $CLUSTER_DIR - docker-compose $CMD + echo OVERRIDE_ENV="$ENV_FILE" docker-compose $CMD + OVERRIDE_ENV="$ENV_FILE" docker-compose $CMD ;; - "*") + "*" ) + category $* cd $CLUSTER_DIR - docker-compose $CMD + OVERRIDE_ENV="$ENV_FILE" docker-compose $CMD ;; esac diff --git a/integration-tests-ex/cases/cluster/Common/druid.yaml b/integration-tests-ex/cases/cluster/Common/druid.yaml index b8483c813172..bd5caad2232b 100644 --- a/integration-tests-ex/cases/cluster/Common/druid.yaml +++ b/integration-tests-ex/cases/cluster/Common/druid.yaml @@ -60,6 +60,7 @@ services: env_file: - environment-configs/common.env - environment-configs/overlord.env + - ${OVERRIDE_ENV} coordinator: image: ${DRUID_IT_IMAGE_NAME} @@ -76,6 +77,7 @@ services: env_file: - environment-configs/common.env - environment-configs/coordinator.env + - ${OVERRIDE_ENV} historical: image: ${DRUID_IT_IMAGE_NAME} @@ -92,6 +94,7 @@ services: env_file: - environment-configs/common.env - environment-configs/historical.env + - ${OVERRIDE_ENV} middlemanager: image: ${DRUID_IT_IMAGE_NAME} @@ -120,6 +123,7 @@ services: env_file: - environment-configs/common.env - environment-configs/middlemanager.env + - ${OVERRIDE_ENV} indexer: image: ${DRUID_IT_IMAGE_NAME} @@ -136,6 +140,7 @@ services: env_file: - environment-configs/common.env - environment-configs/indexer.env + - ${OVERRIDE_ENV} broker: image: ${DRUID_IT_IMAGE_NAME} @@ -152,6 +157,7 @@ services: env_file: - environment-configs/common.env - environment-configs/broker.env + - ${OVERRIDE_ENV} router: image: ${DRUID_IT_IMAGE_NAME} @@ -168,3 +174,4 @@ services: env_file: - environment-configs/common.env - environment-configs/router.env + - ${OVERRIDE_ENV} diff --git a/integration-tests-ex/cases/cluster/Common/environment-configs/common.env b/integration-tests-ex/cases/cluster/Common/environment-configs/common.env index ad6f9e7d5fc5..350a1f5b7923 100644 --- a/integration-tests-ex/cases/cluster/Common/environment-configs/common.env +++ b/integration-tests-ex/cases/cluster/Common/environment-configs/common.env @@ -41,13 +41,17 @@ DRUID_INSTANCE= # Hostname # druid.host is set on each host by the launch script -# Extensions specified in the load list will be loaded by Druid -# Extensions are installed as part of Druid. The prefix -# must be the same as the value of $DRUID_HOME in the container. -# Optional as this is the default? -#druid_extensions_directory=/usr/local/druid/extensions -#druid_extensions_loadList=["mysql-metadata-storage","druid-basic-security","simple-client-sslcontext","it-tools","druid-lookups-cached-global","druid-histogram","druid-datasketches","druid-parquet-extensions","druid-avro-extensions","druid-protobuf-extensions","druid-orc-extensions","druid-kafka-indexing-service","druid-s3-extensions"] -druid_extensions_loadList=["mysql-metadata-storage","it-tools","druid-lookups-cached-global","druid-histogram","druid-datasketches","druid-parquet-extensions","druid-avro-extensions","druid-protobuf-extensions","druid-orc-extensions","druid-kafka-indexing-service","druid-s3-extensions"] +# Extensions specified in the load list will be loaded by Druid at runtime. +# The extension jars must be installed as part of Druid, or via the image +# build script. +# +# The launch script creates druid_extensions_loadList by combining two test-specific +# variables: druid_standard_loadList defined here, and druid_test_loadList, defined +# in a docker-compose.yaml file, for any test-specific extensions. +# See compose.md for more details. +druid_standard_loadList=mysql-metadata-storage,it-tools,druid-lookups-cached-global,druid-histogram,druid-datasketches,druid-parquet-extensions,druid-avro-extensions,druid-protobuf-extensions,druid-orc-extensions,druid-kafka-indexing-service,druid-s3-extensions + +# Location of Hadoop dependencies provided at runtime in the shared directory. druid_extensions_hadoopDependenciesDir=/shared/hadoop-dependencies # Logging @@ -108,3 +112,6 @@ druid_coordinator_period_metadataStoreManagementPeriod=PT10S # Testing the legacy config from https://github.com/apache/druid/pull/10267 # Can remove this when the flag is no longer needed druid_indexer_task_ignoreTimestampSpecForDruidInputSource=true + + +# TODO: Pass this from the test (AzureDeepStorage) diff --git a/integration-tests-ex/cases/pom.xml b/integration-tests-ex/cases/pom.xml index f1cb1b282daf..5c34af881d68 100644 --- a/integration-tests-ex/cases/pom.xml +++ b/integration-tests-ex/cases/pom.xml @@ -74,10 +74,19 @@ druid-services ${project.parent.version} + + org.apache.druid + druid-indexing-service + ${project.parent.version} + com.google.inject guice + + com.google.inject.extensions + guice-multibindings + org.apache.curator curator-framework @@ -159,6 +168,24 @@ mysql-metadata-storage ${project.parent.version} + + org.apache.druid.extensions + druid-azure-extensions + ${project.parent.version} + provided + + + org.apache.druid.extensions + druid-hdfs-storage + ${project.parent.version} + provided + + + com.amazonaws + aws-java-sdk-bundle + + + org.apache.commons commons-lang3 @@ -174,9 +201,18 @@ test - junit - junit - test + com.google.code.findbugs + jsr305 + + + junit + junit + test + + + pl.pragmatists + JUnitParams + test @@ -192,37 +228,63 @@ + + org.apache.maven.plugins + maven-dependency-plugin + + + + org.glassfish.hk2.external:jakarta.inject + + + + mysql:mysql-connector-java:jar + + + - - IT-HighAvailability + + IT-HighAvailability + + false + + + HighAvailability + + + + IT-BatchIndex false - - HighAvailability - - - - IT-BatchIndex + + BatchIndex + + + + IT-InputFormat false - - BatchIndex - - - - IT-InputFormat + + InputFormat + + + + IT-AzureDeepStorage false - - InputFormat - - + + AzureDeepStorage + + docker-tests @@ -281,8 +343,8 @@ bash cluster.sh - ${it.category} up + ${it.category} @@ -298,8 +360,8 @@ bash cluster.sh - ${it.category} down + ${it.category} diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/InputFormat.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/InputFormat.java deleted file mode 100644 index 6e8e182b0c15..000000000000 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/InputFormat.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.testsEx.categories; - -import org.apache.druid.testsEx.config.Cluster; - -/** - * Input format category. Uses the same setup as {@link BatchIndex}. - */ -@Cluster(BatchIndex.class) -public class InputFormat -{ -} diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ClusterConfig.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ClusterConfig.java index a98072196a4b..363f7648b07f 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ClusterConfig.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ClusterConfig.java @@ -119,6 +119,9 @@ public ClusterConfig(ClusterConfig from) if (from.properties != null) { this.properties = new HashMap<>(from.properties); } + if (from.settings != null) { + this.settings = new HashMap<>(from.settings); + } if (from.metastoreInit != null) { this.metastoreInit = new ArrayList<>(from.metastoreInit); } @@ -159,9 +162,9 @@ public static ClusterConfig loadFromResource(String resource) } } - public ResolvedConfig resolve() + public ResolvedConfig resolve(String clusterName) { - return new ResolvedConfig(resolveIncludes()); + return new ResolvedConfig(clusterName, resolveIncludes()); } public ClusterConfig resolveIncludes() diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ClusterConfigTest.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ClusterConfigTest.java index 912f8d91c0f3..1531edff0fd2 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ClusterConfigTest.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ClusterConfigTest.java @@ -24,6 +24,12 @@ import org.apache.druid.testsEx.config.ResolvedService.ResolvedZk; import org.junit.Test; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.Properties; @@ -37,13 +43,13 @@ public class ClusterConfigTest { @Test - public void testYaml() + public void testYaml() throws FileNotFoundException { ClusterConfig config = ClusterConfig.loadFromResource("/config-test/test.yaml"); // Uncomment this line to see the full config with includes resolved. //System.out.println(config.resolveIncludes()); - ResolvedConfig resolved = config.resolve(); + ResolvedConfig resolved = config.resolve("Test"); assertEquals(ClusterType.docker, resolved.type()); assertEquals(ResolvedConfig.DEFAULT_READY_TIMEOUT_SEC, resolved.readyTimeoutSec()); assertEquals(ResolvedConfig.DEFAULT_READY_POLL_MS, resolved.readyPollMs()); @@ -79,6 +85,16 @@ public void testYaml() assertEquals("http://localhost:8888", service.clientUrl()); assertEquals("http://localhost:8888", resolved.routerUrl()); + File userEnv = new File( + new File( + System.getProperty("user.home"), + "druid-it"), + "Test.env"); + try (PrintWriter out = new PrintWriter(new OutputStreamWriter(new FileOutputStream(userEnv), StandardCharsets.UTF_8))) { + out.println("druid_user_var=user"); + } + + System.setProperty("druid_sys_prop", "sys"); Map props = resolved.toProperties(); // Added from ZK section assertEquals("localhost:2181", props.get("druid.zk.service.zkHosts")); @@ -90,6 +106,10 @@ public void testYaml() assertEquals("secret", props.get("druid.test.config.s3AccessKey")); // From settings, overridden in properties assertEquals("myRegion", props.get("druid.test.config.cloudRegion")); + // System property + assertEquals("sys", props.get("druid.test.config.sys_prop")); + // From user override + assertEquals("user", props.get("druid.test.config.user_var")); // Test plumbing through the test config Properties properties = new Properties(); diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/DruidTestRunner.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/DruidTestRunner.java index 0d44eb418f46..5e65ec7f9d4e 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/DruidTestRunner.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/DruidTestRunner.java @@ -19,9 +19,9 @@ package org.apache.druid.testsEx.config; +import junitparams.JUnitParamsRunner; import org.apache.druid.java.util.common.UOE; import org.junit.experimental.categories.Category; -import org.junit.runners.BlockJUnit4ClassRunner; import org.junit.runners.model.InitializationError; import org.junit.runners.model.Statement; @@ -42,8 +42,10 @@ * test members before starting the lifecycle, so that the injection creates * a reference, which creates the object, which registers it in the lifecycle. We * should fix this issue. Until then, the awkwardness is hidden in this test runner. + *

+ * Extends the parameterize test runner, so your Druid ITs can also use parameters. */ -public class DruidTestRunner extends BlockJUnit4ClassRunner +public class DruidTestRunner extends JUnitParamsRunner { private class CloseInitializer extends Statement { @@ -87,7 +89,7 @@ protected Object createTest() throws Exception private Initializer buildInitializer(Object test) { Class testClass = test.getClass(); - Category annotations[] = testClass.getAnnotationsByType(Category.class); + Category[] annotations = testClass.getAnnotationsByType(Category.class); if (annotations.length == 0) { throw new UOE( "Class % must have a @Category annotation", @@ -100,7 +102,7 @@ private Initializer buildInitializer(Object test) testClass.getSimpleName() ); } - Class categories[] = annotations[0].value(); + Class[] categories = annotations[0].value(); if (categories.length == 0) { throw new UOE( "Class % must have a @Category value", @@ -148,7 +150,7 @@ private Initializer buildInitializer(Object test) */ private Class category(Class testClass) { - Category annotations[] = testClass.getAnnotationsByType(Category.class); + Category[] annotations = testClass.getAnnotationsByType(Category.class); if (annotations.length == 0) { throw new UOE( "Class % must have a @Category annotation", @@ -161,7 +163,7 @@ private Class category(Class testClass) testClass.getSimpleName() ); } - Class categories[] = annotations[0].value(); + Class[] categories = annotations[0].value(); if (categories.length == 0) { throw new UOE( "Class % must have a @Category value", @@ -184,7 +186,7 @@ private Class category(Class testClass) private String inferCluster(Class category) { String categoryName = category.getSimpleName(); - Cluster annotations[] = category.getAnnotationsByType(Cluster.class); + Cluster[] annotations = category.getAnnotationsByType(Cluster.class); if (annotations.length == 0) { return categoryName; } diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/Initializer.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/Initializer.java index f61d9501b9e5..a2899a084488 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/Initializer.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/Initializer.java @@ -76,6 +76,7 @@ import org.apache.druid.testsEx.cluster.MetastoreClient; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -334,10 +335,7 @@ public Builder modules(List modules) public Builder modules(Module...modules) { - for (Module module : modules) { - this.modules.add(module); - } - return this; + return modules(Arrays.asList(modules)); } /** @@ -399,7 +397,7 @@ public synchronized Initializer build() private Initializer(Builder builder) { if (builder.configFile != null) { - this.clusterConfig = loadConfigFile(builder.configFile); + this.clusterConfig = loadConfigFile(builder.clusterName, builder.configFile); } else { this.clusterConfig = loadConfig(builder.clusterName, builder.configFile); } @@ -454,13 +452,13 @@ private static ResolvedConfig loadConfig(String category, String configName) } String loadName = StringUtils.format(CLUSTER_CONFIG_RESOURCE, category, configName); ClusterConfig config = ClusterConfig.loadFromResource(loadName); - return config.resolve(); + return config.resolve(category); } - private static ResolvedConfig loadConfigFile(String path) + private static ResolvedConfig loadConfigFile(String category, String path) { ClusterConfig config = ClusterConfig.loadFromFile(path); - return config.resolve(); + return config.resolve(category); } private static Injector makeInjector( @@ -499,7 +497,7 @@ private static Injector makeInjector( } /** - * Define test properties similar to how the server does. Property precendence + * Define test properties similar to how the server does. Property precedence * is: *

    *
  • System properties (highest)
    • @@ -523,6 +521,8 @@ private static Properties properties( } } finalProperties.putAll(System.getProperties()); + log.info("Properties:"); + log.info(finalProperties.toString()); return finalProperties; } diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedConfig.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedConfig.java index e90853e570d3..6bdfe96b2f93 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedConfig.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedConfig.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableMap; import org.apache.druid.curator.CuratorConfig; import org.apache.druid.curator.ExhibitorConfig; +import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; import org.apache.druid.testing.IntegrationTestingConfigProvider; import org.apache.druid.testsEx.config.ClusterConfig.ClusterType; @@ -30,6 +31,12 @@ import org.apache.druid.testsEx.config.ResolvedService.ResolvedZk; import org.apache.druid.testsEx.config.ServiceConfig.DruidConfig; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -48,6 +55,7 @@ public class ResolvedConfig public static final int DEFAULT_READY_TIMEOUT_SEC = 120; public static final int DEFAULT_READY_POLL_MS = 2000; + private final String category; private final ClusterType type; private final String proxyHost; private final int readyTimeoutSec; @@ -61,8 +69,9 @@ public class ResolvedConfig private final ResolvedMetastore metastore; private final Map druidServices = new HashMap<>(); - public ResolvedConfig(ClusterConfig config) + public ResolvedConfig(String category, ClusterConfig config) { + this.category = category; type = config.type() == null ? ClusterType.docker : config.type(); if (!hasProxy()) { proxyHost = null; @@ -293,6 +302,40 @@ public ExhibitorConfig toExhibitorConfig() .put("google_prefix", "googlePrefix") .build(); + private static void setDruidProperyVar(Map properties, String key, Object value) + { + if (value == null) { + return; + } + if (key.startsWith("druid_")) { + key = key.substring("druid_".length()); + } + String mapped = SETTINGS_MAP.get(key); + key = mapped == null ? key : mapped; + TestConfigs.putProperty(properties, IntegrationTestingConfigProvider.PROPERTY_BASE, key, value.toString()); + } + + private void loadPropertyFile(Map properties, File file) + { + try (BufferedReader in = new BufferedReader( + new InputStreamReader(new FileInputStream(file), StandardCharsets.UTF_8))) { + String line; + while ((line = in.readLine()) != null) { + if (Strings.isNullOrEmpty(line) || line.startsWith("#")) { + continue; + } + String[] parts = line.split("="); + if (parts.length != 2) { + continue; + } + setDruidProperyVar(properties, parts[0], parts[1]); + } + } + catch (IOException e) { + throw new IAE(e, "Cannot read file %s", file.getAbsolutePath()); + } + } + /** * Convert the config in this structure the the properties * used to configure Guice. @@ -317,19 +360,47 @@ public Map toProperties() // Settings are converted to properties so they can be overridden // by environment variables and -D command-line settings. for (Map.Entry entry : settings.entrySet()) { - String key = entry.getKey(); - if (key.startsWith("druid_")) { - key = key.substring("druid_".length()); - } - String mapped = SETTINGS_MAP.get(key); - key = mapped == null ? key : mapped; - TestConfigs.putProperty(properties, IntegrationTestingConfigProvider.PROPERTY_BASE, key, entry.getValue()); + setDruidProperyVar(properties, entry.getKey(), entry.getValue()); } // Add explicit properties if (this.properties != null) { properties.putAll(this.properties); } + + // Override with a user-specific config file. + File userEnv = new File( + new File( + System.getProperty("user.home"), + "druid-it"), + category + ".env"); + if (userEnv.exists()) { + loadPropertyFile(properties, userEnv); + } + + // Override with a user-specific config file. + String overrideEnv = System.getenv("OVERRIDE_ENV"); + if (overrideEnv != null) { + loadPropertyFile(properties, new File(overrideEnv)); + } + + // Override with any environment variables of the form "druid_" + for (Map.Entry entry : System.getenv().entrySet()) { + String key = entry.getKey(); + if (!key.startsWith("druid_")) { + continue; + } + setDruidProperyVar(properties, key, entry.getValue()); + } + + // Override with any system properties of the form "druid_" + for (Map.Entry entry : System.getProperties().entrySet()) { + String key = (String) entry.getKey(); + if (!key.startsWith("druid_")) { + continue; + } + setDruidProperyVar(properties, key, entry.getValue()); + } return properties; } } diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedMetastore.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedMetastore.java index 4534f30e51dc..f65790eb1c9f 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedMetastore.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedMetastore.java @@ -19,8 +19,8 @@ package org.apache.druid.testsEx.config; +import com.google.common.collect.ImmutableMap; import org.apache.commons.lang3.RegExUtils; -import org.apache.curator.shaded.com.google.common.collect.ImmutableMap; import org.apache.druid.metadata.MetadataStorageConnectorConfig; import java.util.HashMap; diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/ITLocalInputSourceAllInputFormatTest.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/ITLocalInputSourceAllInputFormatTest.java deleted file mode 100644 index 83414c26668f..000000000000 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/ITLocalInputSourceAllInputFormatTest.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.testsEx.indexer; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import org.apache.druid.java.util.common.Pair; -import org.apache.druid.testsEx.categories.InputFormat; -import org.apache.druid.testsEx.config.DruidTestRunner; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.util.List; -import java.util.Map; - -@RunWith(DruidTestRunner.class) -@Category(InputFormat.class) -public class ITLocalInputSourceAllInputFormatTest extends AbstractLocalInputSourceParallelIndexTest -{ - @Test - public void testAvroInputFormatIndexDataIngestionSpecWithSchema() throws Exception - { - @SuppressWarnings("unchecked") - List> fieldList = ImmutableList.of( - ImmutableMap.of("name", "timestamp", "type", "string"), - ImmutableMap.of("name", "page", "type", "string"), - ImmutableMap.of("name", "language", "type", "string"), - ImmutableMap.of("name", "user", "type", "string"), - ImmutableMap.of("name", "unpatrolled", "type", "string"), - ImmutableMap.of("name", "newPage", "type", "string"), - ImmutableMap.of("name", "robot", "type", "string"), - ImmutableMap.of("name", "anonymous", "type", "string"), - ImmutableMap.of("name", "namespace", "type", "string"), - ImmutableMap.of("name", "continent", "type", "string"), - ImmutableMap.of("name", "country", "type", "string"), - ImmutableMap.of("name", "region", "type", "string"), - ImmutableMap.of("name", "city", "type", "string"), - ImmutableMap.of("name", "added", "type", "int"), - ImmutableMap.of("name", "deleted", "type", "int"), - ImmutableMap.of("name", "delta", "type", "int") - ); - Map schema = ImmutableMap.of( - "namespace", "org.apache.druid.data.input", - "type", "record", - "name", "wikipedia", - "fields", fieldList); - doIndexTest(InputFormatDetails.AVRO, ImmutableMap.of("schema", schema), new Pair<>(false, false)); - } - - @Test - public void testAvroInputFormatIndexDataIngestionSpecWithoutSchema() throws Exception - { - doIndexTest(InputFormatDetails.AVRO, new Pair<>(false, false)); - } - - @Test - public void testJsonInputFormatIndexDataIngestionSpecWithSchema() throws Exception - { - doIndexTest(InputFormatDetails.JSON, new Pair<>(false, false)); - } - - @Test - public void testTsvInputFormatIndexDataIngestionSpecWithSchema() throws Exception - { - doIndexTest(InputFormatDetails.TSV, ImmutableMap.of("findColumnsFromHeader", true), new Pair<>(false, false)); - } - - @Test - public void testParquetInputFormatIndexDataIngestionSpecWithSchema() throws Exception - { - doIndexTest(InputFormatDetails.PARQUET, new Pair<>(false, false)); - } - - @Test - public void testOrcInputFormatIndexDataIngestionSpecWithSchema() throws Exception - { - doIndexTest(InputFormatDetails.ORC, new Pair<>(false, false)); - } - - @Test - public void testCsvInputFormatIndexDataIngestionSpecWithSchema() throws Exception - { - doIndexTest(InputFormatDetails.CSV, ImmutableMap.of("findColumnsFromHeader", true), new Pair<>(false, false)); - } -} diff --git a/integration-tests-ex/cases/src/test/resources/cluster/AzureDeepStorage/docker.yaml b/integration-tests-ex/cases/src/test/resources/cluster/AzureDeepStorage/docker.yaml new file mode 100644 index 000000000000..ac07bab98430 --- /dev/null +++ b/integration-tests-ex/cases/src/test/resources/cluster/AzureDeepStorage/docker.yaml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#------------------------------------------------------------------------- + +# Definition of the batch index test cluster. +# See https://yaml.org/spec/1.2.2 for more about YAML +include: + - /cluster/Common/zk-metastore.yaml + +druid: + coordinator: + instances: + - port: 8081 + overlord: + instances: + - port: 8090 + broker: + instances: + - port: 8082 + router: + instances: + - port: 8888 + historical: + instances: + - port: 8083 + indexer: + instances: + - port: 8091 + +# Properties to be set in the Properties object used in +# Guice configuration in lieu of the server-side runtime.properties +# file. +# +# druid.global.http.numMaxThreads avoids creating 40+ Netty threads. +# We only ever use 1. +# druid.test.config.dockerIp is used by some older test code. Remove +# it when that code is updated. +properties: + druid.global.http.numMaxThreads: 3 + druid.broker.http.numMaxThreads: 3 + druid.test.config.dockerIp: localhost + druid.test.config.cloudBucket: "new-it-framework" + druid.test.config.cloudPath: "" + docker.build.hadoop: true + start.hadoop.docker: true + override.config.path: "/Users/abhishekagrawal/pr_druid_it/druid/integration-tests-ex/it-azure-deep-storage/azure-config" diff --git a/integration-tests-ex/check-results.sh b/integration-tests-ex/check-results.sh index 44aac0aa9f6f..833971c4c144 100755 --- a/integration-tests-ex/check-results.sh +++ b/integration-tests-ex/check-results.sh @@ -16,6 +16,8 @@ # limitations under the License. #-------------------------------------------------------------------- +# Probably deprecated and to be removed. + # Run from Travis which has no good way to attach logs to a # build. Instead, we check if any IT failed. If so, we append # the last 100 lines of each server log to stdout. We have to diff --git a/integration-tests-ex/docs/compose.md b/integration-tests-ex/docs/compose.md index b26c1a0e4318..01b41896adbc 100644 --- a/integration-tests-ex/docs/compose.md +++ b/integration-tests-ex/docs/compose.md @@ -37,7 +37,7 @@ See also: ## File Structure -Docker Compose files live in the `druid-integration-test-cases` module (`test-cases` folder) +Docker Compose files live in the `druid-it-cases` module (`test-cases` folder) in the `cluster` directory. There is a separate subdirectory for each cluster type (subset of test categories), plus a `Common` folder for shared files. @@ -64,6 +64,44 @@ that we use to define base configurations. Test-specific configurations extend and customize the above. +### Druid Configuration + +Docker compose passes information to Docker in the form of environment variables. +The test use a variation of the environment-variable-based configuration used in +the [public Docker image](https://druid.apache.org/docs/latest/tutorials/docker.html). +That is, variables of the form `druid_my_config` are converted, by the image launch +script, into properties of the form `my.config`. These properties are then written +to a launch-specific `runtime.properties` file. + +Rather than have a test version of `runtime.properties`, instead we have a set of +files that define properties as environment variables. All are located in +`cases/cluster/Common/environment-configs`: + +* `common.env` - Properties common to all services. This is the test equivalent to + the `common.runtime.properties` file. +* `.env` - Properties unique to one service. This is the test equivalent to + the `service/runtime.properties` files. + +### Special Environment Variables + +Druid properties can be a bit awkward and verbose in a test environment. A number of +test-specific properties help: + +* `druid_standard_loadList` - Common extension load list for all tests, in the form + of a comma-delimited list of extensions (without the brackets.) Defined in + `common.env`. +* `druid_test_loadList` - A list of additional extensions to load for a specific test. + Defined in the `docker-compose.yaml` file for that test category. Do not include + quotes. + +Example test-specific list: + +```text +druid_test_loadList=druid-azure-extensions,my-extension +``` + +The launch script combines the two lists, and adds the required brackets and quotes. + ## Test-Specific Cluster Each test has a directory named `cluster/`. Docker Compose uses this name @@ -135,6 +173,58 @@ Outside of the application network, containers are accessible only via the host ports defined in the Docker Compose files. Thus, ZK is known as `localhost:2181` to tests and other code running outside of Docker. +## Test-Specific Configuration + +In addition to the Druid configuration discussed above, the framework provides +three ways to pass test-specific configuration to the tests. All of these methods +override any configuration in the `docker-compose` or cluster `env` files. + +The values here are passed into the Druid server as configuration values. The +values apply to all services. (This mechanism does not allow service-specific +values.) In all three approaches, use the `druid_` environment variable form. + +Precendence is in the order below with the user file lowest priority and environment +variables highest. + +### User-specific `~/druid-it/ + +# Test Creation Guide + +You've played with the existing tests and you are ready to create a new test. +This section walks you through the process. If you are converting an existing +test, then see the [conversion guide](conversion.md) instead. The details +of each step are covered in other files, we'll link them from here. + +## Category + +The first quesetion is: should your new test go into an existing category, +or should you create a new one? + +You should use an existing category if: + +* Your test is a new case within an obviously-existing category. +* Your test needs the same setup as an existing category, and is quick + to run. Using the existing category avoids the need to fire up a + Docker cluster just for your test. + +You should create a new category if: + +* Your test uses a customized setup: set of services, service + configuration, set of external dependencies, instead. +* Your test will run for an extended time, and is best run in + parallel with other tests in a build envrionment. Your test + can share a cluster configuration with an existing test, but + the new category allows the test to run by itself. + +When your test *can* reuse an existing cluser definition, then the question is +about time. It takes significan time (minutes) to start a Docker cluster. We clearly +don't want to pay that cost for a test that runs for seconds, if we could just add the +test to another category. On the other hand, if you've gone crazy and added a huge +suite of tests that take 20 minutes to run, then there is a huge win to be had by +running the tests in parallel, even if they reuse an existing cluster configuration. +Use your best judgment. + +The existing categories are listed in the +`org.apache.druid.testsEx.categories` package. The classes there represent +[JUnit categories]( +https://junit.org/junit4/javadoc/4.12/org/junit/experimental/categories/Categories.html). +See [Test Category](tests.md#Test+Category) for details. + +If you create a new category, but want to reuse the configuration of +an existing category, add the `@Cluster` annotation as described in the above +link. Note: be sure to link to a "base" category, not to a category that, itself, +has a `@Cluster` annotation. + +If you use the `@Cluster` annotation, you must also add a mapping in the +`cluster.sh` file. See the top of the file for an example. + +## Cluster Configuration + +If you create a new category, you must define a new cluster. There are two parts: + +* Docker compose +* Test configuration + +### Docker Compose + +Create a new folder: `custer/`, then create a `docker-compose.yaml` file +in that folder. Define your cluster by borrowing heavily from existing files. +See [compose](compose.md) for details. + +The only trick is if you want to include a new external dependency. The preferred +approach is to use an "official" image. If you must, you can create a custom image +in the `it-image` module. (We've not yet done that, so if you need a custom image, +let us know and we'll figure it out.) + +### Test Configuration + +Tests need a variety of configuration information. This is, at present, more +complex than we might like. You will at least need: + +* Describe the Docker Compose cluster +* Provide test-specific properties + +You may also need: + +* Test-specific Guice modules +* Environment variable bindings to various properties +* MySQL statements to pre-populate the Druid metastore DB +* And so on. + +### Test Config File + +The cluster and properties are defined in a config file. Create a folder +`src/test/resources/cluster/`. Then add a file called `docker.yaml`. +Crib the contents from the same category from which you borrowed the Docker +Compose definitions. Strip out properties and metastore statements you don't +need. Add those you do need. See [Test Configuration](test-config.md) for the +gory details of this file. + +### Test Config Code + +You may also want to customize Guice, environment variable bindings, etc. +This is done in the [test setup](tests.md#Initialization) method in your test. + +## Start Simple + +There are *many* things that can go wrong. It is best to start simple. + +### Verify the Cluster + +Start by ensuring your cluster works. + +* Define your cluster as described above. Or, pick one to reuse. +* Verify the cluster using `it.sh up `. +* Look at the Docker desktop UI to ensure the cluster says up. if not, + track down what went wrong. Look at both the Docker (stdout) and + Druid (`target//logs/.log`) files. + +### Starter Test + +Next, create your test file as described above and in [Tests](tests.md). + +* Create the test class. +* Add the required annotations. +* Create a simple test function that just prints "hello, world". +* Create your `docker.yaml` file as decribed above. +* Start your cluster, as described above, if not already started. +* Run the test from your IDE. +* Verify that the test "passes" (that is, it prints the message.) + +If so, then this means that your test connected to your custer and +verified the health of all the services declared in your `docker.yaml` file. + +If something goes wrong, you'll know it is in the basics. Check your +cluster status. Double-check the `docker.yaml` structure. Check ports. +Etc. + +### Client + +Every test is a Druid client. Determine which service API you need. Find an +existing test client. The `DruidClusterAdminClient` is the "modern" way to +interact with the cluster, but thus far has a limited set of methods. There +are older clients as well, but they tend to be quirky. Feel free to extend +`DruidClusterAdminClient`, or use the older one: whatever works. + +Inject the client into your test. See existing tests for how this is done. + +Revise your "starter" test to do some trivial operation using the client. +Retest to ensure things work. + +### Test Cases + +From here, you can start writing tests. Explore the existing mechanisms +(including those in the original `druid-integration-tests` module which may +not yet have been ported to the new framework yet.) For example, there are +ways to store specs as files and parameterize them in tests. There is a +syntax for running queries and specifying expected results. + +You may have to create a new tool to help with your test. If you do, +try to use the new mechanisms, such as `ResolvedClusterConfig` rather than +using the old, cumbersome ones. Post questions in Slack so we can help. + +### Extensions + +Your test may need a "non-default" extension. See [Special Environment Variables]( +compose.md#Special+Environment+Variables) for how to specify test-specific +extensions. (Hint: don't copy/paste the full load list!) + +Extensions have two aspects in ITs. They act like extensions in the Druid servers +running in Docker. So, the extension must be avaialble in the Docker image. All +standard Druid extensions which are available in the Druid distribution, are also +available in the image. The may not be enabled, however. Hence the need to define +the custom load list. + +Your test may use code from the extension. To the *tests*, however, the extension +is just another jar: it must be listed in the `pom.xml` file. There is no such +thing as a "Druid extensions" to the tests themselves. + +If you test an extension that is *not* part of the Druid distributeion, then it +has to get into the image. Reach out on the slack mailing list so we can discuss +solutions (such as mounting a directory that contains the extension). + +### Retries + +The old IT framework was very liberal in its use of retries. Retires were +used to handle: + +* the time lag in starting a cluster, +* the latency inherent in events propagaing through a distributed system + (such as when segments get published), +* random network failures, +* flaky tests. + +The new framework takes a stricter view. The framework itself will ensure +service are ready (using the Druid API for that purpose.) If a server reports +itself ready, but still fails on one of your API calls, then we've got a bug +to fix. Don't use retries to work around this issue because users won't know +to do this. + +In the new framwork, tests should not be flaky. Flaky tests are a drag on +development; they waste time. If your test is flaky, please fix it. Don't count +on the amount of times things take: a busy build system will run much slower than +your dedicated laptop. And so on. + +Ideally, Druid would provide a way to positively confirm that an action has +occurred. Perhaps this might be a test-only API. Otherwise, a retry is fine, but +should be coded into your test. (Or, better, implemented in a client.) Do this only +if we document that, for that API, users should poll. Otherwise, again, users of +the API under test won't know to retry, and so the test shouldn't do so either. + +This leaves random failures. The right place to handle those is in the client, +since they are independent of the usage of the API. + +The result of the above is that you should not need (or use) the `ITRetryUtil` +mechanism. No reason for your test to retry 240 times if something is really wrong +or your test is flaky. + +This is an area under development. If you see a reason to retry, lets discuss it +and put it in the proper place. + +### Travis + +Run your tests in the IDE. Try them using `it.sh test `. If that passes +add the test to Travis. The details on how to do so are still being worked out. +Likely, you will just copy/paste an existing test "stanza" to define your new +test. Your test will run in parallel with all other IT categories, which is why +we offered the advice above: the test has to have a good reason to fire up yet +another build task. + diff --git a/integration-tests-ex/docs/next-steps.md b/integration-tests-ex/docs/next-steps.md index ede844fb90cf..cef1b49f4849 100644 --- a/integration-tests-ex/docs/next-steps.md +++ b/integration-tests-ex/docs/next-steps.md @@ -19,7 +19,7 @@ # Future Work -The present version is very much a work in progress. Work completed to +The present version establishes the new IT framework. Work completed to date includes: * Restructure the Docker images to use the Druid produced from the diff --git a/integration-tests-ex/docs/scripts.md b/integration-tests-ex/docs/scripts.md index a4df5ef65d59..2c9ceb2a8746 100644 --- a/integration-tests-ex/docs/scripts.md +++ b/integration-tests-ex/docs/scripts.md @@ -24,8 +24,8 @@ what each one does. This guide lists each of them. ## `integration-tests-ex` -* `check-results.sh` - Workaround for Travis to extract container logs for - failed tests. +* `it.sh` - Utility to perform many IT-related actions such as building Druid, + running ITs, starting a cluster, etc. Use `it.sh help` to see the list of commands. ### `it-image` @@ -55,20 +55,20 @@ what each one does. This guide lists each of them. * `cluster//*.yaml` - Base Docker Compose scripts that define the "standard" Druid cluster. Tests use these files to avoid redundant copy/past of the standard items. -* `cluster.sh` - Launches or tear down a cluster for a test. Called from Maven - and can be used manually. See below. +* `cluster.sh` - Launches or tears down a cluster for a test. Called from Maven + and `it.sh`. Can be used manually. See below. The options for `cluster.sh` are: ```bash -cluster.sh category [-h|help|up|down|status|compose-cmd] +cluster.sh [-h|help|up|down|status|compose-cmd] category ``` -* `category` - the test category to launch. Performs mapping from the category name - to cluster name when categories share definitions. * `up` - starts the cluster. * `down` - shuts down the cluster. * `status` - displays cluster status for debugging. Expecially useful for debugging issues in Travis where we cannot directly inspect the Docker cluster itself. * `help` - repeats the usage line. * Others - passes the command on to Docker Compose. +* `category` - the test category to launch. Performs mapping from the category name + to cluster name when categories share definitions. diff --git a/integration-tests-ex/docs/tests.md b/integration-tests-ex/docs/tests.md index ac6b4f63fe11..1ef74a773381 100644 --- a/integration-tests-ex/docs/tests.md +++ b/integration-tests-ex/docs/tests.md @@ -150,6 +150,7 @@ The new names correspond to class names. The Test NG names were strings. ## Test Runner +The ITs are JUnit test, but use a special test runner to handle configuration. Test configuration is complex. The easiest way to configure, once the configuration files are set, is to use the `DruidTestRunner` class: @@ -175,6 +176,12 @@ a test method runs. For simple tests, this is all you need. The test runner validates that the test has a category, and handles the above mapping from category to cluster definition. +### Parameterization + +The `DruidTestRunner` extends `JUnitParamsRunner` to allow parameterized tests. +This class stays discretely out of the way if you don't care about parameters. +To use parameters, see the `CalciteJoinQueryTest` class for an example. + ## Initialization The JUnit-based integration tests are designed to be as simple as possible @@ -215,7 +222,7 @@ code. Tests may wish to load additional modules specific to that test. ## Custom Configuration There are times when a test needs additional Guice modules beyond what the -1Initializer` provides. In such cases, you can add a method to customize +`Initializer` provides. In such cases, you can add a method to customize configuration. ### Guice Modules @@ -253,6 +260,20 @@ the environment variable is set. You should also bind a default value: builder.propertyEnvVarBinding("druid.my.property", "ULTIMATE_ANSWER"); ``` +A property can also be passed in as either a system property or an environment +variable of the "Docker property environment variable form": + +```bash +druid_property_a=foo +./it.sh Category test +``` + +Or, directly on the command line: + +```text +-Ddruid_property_b=bar +``` + Property precedence is: * Properties set in code, as above. @@ -281,7 +302,7 @@ field in your class. Otherwise, give the builder a hint: builder.eagerInstance(ThePeskyComponent.class); ``` -## Test Structure +## Test Operation When working with tests, it is helpful to know a bit more about the "magic" behind `DruidTestRunner`. diff --git a/integration-tests-ex/image/docker/launch.sh b/integration-tests-ex/image/docker/launch.sh index 4b8d0293c8c7..1f64b4e14df0 100644 --- a/integration-tests-ex/image/docker/launch.sh +++ b/integration-tests-ex/image/docker/launch.sh @@ -33,14 +33,38 @@ cd / # TODO: enable only for security-related tests? #/tls/generate-server-certs-and-keystores.sh -. /druid.sh # The image contains both the MySQL and MariaDB JDBC drivers. # The MySQL driver is selected by the Docker Compose file. # Set druid.metadata.mysql.driver.driverClassName to the preferred # driver. +# Test-specific way to define extensions. Compose defines two test-specific +# variables. We combine these to create the final form converted to a property. +if [ -n "$druid_extensions_loadList" ]; then + echo "Using the provided druid_extensions_loadList=$druid_extensions_loadList" +else + mkdir -p /tmp/conf + EXTNS_FILE=/tmp/conf/extns + echo $druid_standard_loadList | tr "," "\n" > $EXTNS_FILE + if [ -n "$druid_test_loadList" ]; then + echo $druid_test_loadList | tr "," "\n" >> $EXTNS_FILE + fi + druid_extensions_loadList="[" + delim="" + while read -r line; do + druid_extensions_loadList="$druid_extensions_loadList$delim\"$line\"" + delim="," + done < $EXTNS_FILE + export druid_extensions_loadList="${druid_extensions_loadList}]" + unset druid_standard_loadList + unset druid_test_loadList + rm $EXTNS_FILE + echo "Effective druid_extensions_loadList=$druid_extensions_loadList" +fi + # Create druid service config files with all the config variables +. /druid.sh setupConfig # Export the service config file path to use in supervisord conf file @@ -92,7 +116,9 @@ fi LOG_FILE=$LOG_DIR/${INSTANCE_NAME}.log echo "" >> $LOG_FILE -echo "--- Service runtime.properties ---" >> $LOG_FILE +echo "--- env ---" >> $LOG_FILE +env >> $LOG_FILE +echo "--- runtime.properties ---" >> $LOG_FILE cat $DRUID_SERVICE_CONF_DIR/*.properties >> $LOG_FILE echo "---" >> $LOG_FILE echo "" >> $LOG_FILE diff --git a/integration-tests-ex/image/pom.xml b/integration-tests-ex/image/pom.xml index 55b3a9059e95..e352aa41793e 100644 --- a/integration-tests-ex/image/pom.xml +++ b/integration-tests-ex/image/pom.xml @@ -200,15 +200,15 @@ Reference: https://dzone.com/articles/build-docker-image-from-maven - ${project.version} ${mysql.version} - ${mysql.image.version} ${mariadb.version} - ${apache.kafka.version} + com.mysql.jdbc.Driver + ${mysql.image.version} ${confluent-version} + ${apache.kafka.version} ${zookeeper.version} ${hadoop.compile.version} - com.mysql.jdbc.Driver + ${project.version} ${druid.it.image-name} ${project.build.directory} - integration-tests-ex/cases/resources/data/** + **/test/resources/**/* + **/docker/client_tls/* + resources/data/**/* **/derby.log **/jvm.config **/*.avsc @@ -1895,14 +1896,17 @@ .github/pull_request_template.md .github/dependabot.yml git.version - node_modules/** + website/node_modules/** src/**/*.snap examples/conf/** .asf.yaml **/dependency-reduced-pom.xml .editorconfig **/hadoop.indexer.libs.version - **/codegen/** + **/codegen/**/* + **/.settings/**/* + **/.classpath + **/.project diff --git a/server/src/main/java/org/apache/druid/guice/DruidInjectorBuilder.java b/server/src/main/java/org/apache/druid/guice/DruidInjectorBuilder.java index 23adc8c31535..81e28431e2db 100644 --- a/server/src/main/java/org/apache/druid/guice/DruidInjectorBuilder.java +++ b/server/src/main/java/org/apache/druid/guice/DruidInjectorBuilder.java @@ -87,8 +87,6 @@ public DruidInjectorBuilder(final DruidInjectorBuilder from) /** * Add an arbitrary set of modules. - * - * @see #add(Object) */ public DruidInjectorBuilder add(Object...input) { From 7e98bdbe960740a834f8a1e29e968c010bdbdf2f Mon Sep 17 00:00:00 2001 From: Paul Rogers Date: Fri, 19 Aug 2022 21:07:57 -0700 Subject: [PATCH 03/11] Disabled flaky tests to get a clean build --- .../prefetch/PrefetchableTextFilesFirehoseFactoryTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/src/test/java/org/apache/druid/data/input/impl/prefetch/PrefetchableTextFilesFirehoseFactoryTest.java b/core/src/test/java/org/apache/druid/data/input/impl/prefetch/PrefetchableTextFilesFirehoseFactoryTest.java index dd9c384e91a5..38fb843a34c0 100644 --- a/core/src/test/java/org/apache/druid/data/input/impl/prefetch/PrefetchableTextFilesFirehoseFactoryTest.java +++ b/core/src/test/java/org/apache/druid/data/input/impl/prefetch/PrefetchableTextFilesFirehoseFactoryTest.java @@ -41,6 +41,7 @@ import org.junit.Assert; import org.junit.BeforeClass; import org.junit.ClassRule; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -249,6 +250,7 @@ public void testWithCacheAndFetch() throws IOException } @Test + @Ignore("See issue #12638") public void testWithLargeCacheAndSmallFetch() throws IOException { final TestPrefetchableTextFilesFirehoseFactory factory = @@ -336,6 +338,7 @@ public void testTimeout() throws IOException } @Test + @Ignore("See issue #12638") public void testReconnectWithCacheAndPrefetch() throws IOException { final TestPrefetchableTextFilesFirehoseFactory factory = From 754ba9ca3ad8d9e36cbfc04a0fba35f2060a8b07 Mon Sep 17 00:00:00 2001 From: Paul Rogers Date: Sat, 20 Aug 2022 21:35:23 -0700 Subject: [PATCH 04/11] Build fixes --- .travis.yml | 12 +-- integration-tests-ex/check-results.sh | 91 ------------------- .../clients/OverlordResourceTestClient.java | 3 +- 3 files changed, 6 insertions(+), 100 deletions(-) delete mode 100755 integration-tests-ex/check-results.sh diff --git a/.travis.yml b/.travis.yml index fceb60785e52..14d16e5e363b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,7 +46,7 @@ addons: # Add various options to make 'mvn install' fast and skip javascript compile (-Ddruid.console.skip=true) since it is not # needed. Depending on network speeds, "mvn -q install" may take longer than the default 10 minute timeout to print any # output. To compensate, use travis_wait to extend the timeout. -install: ./check_test_suite.py && travis_terminate 0 || echo 'Running Maven install...' && MAVEN_OPTS='-Xmx3000m' travis_wait 15 ${MVN} clean install -q -ff -pl '!distribution,!:druid-it-tools,!:druid-it-image' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C && ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} +install: ./check_test_suite.py && travis_terminate 0 || echo 'Running Maven install...' && MAVEN_OPTS='-Xmx3000m' travis_wait 15 ${MVN} clean install -q -ff -pl '!distribution,!:druid-it-tools,!:druid-it-image,!:druid-it-cases' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C && ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} # There are 3 stages of tests # 1. Tests - phase 1 @@ -676,24 +676,22 @@ jobs: jdk: openjdk8 env: TESTNG_GROUPS='-Dgroups=query' USE_INDEXER='middleManager' MYSQL_DRIVER_CLASSNAME='org.mariadb.jdbc.Driver' OVERRIDE_CONFIG_PATH='./environment-configs/test-groups/prepopulated-data' + # Revised ITs. - &integration_tests_ex + name: "(Compile=openjdk8, Run=openjdk8) leadership and high availability integration tests (new)" stage: Tests - phase 2 jdk: openjdk8 services: *integration_test_services env: JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' - - # Revised ITs. - - <<: *integration_tests_ex - name: "(Compile=openjdk8, Run=openjdk8) leadership and high availability integration tests (new)" # Uses the install defined above. Then, builds the test tools and docker image, # and runs one IT. If tests fail, echos log lines of any of # the Druid services that did not exit normally. - script: it.sh travis HighAvailability + script: ./it.sh travis HighAvailability - <<: *integration_tests_ex name: "(Compile=openjdk8, Run=openjdk8) batch index integration test with Indexer (new)" env: JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer' - script: it.sh travis BatchIndex + script: ./it.sh travis BatchIndex # END - Integration tests for Compile with Java 8 and Run with Java 8 diff --git a/integration-tests-ex/check-results.sh b/integration-tests-ex/check-results.sh deleted file mode 100755 index 833971c4c144..000000000000 --- a/integration-tests-ex/check-results.sh +++ /dev/null @@ -1,91 +0,0 @@ -#! /bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#-------------------------------------------------------------------- - -# Probably deprecated and to be removed. - -# Run from Travis which has no good way to attach logs to a -# build. Instead, we check if any IT failed. If so, we append -# the last 100 lines of each server log to stdout. We have to -# stay wihtin the 4MB limit which Travis applies, so we only -# emit logs for the first failure, and only for servers that -# don't report normal completion. -# -# The only good way to check for test failures is to parse -# the Failsafe summary for each test located in -# /target/failsafe-reports/failsafe-summary.xml -# -# This directory has many subdirectories, some of which are -# tests. We rely on the fact that a test starts with "it-" AND -# contains a failsafe report. (Some projects start with "it-" -# but are not tests.) - -# Run in the docker-tests directory -cd $(dirname $0) - -# Scan for candidate projects -for PROJECT in it-* -do - # Check if a failsafe report exists. It will exist if the directory is - # a test project and failsafe ran on that directory. - REPORTS="$PROJECT/target/failsafe-reports/failsafe-summary.xml" - if [ -f "$REPORTS" ] - then - # OK, so Bash isn't the world's best text processing language... - ERRS=1 - FAILS=1 - while IFS= read -r line - do - if [ "$line" = " 0" ] - then - ERRS=0 - fi - if [ "$line" = " 0" ] - then - FAILS=0 - fi - done < "$REPORTS" - if [ $ERRS -eq 1 -o $FAILS -eq 1 ] - then - FOUND_LOGS=0 - echo "======= $PROJECT Failed ==========" - # All logs except zookeeper - for log in $(ls $PROJECT/target/shared/logs/[a-y]*.log) - do - # We assume that a successful exit includes a line with the - # following: - # Stopping lifecycle [module] stage [INIT] - tail -5 "$log" | grep -Fq 'Stopping lifecycle [module] stage [INIT]' - if [ $? -ne 0 ] - then - # Assume failure and report tail - echo $(basename $log) "logtail ========================" - tail -100 "$log" - FOUND_LOGS=1 - fi - done - - # Only emit the first failure to avoid output bloat - if [ $FOUND_LOGS -eq 1 ] - then - exit 0 - else - echo "All Druid services exited normally." - fi - fi - fi -done diff --git a/integration-tests/src/main/java/org/apache/druid/testing/clients/OverlordResourceTestClient.java b/integration-tests/src/main/java/org/apache/druid/testing/clients/OverlordResourceTestClient.java index 8ac4551eb628..e3f2a98dc637 100644 --- a/integration-tests/src/main/java/org/apache/druid/testing/clients/OverlordResourceTestClient.java +++ b/integration-tests/src/main/java/org/apache/druid/testing/clients/OverlordResourceTestClient.java @@ -323,9 +323,8 @@ public Boolean call() { TaskState status = getTaskStatus(taskID).getStatusCode(); if (status == TaskState.FAILED) { - String msg = getTaskErrorMessage(taskID); LOG.error("Task failed: %s", taskID); - LOG.error("Message: %s", msg); + LOG.error("Message: %s", getTaskErrorMessage(taskID)); throw new ISE("Indexer task FAILED"); } return status == TaskState.SUCCESS; From 1a5af178734087182d05eb985a0eb2d5ba631aae Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Mon, 22 Aug 2022 02:50:55 +0530 Subject: [PATCH 05/11] first test, serde causing problems --- .../Common/environment-configs/common.env | 2 +- integration-tests-ex/cases/pom.xml | 6 + .../druid/testsEx/msq/ITMultiStageQuery.java | 83 +++++++++++++ integration-tests/pom.xml | 6 + .../MsqOverlordResourceTestClient.java | 59 ++++++++++ .../druid/testing/clients/MsqTestClient.java | 42 +++++++ .../testing/utils/MsqQueryWithResults.java | 41 +++++++ .../testing/utils/MsqTestQueryHelper.java | 110 ++++++++++++++++++ 8 files changed, 348 insertions(+), 1 deletion(-) create mode 100644 integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQuery.java create mode 100644 integration-tests/src/main/java/org/apache/druid/testing/clients/MsqOverlordResourceTestClient.java create mode 100644 integration-tests/src/main/java/org/apache/druid/testing/clients/MsqTestClient.java create mode 100644 integration-tests/src/main/java/org/apache/druid/testing/utils/MsqQueryWithResults.java create mode 100644 integration-tests/src/main/java/org/apache/druid/testing/utils/MsqTestQueryHelper.java diff --git a/integration-tests-ex/cases/cluster/Common/environment-configs/common.env b/integration-tests-ex/cases/cluster/Common/environment-configs/common.env index ad6f9e7d5fc5..1cecb8a796ae 100644 --- a/integration-tests-ex/cases/cluster/Common/environment-configs/common.env +++ b/integration-tests-ex/cases/cluster/Common/environment-configs/common.env @@ -47,7 +47,7 @@ DRUID_INSTANCE= # Optional as this is the default? #druid_extensions_directory=/usr/local/druid/extensions #druid_extensions_loadList=["mysql-metadata-storage","druid-basic-security","simple-client-sslcontext","it-tools","druid-lookups-cached-global","druid-histogram","druid-datasketches","druid-parquet-extensions","druid-avro-extensions","druid-protobuf-extensions","druid-orc-extensions","druid-kafka-indexing-service","druid-s3-extensions"] -druid_extensions_loadList=["mysql-metadata-storage","it-tools","druid-lookups-cached-global","druid-histogram","druid-datasketches","druid-parquet-extensions","druid-avro-extensions","druid-protobuf-extensions","druid-orc-extensions","druid-kafka-indexing-service","druid-s3-extensions"] +druid_extensions_loadList=["mysql-metadata-storage","it-tools","druid-lookups-cached-global","druid-histogram","druid-datasketches","druid-parquet-extensions","druid-avro-extensions","druid-protobuf-extensions","druid-orc-extensions","druid-kafka-indexing-service","druid-s3-extensions","druid-multi-stage-query"] druid_extensions_hadoopDependenciesDir=/shared/hadoop-dependencies # Logging diff --git a/integration-tests-ex/cases/pom.xml b/integration-tests-ex/cases/pom.xml index f1cb1b282daf..51c2f7983c91 100644 --- a/integration-tests-ex/cases/pom.xml +++ b/integration-tests-ex/cases/pom.xml @@ -45,6 +45,12 @@ druid-integration-tests ${project.parent.version} + + + org.apache.druid.extensions + druid-multi-stage-query + ${project.parent.version} + org.apache.druid diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQuery.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQuery.java new file mode 100644 index 000000000000..e5e11e08fe65 --- /dev/null +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQuery.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.testsEx.msq; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; +import org.apache.druid.msq.indexing.report.MSQTaskReport; +import org.apache.druid.testing.IntegrationTestingConfig; +import org.apache.druid.testing.clients.MsqTestClient; +import org.apache.druid.testing.utils.MsqTestQueryHelper; +import org.apache.druid.testsEx.categories.BatchIndex; +import org.apache.druid.testsEx.config.DruidTestRunner; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Map; + +@RunWith(DruidTestRunner.class) +@Category(BatchIndex.class) +public class ITMultiStageQuery +{ + @Inject + private MsqTestQueryHelper msqHelper; + + @Inject + private MsqTestClient msqClient; + + @Inject + private IntegrationTestingConfig config; + @Inject + + private ObjectMapper jsonMapper; + + @Test + public void test() throws Exception + { + String query = + "INSERT INTO dst SELECT *\n" + + "FROM TABLE(extern(\n" + + " '{\n" + + " \"type\": \"inline\",\n" + + " \"data\": \"a,b,1\\nc,d,2\\n\"\n" + + " }',\n" + + " '{\n" + + " \"type\": \"csv\",\n" + + " \"columns\": [\"x\",\"y\",\"z\"],\n" + + " \"listDelimiter\": null,\n" + + " \"findColumnsFromHeader\": false,\n" + + " \"skipHeaderRows\": 0\n" + + " }',\n" + + " '[\n" + + " {\"name\": \"x\", \"type\": \"STRING\"},\n" + + " {\"name\": \"y\", \"type\": \"STRING\"},\n" + + " {\"name\": \"z\", \"type\": \"LONG\"}\n" + + " ]'\n" + + "))\n" + + "PARTITIONED BY ALL TIME"; + String taskId = msqHelper.submitMsqTask(query); + msqHelper.pollTaskIdForCompletion(taskId, 0); + Map reports = msqHelper.fetchStatusReports(taskId); + int x = 5; + x += 1; + } +} + diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 6ea011227191..76272e0b9521 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -239,6 +239,12 @@ simple-client-sslcontext ${project.parent.version} + + org.apache.druid.extensions + druid-multi-stage-query + ${project.parent.version} + + org.apache.druid druid-services diff --git a/integration-tests/src/main/java/org/apache/druid/testing/clients/MsqOverlordResourceTestClient.java b/integration-tests/src/main/java/org/apache/druid/testing/clients/MsqOverlordResourceTestClient.java new file mode 100644 index 000000000000..81c3782c57cf --- /dev/null +++ b/integration-tests/src/main/java/org/apache/druid/testing/clients/MsqOverlordResourceTestClient.java @@ -0,0 +1,59 @@ +package org.apache.druid.testing.clients; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; +import org.apache.druid.indexing.common.TaskReport; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.http.client.HttpClient; +import org.apache.druid.java.util.http.client.response.StatusResponseHolder; +import org.apache.druid.msq.indexing.report.MSQTaskReport; +import org.apache.druid.testing.IntegrationTestingConfig; +import org.apache.druid.testing.guice.TestClient; +import org.jboss.netty.handler.codec.http.HttpMethod; + +import java.util.HashMap; +import java.util.Map; + +public class MsqOverlordResourceTestClient extends OverlordResourceTestClient +{ + ObjectMapper jsonMapper; + + @Inject + MsqOverlordResourceTestClient( + ObjectMapper jsonMapper, + @TestClient HttpClient httpClient, + IntegrationTestingConfig config + ) + { + super(jsonMapper, httpClient, config); + this.jsonMapper = jsonMapper; + } + + public Map getTaskReportForMsqTask(String taskId) + { + try { + StatusResponseHolder response = makeRequest( + HttpMethod.GET, + StringUtils.format( + "%s%s", + getIndexerURL(), + StringUtils.format("task/%s/reports", StringUtils.urlEncode(taskId)) + ) + ); + return jsonMapper.readValue(response.getContent(), MsqTaskReportType.class); + } + catch (ISE e) { + throw e; + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static class MsqTaskReportType extends HashMap + { + + } +} diff --git a/integration-tests/src/main/java/org/apache/druid/testing/clients/MsqTestClient.java b/integration-tests/src/main/java/org/apache/druid/testing/clients/MsqTestClient.java new file mode 100644 index 000000000000..647967caad63 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/druid/testing/clients/MsqTestClient.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.testing.clients; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; +import org.apache.druid.java.util.http.client.HttpClient; +import org.apache.druid.sql.http.SqlQuery; +import org.apache.druid.testing.IntegrationTestingConfig; +import org.apache.druid.testing.guice.TestClient; + +import javax.ws.rs.core.MediaType; + +public class MsqTestClient extends AbstractQueryResourceTestClient +{ + @Inject + MsqTestClient( + ObjectMapper jsonMapper, + @TestClient HttpClient httpClient, + IntegrationTestingConfig config + ) + { + super(jsonMapper, null, httpClient, config.getRouterUrl(), MediaType.APPLICATION_JSON, null); + } +} diff --git a/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqQueryWithResults.java b/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqQueryWithResults.java new file mode 100644 index 000000000000..124787e67865 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqQueryWithResults.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.testing.utils; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.sql.http.SqlQuery; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class MsqQueryWithResults extends AbstractQueryWithResults +{ + + @JsonCreator + public MsqQueryWithResults( + @JsonProperty("query") SqlQuery query, + @JsonProperty("expectedResults") List> expectedResults + ) + { + super(query, expectedResults, Collections.emptyList()); + } +} diff --git a/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqTestQueryHelper.java b/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqTestQueryHelper.java new file mode 100644 index 000000000000..351fd2c06f57 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqTestQueryHelper.java @@ -0,0 +1,110 @@ +package org.apache.druid.testing.utils; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; +import com.google.inject.Inject; +import org.apache.druid.indexer.TaskStatusPlus; +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.http.client.response.StatusResponseHolder; +import org.apache.druid.msq.indexing.report.MSQTaskReport; +import org.apache.druid.msq.sql.SqlTaskStatus; +import org.apache.druid.sql.http.SqlQuery; +import org.apache.druid.testing.IntegrationTestingConfig; +import org.apache.druid.testing.clients.MsqOverlordResourceTestClient; +import org.apache.druid.testing.clients.MsqTestClient; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; + +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +public class MsqTestQueryHelper extends AbstractTestQueryHelper +{ + final ObjectMapper jsonMapper; + final IntegrationTestingConfig config; + final MsqOverlordResourceTestClient overlordClient; + final MsqTestClient msqClient; + + + @Inject + MsqTestQueryHelper( + ObjectMapper jsonMapper, + MsqTestClient queryClient, + IntegrationTestingConfig config, + MsqOverlordResourceTestClient overlordClient, + MsqTestClient msqClient + ) + { + super(jsonMapper, queryClient, config); + this.jsonMapper = jsonMapper; + this.config = config; + this.overlordClient = overlordClient; + this.msqClient = msqClient; + } + + @Override + public String getQueryURL(String schemeAndHost) + { + return StringUtils.format("%s/druid/v2/sql/task", schemeAndHost); + } + + public String submitMsqTask(String sqlQueryString) throws ExecutionException, InterruptedException + { + return submitMsqTask(new SqlQuery(sqlQueryString, null, false, false, false, ImmutableMap.of(), null)); + } + + // Run the task, wait for it to complete, fetch the reports, verify the results, + public String submitMsqTask(SqlQuery sqlQuery) throws ExecutionException, InterruptedException + { + String queryUrl = getQueryURL(config.getRouterUrl()); + Future responseHolderFuture = msqClient.queryAsync(queryUrl, sqlQuery); + // It is okay to block here for the result because MSQ tasks return the task Id associated with it, which shouldn't + // consume a lot of time + StatusResponseHolder statusResponseHolder = responseHolderFuture.get(); + HttpResponseStatus httpResponseStatus = statusResponseHolder.getStatus(); + if (!httpResponseStatus.equals(HttpResponseStatus.ACCEPTED)) { + throw new ISE("Unable to submit the task successfully"); + } + String content = statusResponseHolder.getContent(); + SqlTaskStatus sqlTaskStatus; + try { + sqlTaskStatus = jsonMapper.readValue(content, SqlTaskStatus.class); + } + catch (JsonProcessingException e) { + throw new ISE("Unable to parse the response"); + } + if (sqlTaskStatus.getState().isFailure()) { + throw new ISE("Unable to start the task successfully.\nPossible exception: %s", sqlTaskStatus.getError()); + } + return sqlTaskStatus.getTaskId(); + } + + public void pollTaskIdForCompletion(String taskId, long maxTimeoutSeconds) + { + if (maxTimeoutSeconds < 0) { + throw new IAE("Timeout cannot be negative"); + } else if (maxTimeoutSeconds == 0) { + maxTimeoutSeconds = Long.MAX_VALUE; + } + long time = 0; + do { + TaskStatusPlus taskStatusPlus = overlordClient.getTaskStatus(taskId); + if (taskStatusPlus.getStatusCode().isComplete()) { + return; + } + try { + Thread.sleep(1000); + } + catch (InterruptedException e) { + throw new ISE(e, "Interrupted while polling for task [%s] completion", taskId); + } + } while (time++ < maxTimeoutSeconds); + } + + public Map fetchStatusReports(String taskId) { + return overlordClient.getTaskReportForMsqTask(taskId); + } +} From c0357d2f975988d7b1a7599b253e0a11d855c0e5 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Mon, 22 Aug 2022 15:46:18 +0530 Subject: [PATCH 06/11] serde working --- .../msq/indexing/report/MSQStagesReport.java | 2 +- .../msq/indexing/report/MSQTaskReport.java | 7 ++++ .../druid/testsEx/msq/ITMultiStageQuery.java | 33 +++++++++++++++++++ .../MsqOverlordResourceTestClient.java | 12 +++++-- .../testing/utils/MsqTestQueryHelper.java | 5 +-- 5 files changed, 53 insertions(+), 6 deletions(-) diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/report/MSQStagesReport.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/report/MSQStagesReport.java index 3900f1540f6d..226bf349a4d9 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/report/MSQStagesReport.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/report/MSQStagesReport.java @@ -84,7 +84,7 @@ public static MSQStagesReport create( return new MSQStagesReport(stages); } - @JsonValue + @JsonProperty("stages") public List getStages() { return stages; diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/report/MSQTaskReport.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/report/MSQTaskReport.java index d8f59f6a1806..b96a65ae888c 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/report/MSQTaskReport.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/report/MSQTaskReport.java @@ -55,6 +55,13 @@ public String getReportKey() return REPORT_KEY; } + @JsonProperty("type") + private String getType() + { + return REPORT_KEY; + } + + @Override @JsonProperty public Object getPayload() diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQuery.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQuery.java index e5e11e08fe65..a8e3b6cb0ca6 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQuery.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQuery.java @@ -19,18 +19,27 @@ package org.apache.druid.testsEx.msq; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; import com.google.inject.Inject; +import org.apache.druid.indexer.TaskState; +import org.apache.druid.indexing.common.TaskReport; +import org.apache.druid.msq.indexing.report.MSQStagesReport; +import org.apache.druid.msq.indexing.report.MSQStatusReport; import org.apache.druid.msq.indexing.report.MSQTaskReport; +import org.apache.druid.msq.indexing.report.MSQTaskReportPayload; import org.apache.druid.testing.IntegrationTestingConfig; import org.apache.druid.testing.clients.MsqTestClient; import org.apache.druid.testing.utils.MsqTestQueryHelper; import org.apache.druid.testsEx.categories.BatchIndex; import org.apache.druid.testsEx.config.DruidTestRunner; +import org.joda.time.DateTime; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.util.ArrayDeque; import java.util.Map; @RunWith(DruidTestRunner.class) @@ -79,5 +88,29 @@ public void test() throws Exception int x = 5; x += 1; } + + @Test + public void scratchPad() throws JsonProcessingException + { + MSQTaskReport msqTaskReport = new MSQTaskReport( + "test-id", + new MSQTaskReportPayload( + new MSQStatusReport( + TaskState.RUNNING, + null, + new ArrayDeque<>(), + DateTime.now(), + 1 + ), + new MSQStagesReport(ImmutableList.of()), + null, + null + ) + ); + String serialized = jsonMapper.writeValueAsString(msqTaskReport); + MSQTaskReport deserializEd = jsonMapper.readValue(serialized, MSQTaskReport.class); + int x = 6; + x += 1; + } } diff --git a/integration-tests/src/main/java/org/apache/druid/testing/clients/MsqOverlordResourceTestClient.java b/integration-tests/src/main/java/org/apache/druid/testing/clients/MsqOverlordResourceTestClient.java index 81c3782c57cf..44046f707758 100644 --- a/integration-tests/src/main/java/org/apache/druid/testing/clients/MsqOverlordResourceTestClient.java +++ b/integration-tests/src/main/java/org/apache/druid/testing/clients/MsqOverlordResourceTestClient.java @@ -3,11 +3,12 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; -import org.apache.druid.indexing.common.TaskReport; +import org.apache.druid.guice.annotations.Json; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.http.client.HttpClient; import org.apache.druid.java.util.http.client.response.StatusResponseHolder; +import org.apache.druid.msq.guice.MSQIndexingModule; import org.apache.druid.msq.indexing.report.MSQTaskReport; import org.apache.druid.testing.IntegrationTestingConfig; import org.apache.druid.testing.guice.TestClient; @@ -22,7 +23,7 @@ public class MsqOverlordResourceTestClient extends OverlordResourceTestClient @Inject MsqOverlordResourceTestClient( - ObjectMapper jsonMapper, + @Json ObjectMapper jsonMapper, @TestClient HttpClient httpClient, IntegrationTestingConfig config ) @@ -42,7 +43,12 @@ public Map getTaskReportForMsqTask(String taskId) StringUtils.format("task/%s/reports", StringUtils.urlEncode(taskId)) ) ); - return jsonMapper.readValue(response.getContent(), MsqTaskReportType.class); + return jsonMapper.registerModules(new MSQIndexingModule().getJacksonModules()).readValue( + response.getContent(), + new TypeReference>() + { + } + ); } catch (ISE e) { throw e; diff --git a/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqTestQueryHelper.java b/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqTestQueryHelper.java index 351fd2c06f57..101efd20c432 100644 --- a/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqTestQueryHelper.java +++ b/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqTestQueryHelper.java @@ -21,7 +21,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; -public class MsqTestQueryHelper extends AbstractTestQueryHelper +public class MsqTestQueryHelper extends AbstractTestQueryHelper { final ObjectMapper jsonMapper; final IntegrationTestingConfig config; @@ -104,7 +104,8 @@ public void pollTaskIdForCompletion(String taskId, long maxTimeoutSeconds) } while (time++ < maxTimeoutSeconds); } - public Map fetchStatusReports(String taskId) { + public Map fetchStatusReports(String taskId) + { return overlordClient.getTaskReportForMsqTask(taskId); } } From 51170bc29ea7c8d553198d09186a455efc921ab3 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Tue, 23 Aug 2022 11:58:15 +0530 Subject: [PATCH 07/11] insert and select check --- .../msq/indexing/report/MSQStagesReport.java | 1 - .../druid/testsEx/msq/ITMultiStageQuery.java | 48 +++------- .../MsqOverlordResourceTestClient.java | 29 ++++++- .../MSQResultsReportDeserializable.java | 68 +++++++++++++++ .../models/MSQTaskReportDeserializable.java | 71 +++++++++++++++ .../MSQTaskReportPayloadDeserializable.java | 87 +++++++++++++++++++ .../testing/utils/MsqQueryWithResults.java | 5 +- .../testing/utils/MsqTestQueryHelper.java | 77 ++++++++++++++-- 8 files changed, 340 insertions(+), 46 deletions(-) create mode 100644 integration-tests/src/main/java/org/apache/druid/testing/guice/models/MSQResultsReportDeserializable.java create mode 100644 integration-tests/src/main/java/org/apache/druid/testing/guice/models/MSQTaskReportDeserializable.java create mode 100644 integration-tests/src/main/java/org/apache/druid/testing/guice/models/MSQTaskReportPayloadDeserializable.java diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/report/MSQStagesReport.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/report/MSQStagesReport.java index 226bf349a4d9..2b9251c5892c 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/report/MSQStagesReport.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/report/MSQStagesReport.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonValue; import com.google.common.base.Preconditions; import org.apache.druid.msq.kernel.QueryDefinition; import org.apache.druid.msq.kernel.StageDefinition; diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQuery.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQuery.java index a8e3b6cb0ca6..6a459399095a 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQuery.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQuery.java @@ -19,27 +19,21 @@ package org.apache.druid.testsEx.msq; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; -import org.apache.druid.indexer.TaskState; -import org.apache.druid.indexing.common.TaskReport; -import org.apache.druid.msq.indexing.report.MSQStagesReport; -import org.apache.druid.msq.indexing.report.MSQStatusReport; -import org.apache.druid.msq.indexing.report.MSQTaskReport; -import org.apache.druid.msq.indexing.report.MSQTaskReportPayload; import org.apache.druid.testing.IntegrationTestingConfig; import org.apache.druid.testing.clients.MsqTestClient; +import org.apache.druid.testing.guice.models.MSQTaskReportDeserializable; +import org.apache.druid.testing.utils.MsqQueryWithResults; import org.apache.druid.testing.utils.MsqTestQueryHelper; import org.apache.druid.testsEx.categories.BatchIndex; import org.apache.druid.testsEx.config.DruidTestRunner; -import org.joda.time.DateTime; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; -import java.util.ArrayDeque; import java.util.Map; @RunWith(DruidTestRunner.class) @@ -84,33 +78,19 @@ public void test() throws Exception + "PARTITIONED BY ALL TIME"; String taskId = msqHelper.submitMsqTask(query); msqHelper.pollTaskIdForCompletion(taskId, 0); - Map reports = msqHelper.fetchStatusReports(taskId); - int x = 5; - x += 1; - } + Map reports = msqHelper.fetchStatusReports(taskId); - @Test - public void scratchPad() throws JsonProcessingException - { - MSQTaskReport msqTaskReport = new MSQTaskReport( - "test-id", - new MSQTaskReportPayload( - new MSQStatusReport( - TaskState.RUNNING, - null, - new ArrayDeque<>(), - DateTime.now(), - 1 - ), - new MSQStagesReport(ImmutableList.of()), - null, - null + String resultsQuery = "SELECT * FROM dst"; + String resultsTaskId = msqHelper.submitMsqTask(resultsQuery); + msqHelper.pollTaskIdForCompletion(resultsTaskId, 0); + msqHelper.compareResults(resultsTaskId, new MsqQueryWithResults( + query, + ImmutableList.of( + ImmutableMap.of("x", "a", "y", "b", "z", 1), + ImmutableMap.of("x", "c", "y", "d", "z", 2) ) - ); - String serialized = jsonMapper.writeValueAsString(msqTaskReport); - MSQTaskReport deserializEd = jsonMapper.readValue(serialized, MSQTaskReport.class); - int x = 6; + )); + int x = 5; x += 1; } } - diff --git a/integration-tests/src/main/java/org/apache/druid/testing/clients/MsqOverlordResourceTestClient.java b/integration-tests/src/main/java/org/apache/druid/testing/clients/MsqOverlordResourceTestClient.java index 44046f707758..8bda377c3ff3 100644 --- a/integration-tests/src/main/java/org/apache/druid/testing/clients/MsqOverlordResourceTestClient.java +++ b/integration-tests/src/main/java/org/apache/druid/testing/clients/MsqOverlordResourceTestClient.java @@ -1,7 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.druid.testing.clients; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; import com.google.inject.Inject; import org.apache.druid.guice.annotations.Json; import org.apache.druid.java.util.common.ISE; @@ -12,6 +32,7 @@ import org.apache.druid.msq.indexing.report.MSQTaskReport; import org.apache.druid.testing.IntegrationTestingConfig; import org.apache.druid.testing.guice.TestClient; +import org.apache.druid.testing.guice.models.MSQTaskReportDeserializable; import org.jboss.netty.handler.codec.http.HttpMethod; import java.util.HashMap; @@ -32,7 +53,7 @@ public class MsqOverlordResourceTestClient extends OverlordResourceTestClient this.jsonMapper = jsonMapper; } - public Map getTaskReportForMsqTask(String taskId) + public Map getTaskReportForMsqTask(String taskId) { try { StatusResponseHolder response = makeRequest( @@ -43,9 +64,11 @@ public Map getTaskReportForMsqTask(String taskId) StringUtils.format("task/%s/reports", StringUtils.urlEncode(taskId)) ) ); - return jsonMapper.registerModules(new MSQIndexingModule().getJacksonModules()).readValue( + jsonMapper.registerModules(new MSQIndexingModule().getJacksonModules()); + jsonMapper.registerSubtypes(ImmutableList.of(MSQTaskReportDeserializable.class)); + return jsonMapper.readValue( response.getContent(), - new TypeReference>() + new TypeReference>() { } ); diff --git a/integration-tests/src/main/java/org/apache/druid/testing/guice/models/MSQResultsReportDeserializable.java b/integration-tests/src/main/java/org/apache/druid/testing/guice/models/MSQResultsReportDeserializable.java new file mode 100644 index 000000000000..a4f199969b81 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/druid/testing/guice/models/MSQResultsReportDeserializable.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.testing.guice.models; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import org.apache.druid.segment.column.RowSignature; + +import javax.annotation.Nullable; +import java.util.List; + +public class MSQResultsReportDeserializable +{ + private final RowSignature signature; + @Nullable + private final List sqlTypeNames; + private final List results; + + @JsonCreator + public MSQResultsReportDeserializable( + @JsonProperty("signature") final RowSignature signature, + @JsonProperty("sqlTypeNames") @Nullable final List sqlTypeNames, + @JsonProperty("results") final List results + ) + { + this.signature = Preconditions.checkNotNull(signature, "signature"); + this.sqlTypeNames = sqlTypeNames; + this.results = Preconditions.checkNotNull(results, "results"); + } + + @JsonProperty("signature") + public RowSignature getSignature() + { + return signature; + } + + @JsonProperty("sqlTypeNames") + @JsonInclude(JsonInclude.Include.NON_NULL) + public List getSqlTypeNames() + { + return sqlTypeNames; + } + + @JsonProperty("results") + public List getResults() + { + return results; + } +} diff --git a/integration-tests/src/main/java/org/apache/druid/testing/guice/models/MSQTaskReportDeserializable.java b/integration-tests/src/main/java/org/apache/druid/testing/guice/models/MSQTaskReportDeserializable.java new file mode 100644 index 000000000000..29a4f8121749 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/druid/testing/guice/models/MSQTaskReportDeserializable.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.testing.guice.models; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import org.apache.druid.indexing.common.TaskReport; + +@JsonTypeName(MSQTaskReportDeserializable.REPORT_KEY) +public class MSQTaskReportDeserializable implements TaskReport +{ + public static final String REPORT_KEY = "multiStageQuery"; + + private final String taskId; + private final MSQTaskReportPayloadDeserializable payload; + + @JsonCreator + public MSQTaskReportDeserializable( + @JsonProperty("taskId") final String taskId, + @JsonProperty("payload") final MSQTaskReportPayloadDeserializable payload + ) + { + this.taskId = taskId; + this.payload = payload; + } + + @Override + @JsonProperty + public String getTaskId() + { + return taskId; + } + + @Override + public String getReportKey() + { + return REPORT_KEY; + } + + @JsonProperty("type") + private String getType() + { + return REPORT_KEY; + } + + + @Override + @JsonProperty + public Object getPayload() + { + return payload; + } +} diff --git a/integration-tests/src/main/java/org/apache/druid/testing/guice/models/MSQTaskReportPayloadDeserializable.java b/integration-tests/src/main/java/org/apache/druid/testing/guice/models/MSQTaskReportPayloadDeserializable.java new file mode 100644 index 000000000000..a72228289b1f --- /dev/null +++ b/integration-tests/src/main/java/org/apache/druid/testing/guice/models/MSQTaskReportPayloadDeserializable.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.testing.guice.models; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.msq.counters.CounterSnapshotsTree; +import org.apache.druid.msq.indexing.report.MSQStagesReport; +import org.apache.druid.msq.indexing.report.MSQStatusReport; + +import javax.annotation.Nullable; + +public class MSQTaskReportPayloadDeserializable +{ + private final MSQStatusReport status; + + @Nullable + private final MSQStagesReport stages; + + @Nullable + private final CounterSnapshotsTree counters; + + @Nullable + private final MSQResultsReportDeserializable results; + + @JsonCreator + public MSQTaskReportPayloadDeserializable( + @JsonProperty("status") MSQStatusReport status, + @JsonProperty("stages") @Nullable MSQStagesReport stages, + @JsonProperty("counters") @Nullable CounterSnapshotsTree counters, + @JsonProperty("results") @Nullable MSQResultsReportDeserializable results + ) + { + this.status = status; + this.stages = stages; + this.counters = counters; + this.results = results; + } + + @JsonProperty + public MSQStatusReport getStatus() + { + return status; + } + + @Nullable + @JsonProperty + @JsonInclude(JsonInclude.Include.NON_NULL) + public MSQStagesReport getStages() + { + return stages; + } + + @Nullable + @JsonProperty + @JsonInclude(JsonInclude.Include.NON_NULL) + public CounterSnapshotsTree getCounters() + { + return counters; + } + + @Nullable + @JsonProperty + @JsonInclude(JsonInclude.Include.NON_NULL) + public MSQResultsReportDeserializable getResults() + { + return results; + } +} diff --git a/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqQueryWithResults.java b/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqQueryWithResults.java index 124787e67865..7c246aeff532 100644 --- a/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqQueryWithResults.java +++ b/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqQueryWithResults.java @@ -21,18 +21,17 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.druid.sql.http.SqlQuery; import java.util.Collections; import java.util.List; import java.util.Map; -public class MsqQueryWithResults extends AbstractQueryWithResults +public class MsqQueryWithResults extends AbstractQueryWithResults { @JsonCreator public MsqQueryWithResults( - @JsonProperty("query") SqlQuery query, + @JsonProperty("query") String query, @JsonProperty("expectedResults") List> expectedResults ) { diff --git a/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqTestQueryHelper.java b/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqTestQueryHelper.java index 101efd20c432..39ce6c72cf9a 100644 --- a/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqTestQueryHelper.java +++ b/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqTestQueryHelper.java @@ -1,9 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.druid.testing.utils; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; +import org.apache.druid.indexer.TaskState; import org.apache.druid.indexer.TaskStatusPlus; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; @@ -11,12 +32,20 @@ import org.apache.druid.java.util.http.client.response.StatusResponseHolder; import org.apache.druid.msq.indexing.report.MSQTaskReport; import org.apache.druid.msq.sql.SqlTaskStatus; +import org.apache.druid.segment.column.RowSignature; import org.apache.druid.sql.http.SqlQuery; import org.apache.druid.testing.IntegrationTestingConfig; import org.apache.druid.testing.clients.MsqOverlordResourceTestClient; import org.apache.druid.testing.clients.MsqTestClient; +import org.apache.druid.testing.guice.models.MSQResultsReportDeserializable; +import org.apache.druid.testing.guice.models.MSQTaskReportDeserializable; +import org.apache.druid.testing.guice.models.MSQTaskReportPayloadDeserializable; import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -59,7 +88,7 @@ public String submitMsqTask(String sqlQueryString) throws ExecutionException, In // Run the task, wait for it to complete, fetch the reports, verify the results, public String submitMsqTask(SqlQuery sqlQuery) throws ExecutionException, InterruptedException { - String queryUrl = getQueryURL(config.getRouterUrl()); + String queryUrl = getQueryURL(config.getBrokerUrl()); Future responseHolderFuture = msqClient.queryAsync(queryUrl, sqlQuery); // It is okay to block here for the result because MSQ tasks return the task Id associated with it, which shouldn't // consume a lot of time @@ -82,7 +111,7 @@ public String submitMsqTask(SqlQuery sqlQuery) throws ExecutionException, Interr return sqlTaskStatus.getTaskId(); } - public void pollTaskIdForCompletion(String taskId, long maxTimeoutSeconds) + public TaskState pollTaskIdForCompletion(String taskId, long maxTimeoutSeconds) { if (maxTimeoutSeconds < 0) { throw new IAE("Timeout cannot be negative"); @@ -92,8 +121,9 @@ public void pollTaskIdForCompletion(String taskId, long maxTimeoutSeconds) long time = 0; do { TaskStatusPlus taskStatusPlus = overlordClient.getTaskStatus(taskId); - if (taskStatusPlus.getStatusCode().isComplete()) { - return; + TaskState statusCode = taskStatusPlus.getStatusCode(); + if (statusCode != null && statusCode.isComplete()) { + return taskStatusPlus.getStatusCode(); } try { Thread.sleep(1000); @@ -102,10 +132,47 @@ public void pollTaskIdForCompletion(String taskId, long maxTimeoutSeconds) throw new ISE(e, "Interrupted while polling for task [%s] completion", taskId); } } while (time++ < maxTimeoutSeconds); + return overlordClient.getTaskStatus(taskId).getStatusCode(); } - public Map fetchStatusReports(String taskId) + public Map fetchStatusReports(String taskId) { return overlordClient.getTaskReportForMsqTask(taskId); } + + public void compareResults(String taskId, MsqQueryWithResults expectedQueryWithResults) + { + Map statusReport = fetchStatusReports(taskId); + MSQTaskReportDeserializable taskReport = statusReport.get(MSQTaskReport.REPORT_KEY); + if (taskReport == null) { + throw new ISE("Unable to fetch the status report for the task [%]", taskId); + } + MSQTaskReportPayloadDeserializable taskReportPayload = Preconditions.checkNotNull( + (MSQTaskReportPayloadDeserializable) taskReport.getPayload(), + "payload" + ); + MSQResultsReportDeserializable resultsReport = Preconditions.checkNotNull( + taskReportPayload.getResults(), + "Results report for the task id is empty" + ); + + List> actualResults = new ArrayList<>(); + + List results = resultsReport.getResults(); + RowSignature rowSignature = resultsReport.getSignature(); + + for (Object[] row : results) { + Map rowWithFieldNames = new HashMap<>(); + for (int i = 0; i < row.length; ++i) { + rowWithFieldNames.put(rowSignature.getColumnName(i), row[i]); + } + actualResults.add(rowWithFieldNames); + } + + QueryResultVerifier.compareResults( + actualResults, + expectedQueryWithResults.getExpectedResults(), + Collections.emptyList() + ); + } } From 9ac998f343d98f54ef6be4776fae4aabd53ddead Mon Sep 17 00:00:00 2001 From: Paul Rogers Date: Tue, 23 Aug 2022 10:03:06 -0700 Subject: [PATCH 08/11] Merge fix --- .../tests/indexer/AbstractIndexerTest.java | 4 ++ .../queries/high_availability_sys.json | 39 +++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 integration-tests/src/test/resources/queries/high_availability_sys.json diff --git a/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractIndexerTest.java b/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractIndexerTest.java index 0c88619bc128..1bb6e8e12c8d 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractIndexerTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractIndexerTest.java @@ -24,6 +24,7 @@ import org.apache.commons.io.IOUtils; import org.apache.druid.guice.annotations.Json; import org.apache.druid.guice.annotations.Smile; +import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.logger.Logger; @@ -165,6 +166,9 @@ protected void waitForAllTasksToCompleteForDataSource(final String dataSource) public static String getResourceAsString(String file) throws IOException { try (final InputStream inputStream = getResourceAsStream(file)) { + if (inputStream == null) { + throw new ISE("Failed to load resource: [%s]", file); + } return IOUtils.toString(inputStream, StandardCharsets.UTF_8); } } diff --git a/integration-tests/src/test/resources/queries/high_availability_sys.json b/integration-tests/src/test/resources/queries/high_availability_sys.json new file mode 100644 index 000000000000..d5d60d4f2979 --- /dev/null +++ b/integration-tests/src/test/resources/queries/high_availability_sys.json @@ -0,0 +1,39 @@ +[ + { + "description": "query sys.servers to make sure all expected servers are available", + "query": { + "query": "SELECT host, server_type, is_leader FROM sys.servers ORDER BY host" + }, + "expectedResults": [ + {"host":"%%BROKER%%","server_type":"broker", "is_leader": %%NON_LEADER%%}, + {"host":"%%COORDINATOR_ONE%%","server_type":"coordinator", "is_leader": %%COORDINATOR_ONE_LEADER%%}, + {"host":"%%COORDINATOR_TWO%%","server_type":"coordinator", "is_leader": %%COORDINATOR_TWO_LEADER%%}, + {"host":"%%OVERLORD_ONE%%","server_type":"overlord", "is_leader": %%OVERLORD_ONE_LEADER%%}, + {"host":"%%OVERLORD_TWO%%","server_type":"overlord", "is_leader": %%OVERLORD_TWO_LEADER%%}, + {"host":"%%ROUTER%%","server_type":"router", "is_leader": %%NON_LEADER%%} + ] + }, + { + "description": "query sys.segments which is fed via coordinator data", + "query": { + "query": "SELECT datasource, count(*) FROM sys.segments WHERE datasource='wikipedia_editstream' OR datasource='twitterstream' GROUP BY 1 " + }, + "expectedResults": [ + { + "datasource": "wikipedia_editstream", + "EXPR$1": 1 + }, + { + "datasource": "twitterstream", + "EXPR$1": 3 + } + ] + }, + { + "description": "query sys.tasks which is fed via overlord", + "query": { + "query": "SELECT datasource, count(*) FROM sys.tasks WHERE datasource='wikipedia_editstream' OR datasource='twitterstream' GROUP BY 1 " + }, + "expectedResults": [] + } +] \ No newline at end of file From b99fcc2b05e50b24846c6bd476a19c0a2fc21bee Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Wed, 24 Aug 2022 16:10:30 +0530 Subject: [PATCH 09/11] Add cluster annotations for MSQ test cases --- .../testsEx/categories/MultiStageQuery.java | 25 +++++++++++++++++++ .../druid/testsEx/msq/ITMultiStageQuery.java | 4 +-- 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/MultiStageQuery.java diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/MultiStageQuery.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/MultiStageQuery.java new file mode 100644 index 000000000000..e4a8fdc9a0ea --- /dev/null +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/MultiStageQuery.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.testsEx.categories; + +public class MultiStageQuery +{ + +} diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQuery.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQuery.java index 6a459399095a..a68be047e7b4 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQuery.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQuery.java @@ -28,7 +28,7 @@ import org.apache.druid.testing.guice.models.MSQTaskReportDeserializable; import org.apache.druid.testing.utils.MsqQueryWithResults; import org.apache.druid.testing.utils.MsqTestQueryHelper; -import org.apache.druid.testsEx.categories.BatchIndex; +import org.apache.druid.testsEx.categories.MultiStageQuery; import org.apache.druid.testsEx.config.DruidTestRunner; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -37,7 +37,7 @@ import java.util.Map; @RunWith(DruidTestRunner.class) -@Category(BatchIndex.class) +@Category(MultiStageQuery.class) public class ITMultiStageQuery { @Inject From 5a51441fa3e16e0a6a22fb998d71df3eaca695a8 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Wed, 24 Aug 2022 16:15:44 +0530 Subject: [PATCH 10/11] Add cluster config for MSQ --- .../MultiStageQuery/docker-compose.yaml | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 integration-tests-ex/cases/cluster/MultiStageQuery/docker-compose.yaml diff --git a/integration-tests-ex/cases/cluster/MultiStageQuery/docker-compose.yaml b/integration-tests-ex/cases/cluster/MultiStageQuery/docker-compose.yaml new file mode 100644 index 000000000000..f22ff3e6ddbe --- /dev/null +++ b/integration-tests-ex/cases/cluster/MultiStageQuery/docker-compose.yaml @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +networks: + druid-it-net: + name: druid-it-net + ipam: + config: + - subnet: 172.172.172.0/24 + +services: + zookeeper: + extends: + file: ../Common/dependencies.yaml + service: zookeeper + + metadata: + extends: + file: ../Common/dependencies.yaml + service: metadata + + coordinator: + extends: + file: ../Common/druid.yaml + service: coordinator + container_name: coordinator + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + - druid_manager_segments_pollDuration=PT5S + - druid_coordinator_period=PT10S + depends_on: + - zookeeper + - metadata + + overlord: + extends: + file: ../Common/druid.yaml + service: overlord + container_name: overlord + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - zookeeper + - metadata + + broker: + extends: + file: ../Common/druid.yaml + service: broker + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - zookeeper + + router: + extends: + file: ../Common/druid.yaml + service: router + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - zookeeper + + historical: + extends: + file: ../Common/druid.yaml + service: historical + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - zookeeper + + indexer: + extends: + file: ../Common/druid.yaml + service: indexer + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + volumes: + # Test data + - ../../resources:/resources + depends_on: + - zookeeper From 8daef61f3bd8b5561fb0b3e0966935a1b2366b31 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Wed, 24 Aug 2022 16:31:54 +0530 Subject: [PATCH 11/11] Add MSQ config to the pom.xml --- integration-tests-ex/cases/pom.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/integration-tests-ex/cases/pom.xml b/integration-tests-ex/cases/pom.xml index 76141986e0ae..dae10239afa3 100644 --- a/integration-tests-ex/cases/pom.xml +++ b/integration-tests-ex/cases/pom.xml @@ -291,6 +291,15 @@ AzureDeepStorage + + IT-MultiStageQuery + + false + + + MultiStageQuery + + docker-tests