From 07821dfb6d01dfe6506aed4b6f23265929d36895 Mon Sep 17 00:00:00 2001 From: ythorat2 Date: Tue, 14 Nov 2023 20:54:39 -0600 Subject: [PATCH 1/7] fix flaky tests in processing module --- .../nested/NestedDataColumnSupplierTest.java | 16 ++++++++-------- .../nested/NestedDataColumnSupplierV4Test.java | 4 ++-- .../virtual/ExpressionVirtualColumnTest.java | 5 +++-- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java index a97fd3786352..e8f9c4e56720 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java @@ -408,8 +408,8 @@ private void smokeTest(NestedDataComplexColumn column) throws IOException for (int i = 0; i < data.size(); i++) { Map row = data.get(i); Assert.assertEquals( - JSON_MAPPER.writeValueAsString(row), - JSON_MAPPER.writeValueAsString(StructuredData.unwrap(rawSelector.getObject())) + JSON_MAPPER.readTree(JSON_MAPPER.writeValueAsString(row)), + JSON_MAPPER.readTree(JSON_MAPPER.writeValueAsString(StructuredData.unwrap(rawSelector.getObject()))) ); testPath(row, i, "v", vSelector, vDimSelector, vValueIndex, vPredicateIndex, vNulls, null); @@ -539,8 +539,8 @@ private void smokeTestArrays(NestedDataComplexColumn column) throws IOException while (offset.withinBounds()) { Map row = arrayTestData.get(rowCounter); Assert.assertEquals( - JSON_MAPPER.writeValueAsString(row), - JSON_MAPPER.writeValueAsString(StructuredData.unwrap(rawSelector.getObject())) + JSON_MAPPER.readTree(JSON_MAPPER.writeValueAsString(row)), + JSON_MAPPER.readTree(JSON_MAPPER.writeValueAsString(StructuredData.unwrap(rawSelector.getObject()))) ); Object[] s = (Object[]) row.get("s"); @@ -595,8 +595,8 @@ private void smokeTestArrays(NestedDataComplexColumn column) throws IOException Map row = arrayTestData.get(rowCounter); Assert.assertEquals( - JSON_MAPPER.writeValueAsString(row), - JSON_MAPPER.writeValueAsString(StructuredData.unwrap(rawVector[i])) + JSON_MAPPER.readTree(JSON_MAPPER.writeValueAsString(row)), + JSON_MAPPER.readTree(JSON_MAPPER.writeValueAsString(StructuredData.unwrap(rawVector[i]))) ); Object[] s = (Object[]) row.get("s"); Object[] l = (Object[]) row.get("l"); @@ -644,8 +644,8 @@ private void smokeTestArrays(NestedDataComplexColumn column) throws IOException for (int i = 0; i < bitmapVectorOffset.getCurrentVectorSize(); i++, rowCounter += 2) { Map row = arrayTestData.get(rowCounter); Assert.assertEquals( - JSON_MAPPER.writeValueAsString(row), - JSON_MAPPER.writeValueAsString(StructuredData.unwrap(rawVector[i])) + JSON_MAPPER.readTree(JSON_MAPPER.writeValueAsString(row)), + JSON_MAPPER.readTree(JSON_MAPPER.writeValueAsString(StructuredData.unwrap(rawVector[i]))) ); Object[] s = (Object[]) row.get("s"); Object[] l = (Object[]) row.get("l"); diff --git a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java index 7b3a8eac20b4..df4bc9cc6d91 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java @@ -413,8 +413,8 @@ private void smokeTest(NestedDataComplexColumn column) throws IOException for (int i = 0; i < data.size(); i++) { Map row = data.get(i); Assert.assertEquals( - JSON_MAPPER.writeValueAsString(row), - JSON_MAPPER.writeValueAsString(StructuredData.unwrap(rawSelector.getObject())) + JSON_MAPPER.readTree(JSON_MAPPER.writeValueAsString(row)), + JSON_MAPPER.readTree(JSON_MAPPER.writeValueAsString(StructuredData.unwrap(rawSelector.getObject()))) ); testPath(row, i, "v", vSelector, vDimSelector, vValueIndex, vPredicateIndex, vNulls, null); testPath(row, i, "x", xSelector, xDimSelector, xValueIndex, xPredicateIndex, xNulls, ColumnType.LONG); diff --git a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVirtualColumnTest.java b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVirtualColumnTest.java index 9d35e6025ff6..07db3e47b577 100644 --- a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVirtualColumnTest.java +++ b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVirtualColumnTest.java @@ -61,6 +61,7 @@ import javax.annotation.Nullable; import java.util.Arrays; +import java.util.HashSet; public class ExpressionVirtualColumnTest extends InitializedNullHandlingTest { @@ -734,10 +735,10 @@ public void testLongSelectorOfTimeColumn() @Test public void testRequiredColumns() { - Assert.assertEquals(ImmutableList.of("x", "y"), X_PLUS_Y.requiredColumns()); + Assert.assertEquals(new HashSet<>(ImmutableList.of("x", "y")), new HashSet<>(X_PLUS_Y.requiredColumns())); Assert.assertEquals(ImmutableList.of(), CONSTANT_LIKE.requiredColumns()); Assert.assertEquals(ImmutableList.of("z"), Z_LIKE.requiredColumns()); - Assert.assertEquals(ImmutableList.of("x", "z"), Z_CONCAT_X.requiredColumns()); + Assert.assertEquals(new HashSet<>(ImmutableList.of("x", "z")), new HashSet<>(Z_CONCAT_X.requiredColumns())); } @Test From 0a88c393b3a5ee23fe25525ebed4e44f0f9b6cc0 Mon Sep 17 00:00:00 2001 From: same8891 <55152301+same8891@users.noreply.github.com> Date: Thu, 30 Nov 2023 13:19:49 -0600 Subject: [PATCH 2/7] Update ParametrizedUriEmitterTest.java --- .../java/util/emitter/core/ParametrizedUriEmitterTest.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/processing/src/test/java/org/apache/druid/java/util/emitter/core/ParametrizedUriEmitterTest.java b/processing/src/test/java/org/apache/druid/java/util/emitter/core/ParametrizedUriEmitterTest.java index c5dc26432fe0..5dfdc3ecadbe 100644 --- a/processing/src/test/java/org/apache/druid/java/util/emitter/core/ParametrizedUriEmitterTest.java +++ b/processing/src/test/java/org/apache/druid/java/util/emitter/core/ParametrizedUriEmitterTest.java @@ -177,14 +177,13 @@ protected ListenableFuture go(Request request) throws JsonProcessingEx { Assert.assertEquals("http://example.com/val1/val2", request.getUrl()); Assert.assertEquals( - StringUtils.format( + JSON_MAPPER.readTree(StringUtils.format( "[%s,%s]\n", JSON_MAPPER.writeValueAsString(events.get(0)), JSON_MAPPER.writeValueAsString(events.get(1)) - ), - StandardCharsets.UTF_8.decode(request.getByteBufferData().slice()).toString() + )), + JSON_MAPPER.readTree(StandardCharsets.UTF_8.decode(request.getByteBufferData().slice()).toString()) ); - return GoHandlers.immediateFuture(EmitterTest.okResponse()); } }.times(1) From 9d5f40398e0290995fe27d0f683cb178a2ae1126 Mon Sep 17 00:00:00 2001 From: Werfero <79309838+Rette66@users.noreply.github.com> Date: Thu, 30 Nov 2023 18:33:59 -0600 Subject: [PATCH 3/7] Fix flaky test in WikiParquetReaderTest.java --- .../druid/data/input/parquet/WikiParquetReaderTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/WikiParquetReaderTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/WikiParquetReaderTest.java index 82ac2acd2754..a9e2867b266d 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/WikiParquetReaderTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/WikiParquetReaderTest.java @@ -19,6 +19,7 @@ package org.apache.druid.data.input.parquet; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import org.apache.druid.data.input.ColumnsFilter; import org.apache.druid.data.input.InputEntityReader; @@ -39,6 +40,8 @@ */ public class WikiParquetReaderTest extends BaseParquetReaderTest { + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + @Test public void testWiki() throws IOException { @@ -77,7 +80,7 @@ public void testWiki() throws IOException + " \"user\" : \"nuclear\",\n" + " \"timestamp\" : \"2013-08-31T01:02:33Z\"\n" + "}"; - Assert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); + Assert.assertEquals(JSON_MAPPER.readTree(expectedJson), JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); } @Test From f2d095e4c21f70beeedde5abaf848092f3da074d Mon Sep 17 00:00:00 2001 From: prathyushreddylpr <112648268+prathyushreddylpr@users.noreply.github.com> Date: Thu, 30 Nov 2023 19:54:40 -0600 Subject: [PATCH 4/7] Fix Flaky tests in CompatParquetReaderTest, TimestampsParquetReaderTest and FlattenSpecParquetReaderTest Co-authored-by: lakkidi2 Co-authored-by: lxb007981 Co-authored-by: Prathyush Reddy Lakkidi --- .../parquet/CompatParquetReaderTest.java | 25 ++++++++----------- .../parquet/FlattenSpecParquetReaderTest.java | 4 ++- .../parquet/TimestampsParquetReaderTest.java | 6 +++-- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetReaderTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetReaderTest.java index f94949d6bd21..ccd48cfa3987 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetReaderTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetReaderTest.java @@ -19,6 +19,7 @@ package org.apache.druid.data.input.parquet; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import org.apache.druid.data.input.ColumnsFilter; import org.apache.druid.data.input.InputEntityReader; @@ -42,6 +43,8 @@ */ public class CompatParquetReaderTest extends BaseParquetReaderTest { + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + @Test public void testBinaryAsString() throws IOException { @@ -94,19 +97,14 @@ public void testBinaryAsString() throws IOException + " \"field\" : \"hey this is &é(-è_çà)=^$ù*! Ω^^\",\n" + " \"ts\" : 1471800234\n" + "}"; - Assert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); - + Assert.assertEquals(JSON_MAPPER.readTree(expectedJson), JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); final String expectedJsonBinary = "{\n" + " \"field\" : \"aGV5IHRoaXMgaXMgJsOpKC3DqF/Dp8OgKT1eJMO5KiEgzqleXg==\",\n" + " \"ts\" : 1471800234\n" - + "}"; - Assert.assertEquals( - expectedJsonBinary, - DEFAULT_JSON_WRITER.writeValueAsString(sampledAsBinary.get(0).getRawValues()) - ); + + "}"; + Assert.assertEquals(JSON_MAPPER.readTree(expectedJsonBinary), JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampledAsBinary.get(0).getRawValues()))); } - @Test public void testParquet1217() throws IOException { @@ -143,7 +141,7 @@ public void testParquet1217() throws IOException final String expectedJson = "{\n" + " \"col\" : -1\n" + "}"; - Assert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); + Assert.assertEquals(JSON_MAPPER.readTree(expectedJson), JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); } @Test @@ -305,7 +303,7 @@ required group nestedIntsColumn (LIST) { + " } ]\n" + " }\n" + "}"; - Assert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); + Assert.assertEquals(JSON_MAPPER.readTree(expectedJson), JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); } @Test @@ -342,10 +340,9 @@ public void testOldRepeatedInt() throws IOException final String expectedJson = "{\n" + " \"repeatedInt\" : [ 1, 2, 3 ]\n" + "}"; - Assert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); + Assert.assertEquals(JSON_MAPPER.readTree(expectedJson), JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); } - @Test public void testReadNestedArrayStruct() throws IOException { @@ -385,7 +382,7 @@ public void testReadNestedArrayStruct() throws IOException + " \"repeatedMessage\" : [ 3 ]\n" + " } ]\n" + "}"; - Assert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); + Assert.assertEquals(JSON_MAPPER.readTree(expectedJson), JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); } @Test @@ -435,6 +432,6 @@ public void testProtoStructWithArray() throws IOException + " \"someId\" : 9\n" + " }\n" + "}"; - Assert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); + Assert.assertEquals(JSON_MAPPER.readTree(expectedJson), JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); } } diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetReaderTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetReaderTest.java index 726ecde4adf3..d6c104124538 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetReaderTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetReaderTest.java @@ -19,6 +19,7 @@ package org.apache.druid.data.input.parquet; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import org.apache.druid.data.input.ColumnsFilter; import org.apache.druid.data.input.InputEntityReader; @@ -205,8 +206,9 @@ public void testFlat1FlattenSelectListItem() throws IOException flattenSpec ); List sampled = sampleAllRows(reader); + ObjectMapper mapper = new ObjectMapper(); - Assert.assertEquals(FLAT_JSON, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); + Assert.assertEquals(mapper.readTree(FLAT_JSON), mapper.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); } diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java index c0189fe8bc19..5a4dec40c120 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java @@ -19,6 +19,7 @@ package org.apache.druid.data.input.parquet; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import org.apache.druid.data.input.ColumnsFilter; import org.apache.druid.data.input.InputEntityReader; @@ -91,8 +92,9 @@ public void testDateHandling() throws IOException + " \"idx\" : 1,\n" + " \"date_as_date\" : 1497744000000\n" + "}"; - Assert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampledAsString.get(0).getRawValues())); - Assert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampledAsDate.get(0).getRawValues())); + ObjectMapper mapper = new ObjectMapper(); + Assert.assertEquals(mapper.readTree(expectedJson), mapper.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampledAsString.get(0).getRawValues()))); + Assert.assertEquals(mapper.readTree(expectedJson), mapper.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampledAsDate.get(0).getRawValues()))); } @Test From 794a4781ce83ec017735e48b3ac3afa1c4a68751 Mon Sep 17 00:00:00 2001 From: Yashdeep Thorat Date: Fri, 1 Dec 2023 12:01:54 -0600 Subject: [PATCH 5/7] Fix flaky tests in server, services, extensions-core/parquet-extensions, indexing-service modules --- .../parquet/NestedColumnParquetReaderTest.java | 4 +++- .../data/input/parquet/WikiParquetReaderTest.java | 2 +- .../druid/indexing/overlord/TaskLockboxTest.java | 7 ++++--- .../druid/discovery/BaseNodeRoleWatcherTest.java | 2 +- .../overlord/supervisor/SupervisorStatusTest.java | 2 +- .../apache/druid/metadata/input/SqlEntityTest.java | 2 +- .../server/router/TieredBrokerHostSelectorTest.java | 13 +++++++------ 7 files changed, 18 insertions(+), 14 deletions(-) diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/NestedColumnParquetReaderTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/NestedColumnParquetReaderTest.java index 24205993eb6a..66540f91732a 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/NestedColumnParquetReaderTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/NestedColumnParquetReaderTest.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import org.apache.druid.data.input.ColumnsFilter; import org.apache.druid.data.input.InputEntityReader; import org.apache.druid.data.input.InputRow; @@ -39,6 +40,7 @@ import org.junit.Test; import java.io.IOException; +import java.util.HashSet; import java.util.List; public class NestedColumnParquetReaderTest extends BaseParquetReaderTest @@ -218,7 +220,7 @@ public void testNestedColumnSchemalessNestedTestFile() throws IOException ); List rows = readAllRows(reader); - Assert.assertEquals(ImmutableList.of("nestedData", "dim1", "metric1"), rows.get(0).getDimensions()); + Assert.assertEquals(ImmutableSet.of("nestedData", "dim1", "metric1"), new HashSet<>(rows.get(0).getDimensions())); Assert.assertEquals(FlattenSpecParquetInputTest.TS1, rows.get(0).getTimestamp().toString()); Assert.assertEquals(ImmutableList.of("d1v1"), rows.get(0).getDimension("dim1")); Assert.assertEquals("d1v1", rows.get(0).getRaw("dim1")); diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/WikiParquetReaderTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/WikiParquetReaderTest.java index a9e2867b266d..4bb2f2346305 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/WikiParquetReaderTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/WikiParquetReaderTest.java @@ -99,6 +99,6 @@ public void testUint32Datatype() throws IOException + " \"foo\" : \"baz\",\n" + " \"time\" : 1678853101621\n" + "}"; - Assert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); + Assert.assertEquals(JSON_MAPPER.readTree(expectedJson), JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockboxTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockboxTest.java index 01dfc0f5d67d..08d8650bc1f5 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockboxTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockboxTest.java @@ -82,6 +82,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -1193,11 +1194,11 @@ public void testGetLockedIntervals() Assert.assertEquals(2, lockedIntervals.size()); Assert.assertEquals( - Arrays.asList( + new HashSet<>(Arrays.asList( Intervals.of("2017-01-01/2017-02-01"), Intervals.of("2017-04-01/2017-05-01") - ), - lockedIntervals.get(task1.getDataSource()) + )), + new HashSet<>(lockedIntervals.get(task1.getDataSource())) ); Assert.assertEquals( diff --git a/server/src/test/java/org/apache/druid/discovery/BaseNodeRoleWatcherTest.java b/server/src/test/java/org/apache/druid/discovery/BaseNodeRoleWatcherTest.java index 1bf5ba25040b..6e874e481a3c 100644 --- a/server/src/test/java/org/apache/druid/discovery/BaseNodeRoleWatcherTest.java +++ b/server/src/test/java/org/apache/druid/discovery/BaseNodeRoleWatcherTest.java @@ -128,7 +128,7 @@ private DiscoveryDruidNode buildDiscoveryDruidNode(NodeRole role, String host) private void assertListener(TestListener listener, boolean nodeViewInitialized, List nodesAdded, List nodesRemoved) { Assert.assertEquals(nodeViewInitialized, listener.nodeViewInitialized.get()); - Assert.assertEquals(nodesAdded, listener.nodesAddedList); + Assert.assertEquals(new HashSet<>(nodesAdded), new HashSet<>(listener.nodesAddedList)); Assert.assertEquals(nodesRemoved, listener.nodesRemovedList); } diff --git a/server/src/test/java/org/apache/druid/indexing/overlord/supervisor/SupervisorStatusTest.java b/server/src/test/java/org/apache/druid/indexing/overlord/supervisor/SupervisorStatusTest.java index 4c370453a0bc..9a8543d1a830 100644 --- a/server/src/test/java/org/apache/druid/indexing/overlord/supervisor/SupervisorStatusTest.java +++ b/server/src/test/java/org/apache/druid/indexing/overlord/supervisor/SupervisorStatusTest.java @@ -64,6 +64,6 @@ public void testJsonAttr() throws IOException Assert.assertEquals("wikipedia", deserialized.getId()); final String serialized = mapper.writeValueAsString(deserialized); Assert.assertTrue(serialized.contains("\"source\"")); - Assert.assertEquals(json, serialized); + Assert.assertEquals(mapper.readTree(json), mapper.readTree(serialized)); } } diff --git a/server/src/test/java/org/apache/druid/metadata/input/SqlEntityTest.java b/server/src/test/java/org/apache/druid/metadata/input/SqlEntityTest.java index ccd71cfaff5e..3f74841909b4 100644 --- a/server/src/test/java/org/apache/druid/metadata/input/SqlEntityTest.java +++ b/server/src/test/java/org/apache/druid/metadata/input/SqlEntityTest.java @@ -78,7 +78,7 @@ public void testExecuteQuery() throws IOException String expectedJson = mapper.writeValueAsString( Collections.singletonList(((MapBasedInputRow) expectedRow).getEvent()) ); - Assert.assertEquals(actualJson, expectedJson); + Assert.assertEquals(mapper.readTree(actualJson), mapper.readTree(expectedJson)); testUtils.dropTable(TABLE_NAME_1); } diff --git a/services/src/test/java/org/apache/druid/server/router/TieredBrokerHostSelectorTest.java b/services/src/test/java/org/apache/druid/server/router/TieredBrokerHostSelectorTest.java index 8dbf03292360..c6dee2b99d18 100644 --- a/services/src/test/java/org/apache/druid/server/router/TieredBrokerHostSelectorTest.java +++ b/services/src/test/java/org/apache/druid/server/router/TieredBrokerHostSelectorTest.java @@ -55,6 +55,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -367,18 +368,18 @@ public void testGetAllBrokers() { Assert.assertEquals( ImmutableMap.of( - "mediumBroker", ImmutableList.of(), - "coldBroker", ImmutableList.of("coldHost1:8080", "coldHost2:8080"), - "hotBroker", ImmutableList.of("hotHost:8080") + "mediumBroker", ImmutableSet.of(), + "coldBroker", ImmutableSet.of("coldHost1:8080", "coldHost2:8080"), + "hotBroker", ImmutableSet.of("hotHost:8080") ), Maps.transformValues( brokerSelector.getAllBrokers(), - new Function, List>() + new Function, HashSet>() { @Override - public List apply(@Nullable List servers) + public HashSet apply(@Nullable List servers) { - return Lists.transform(servers, server -> server.getHost()); + return new HashSet<>(Lists.transform(servers, server -> server.getHost())); } } ) From 8578b687a87ee99dc8fb4a940a979c1dcf59bd0a Mon Sep 17 00:00:00 2001 From: ythorat2 Date: Fri, 1 Dec 2023 17:06:43 -0600 Subject: [PATCH 6/7] remove unnessary changes for wrongly claimed flaky test --- .../druid/data/input/parquet/CompatParquetReaderTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetReaderTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetReaderTest.java index ccd48cfa3987..51cd2a7d39cf 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetReaderTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetReaderTest.java @@ -141,7 +141,7 @@ public void testParquet1217() throws IOException final String expectedJson = "{\n" + " \"col\" : -1\n" + "}"; - Assert.assertEquals(JSON_MAPPER.readTree(expectedJson), JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); + Assert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); } @Test @@ -340,7 +340,7 @@ public void testOldRepeatedInt() throws IOException final String expectedJson = "{\n" + " \"repeatedInt\" : [ 1, 2, 3 ]\n" + "}"; - Assert.assertEquals(JSON_MAPPER.readTree(expectedJson), JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); + Assert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); } @Test From 120291a9a3796fa2b48a6d69d5421e8a75e83161 Mon Sep 17 00:00:00 2001 From: rRajivramachandran Date: Sat, 2 Dec 2023 12:45:25 -0600 Subject: [PATCH 7/7] Fix flaky tests in classesCompatParquetReaderTest and FlattenSpecParquetReaderTest. --- .../input/parquet/BaseParquetReaderTest.java | 1 + .../parquet/CompatParquetReaderTest.java | 17 ++++++++-------- .../parquet/FlattenSpecParquetReaderTest.java | 20 +++++++++---------- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/BaseParquetReaderTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/BaseParquetReaderTest.java index 25fc2e23554f..0658661fb763 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/BaseParquetReaderTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/BaseParquetReaderTest.java @@ -39,6 +39,7 @@ class BaseParquetReaderTest extends InitializedNullHandlingTest { ObjectWriter DEFAULT_JSON_WRITER = new ObjectMapper().writerWithDefaultPrettyPrinter(); + protected final ObjectMapper objectMapper = new ObjectMapper(); InputEntityReader createReader(String parquetFile, InputRowSchema schema, JSONPathSpec flattenSpec) { diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetReaderTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetReaderTest.java index 51cd2a7d39cf..733461d0a6ed 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetReaderTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetReaderTest.java @@ -19,7 +19,6 @@ package org.apache.druid.data.input.parquet; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import org.apache.druid.data.input.ColumnsFilter; import org.apache.druid.data.input.InputEntityReader; @@ -43,8 +42,6 @@ */ public class CompatParquetReaderTest extends BaseParquetReaderTest { - private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); - @Test public void testBinaryAsString() throws IOException { @@ -97,12 +94,14 @@ public void testBinaryAsString() throws IOException + " \"field\" : \"hey this is &é(-è_çà)=^$ù*! Ω^^\",\n" + " \"ts\" : 1471800234\n" + "}"; - Assert.assertEquals(JSON_MAPPER.readTree(expectedJson), JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); + Assert.assertEquals(objectMapper.readTree(expectedJson), objectMapper.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); final String expectedJsonBinary = "{\n" + " \"field\" : \"aGV5IHRoaXMgaXMgJsOpKC3DqF/Dp8OgKT1eJMO5KiEgzqleXg==\",\n" + " \"ts\" : 1471800234\n" - + "}"; - Assert.assertEquals(JSON_MAPPER.readTree(expectedJsonBinary), JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampledAsBinary.get(0).getRawValues()))); + + "}"; + Assert.assertEquals( + objectMapper.readTree(expectedJsonBinary), + objectMapper.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampledAsBinary.get(0).getRawValues()))); } @Test @@ -303,7 +302,7 @@ required group nestedIntsColumn (LIST) { + " } ]\n" + " }\n" + "}"; - Assert.assertEquals(JSON_MAPPER.readTree(expectedJson), JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); + Assert.assertEquals(objectMapper.readTree(expectedJson), objectMapper.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); } @Test @@ -382,7 +381,7 @@ public void testReadNestedArrayStruct() throws IOException + " \"repeatedMessage\" : [ 3 ]\n" + " } ]\n" + "}"; - Assert.assertEquals(JSON_MAPPER.readTree(expectedJson), JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); + Assert.assertEquals(objectMapper.readTree(expectedJson), objectMapper.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); } @Test @@ -432,6 +431,6 @@ public void testProtoStructWithArray() throws IOException + " \"someId\" : 9\n" + " }\n" + "}"; - Assert.assertEquals(JSON_MAPPER.readTree(expectedJson), JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); + Assert.assertEquals(objectMapper.readTree(expectedJson), objectMapper.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); } } diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetReaderTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetReaderTest.java index d6c104124538..325d3431fe00 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetReaderTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetReaderTest.java @@ -19,7 +19,6 @@ package org.apache.druid.data.input.parquet; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import org.apache.druid.data.input.ColumnsFilter; import org.apache.druid.data.input.InputEntityReader; @@ -94,7 +93,7 @@ public void testFlat1NoFlattenSpec() throws IOException flattenSpec ); List sampled = sampleAllRows(reader); - Assert.assertEquals(FLAT_JSON, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); + Assert.assertEquals(objectMapper.readTree(FLAT_JSON), objectMapper.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); } @Test @@ -127,7 +126,7 @@ public void testFlat1Autodiscover() throws IOException JSONPathSpec.DEFAULT ); List sampled = sampleAllRows(reader); - Assert.assertEquals(FLAT_JSON, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); + Assert.assertEquals(objectMapper.readTree(FLAT_JSON), objectMapper.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); } @Test @@ -168,7 +167,7 @@ public void testFlat1Flatten() throws IOException flattenSpec ); List sampled = sampleAllRows(reader); - Assert.assertEquals(FLAT_JSON, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); + Assert.assertEquals(objectMapper.readTree(FLAT_JSON), objectMapper.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); } @Test @@ -205,10 +204,9 @@ public void testFlat1FlattenSelectListItem() throws IOException schema, flattenSpec ); - List sampled = sampleAllRows(reader); - ObjectMapper mapper = new ObjectMapper(); - Assert.assertEquals(mapper.readTree(FLAT_JSON), mapper.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); + List sampled = sampleAllRows(reader); + Assert.assertEquals(objectMapper.readTree(FLAT_JSON), objectMapper.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); } @@ -245,7 +243,7 @@ public void testNested1NoFlattenSpec() throws IOException flattenSpec ); List sampled = sampleAllRows(reader); - Assert.assertEquals(NESTED_JSON, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); + Assert.assertEquals(objectMapper.readTree(NESTED_JSON), objectMapper.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); } @Test @@ -278,7 +276,7 @@ public void testNested1Autodiscover() throws IOException JSONPathSpec.DEFAULT ); List sampled = sampleAllRows(reader); - Assert.assertEquals(NESTED_JSON, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); + Assert.assertEquals(objectMapper.readTree(NESTED_JSON), objectMapper.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); } @Test @@ -321,7 +319,7 @@ public void testNested1Flatten() throws IOException flattenSpec ); List sampled = sampleAllRows(reader); - Assert.assertEquals(NESTED_JSON, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); + Assert.assertEquals(objectMapper.readTree(NESTED_JSON), objectMapper.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); } @Test @@ -362,7 +360,7 @@ public void testNested1FlattenSelectListItem() throws IOException flattenSpec ); List sampled = sampleAllRows(reader); - Assert.assertEquals(NESTED_JSON, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); + Assert.assertEquals(objectMapper.readTree(NESTED_JSON), objectMapper.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); } @Test