From 70564b6bbbfa48be0865a571b40520762f2ab872 Mon Sep 17 00:00:00 2001 From: lakkidi2 Date: Fri, 1 Sep 2023 12:42:56 -0500 Subject: [PATCH 1/8] fix the flaky test --- .../input/parquet/TimestampsParquetReaderTest.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java index c0189fe8bc19..0b151c0016f6 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java @@ -19,6 +19,8 @@ package org.apache.druid.data.input.parquet; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import org.apache.druid.data.input.ColumnsFilter; import org.apache.druid.data.input.InputEntityReader; @@ -84,15 +86,21 @@ public void testDateHandling() throws IOException ); List sampledAsString = sampleAllRows(readerAsString); List sampledAsDate = sampleAllRows(readerAsDate); - final String expectedJson = "{\n" + final String expectedJsonString = "{\n" + " \"date_as_string\" : \"2017-06-18\",\n" + " \"timestamp_as_timestamp\" : 1497702471815,\n" + " \"timestamp_as_string\" : \"2017-06-17 14:27:51.815\",\n" + " \"idx\" : 1,\n" + " \"date_as_date\" : 1497744000000\n" + "}"; - Assert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampledAsString.get(0).getRawValues())); - Assert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampledAsDate.get(0).getRawValues())); + + ObjectMapper obj = new ObjectMapper(); + JsonNode expectedJson = obj.readTree(expectedJsonString); + + JsonNode sampledAsStringJson = obj.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampledAsString.get(0).getRawValues())); + JsonNode sampledAsDateJson = obj.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampledAsDate.get(0).getRawValues())); + Assert.assertEquals(expectedJson, sampledAsStringJson); + Assert.assertEquals(expectedJson, sampledAsDateJson); } @Test From 4d7a82d22fcea3ba31fbebd95c576d57bec88f31 Mon Sep 17 00:00:00 2001 From: prathyushreddylpr <112648268+prathyushreddylpr@users.noreply.github.com> Date: Thu, 12 Oct 2023 20:57:11 -0500 Subject: [PATCH 2/8] Update FlattenSpecParquetReaderTest.java --- .../parquet/FlattenSpecParquetReaderTest.java | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetReaderTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetReaderTest.java index 726ecde4adf3..d9d16ed52ba6 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetReaderTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetReaderTest.java @@ -19,6 +19,8 @@ package org.apache.druid.data.input.parquet; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import org.apache.druid.data.input.ColumnsFilter; import org.apache.druid.data.input.InputEntityReader; @@ -34,7 +36,11 @@ import org.junit.Test; import java.io.IOException; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.SortedSet; +import java.util.TreeSet; /** * Duplicate of {@link FlattenSpecParquetInputTest} but for {@link ParquetReader} instead of Hadoop @@ -42,10 +48,10 @@ public class FlattenSpecParquetReaderTest extends BaseParquetReaderTest { private static final String FLAT_JSON = "{\n" - + " \"listDim\" : [ \"listDim1v1\", \"listDim1v2\" ],\n" - + " \"dim3\" : 1,\n" - + " \"dim2\" : \"d2v1\",\n" + " \"dim1\" : \"d1v1\",\n" + + " \"dim2\" : \"d2v1\",\n" + + " \"dim3\" : 1,\n" + + " \"listDim\" : [ \"listDim1v1\", \"listDim1v2\" ],\n" + " \"metric1\" : 1,\n" + " \"timestamp\" : 1537229880023\n" + "}"; @@ -93,7 +99,12 @@ public void testFlat1NoFlattenSpec() throws IOException flattenSpec ); List sampled = sampleAllRows(reader); - Assert.assertEquals(FLAT_JSON, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); + + ObjectMapper obj = new ObjectMapper(); + JsonNode expectedJson = obj.readTree(FLAT_JSON); + + JsonNode sampledAsStringJson = obj.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); + Assert.assertEquals(expectedJson, sampledAsStringJson); } @Test @@ -205,8 +216,14 @@ public void testFlat1FlattenSelectListItem() throws IOException flattenSpec ); List sampled = sampleAllRows(reader); - - Assert.assertEquals(FLAT_JSON, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); + Map map = sampled.get(0).getRawValues(); + SortedSet keys = new TreeSet<>(map.keySet()); + Map newMap = new LinkedHashMap(); + for (String key : keys) { + Object value = map.get(key); + newMap.put(key, value); + } + Assert.assertEquals(FLAT_JSON, DEFAULT_JSON_WRITER.writeValueAsString(newMap)); } From 5054fd7a640378be879507e056c8725af3385986 Mon Sep 17 00:00:00 2001 From: prathyushreddylpr <112648268+prathyushreddylpr@users.noreply.github.com> Date: Fri, 13 Oct 2023 00:46:49 -0500 Subject: [PATCH 3/8] Update FlattenSpecParquetReaderTest.java --- .../parquet/FlattenSpecParquetReaderTest.java | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetReaderTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetReaderTest.java index d9d16ed52ba6..3c86cc084cf4 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetReaderTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetReaderTest.java @@ -36,11 +36,7 @@ import org.junit.Test; import java.io.IOException; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; -import java.util.SortedSet; -import java.util.TreeSet; /** * Duplicate of {@link FlattenSpecParquetInputTest} but for {@link ParquetReader} instead of Hadoop @@ -48,10 +44,10 @@ public class FlattenSpecParquetReaderTest extends BaseParquetReaderTest { private static final String FLAT_JSON = "{\n" - + " \"dim1\" : \"d1v1\",\n" - + " \"dim2\" : \"d2v1\",\n" - + " \"dim3\" : 1,\n" + " \"listDim\" : [ \"listDim1v1\", \"listDim1v2\" ],\n" + + " \"dim3\" : 1,\n" + + " \"dim2\" : \"d2v1\",\n" + + " \"dim1\" : \"d1v1\",\n" + " \"metric1\" : 1,\n" + " \"timestamp\" : 1537229880023\n" + "}"; @@ -216,14 +212,11 @@ public void testFlat1FlattenSelectListItem() throws IOException flattenSpec ); List sampled = sampleAllRows(reader); - Map map = sampled.get(0).getRawValues(); - SortedSet keys = new TreeSet<>(map.keySet()); - Map newMap = new LinkedHashMap(); - for (String key : keys) { - Object value = map.get(key); - newMap.put(key, value); - } - Assert.assertEquals(FLAT_JSON, DEFAULT_JSON_WRITER.writeValueAsString(newMap)); + ObjectMapper obj = new ObjectMapper(); + JsonNode expectedJson = obj.readTree(FLAT_JSON); + + JsonNode sampledAsStringJson = obj.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); + Assert.assertEquals(expectedJson, sampledAsStringJson); } From 11a7ad17b20c4ddddfaaf6e860d4485f32c81ae8 Mon Sep 17 00:00:00 2001 From: prathyushreddylpr <112648268+prathyushreddylpr@users.noreply.github.com> Date: Fri, 10 Nov 2023 18:58:20 -0600 Subject: [PATCH 4/8] Update FlattenSpecParquetReaderTest.java --- .../data/input/parquet/FlattenSpecParquetReaderTest.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetReaderTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetReaderTest.java index 3c86cc084cf4..3a01c558159b 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetReaderTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetReaderTest.java @@ -96,11 +96,7 @@ public void testFlat1NoFlattenSpec() throws IOException ); List sampled = sampleAllRows(reader); - ObjectMapper obj = new ObjectMapper(); - JsonNode expectedJson = obj.readTree(FLAT_JSON); - - JsonNode sampledAsStringJson = obj.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); - Assert.assertEquals(expectedJson, sampledAsStringJson); + Assert.assertEquals(FLAT_JSON, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); } @Test From abb838eda15ef0970fee85d2806b4f8f9df1b7d4 Mon Sep 17 00:00:00 2001 From: lxb007981 Date: Sun, 19 Nov 2023 15:28:19 -0600 Subject: [PATCH 5/8] Fix 6 flaky tests using JSON node comparison --- .../parquet/CompatParquetReaderTest.java | 48 ++++++++++++++----- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetReaderTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetReaderTest.java index f94949d6bd21..47d1935741e2 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetReaderTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetReaderTest.java @@ -19,6 +19,8 @@ package org.apache.druid.data.input.parquet; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import org.apache.druid.data.input.ColumnsFilter; import org.apache.druid.data.input.InputEntityReader; @@ -90,20 +92,24 @@ public void testBinaryAsString() throws IOException ); List sampled = sampleAllRows(reader); List sampledAsBinary = sampleAllRows(readerNotAsString); + ObjectMapper objectMapper = new ObjectMapper(); final String expectedJson = "{\n" + " \"field\" : \"hey this is &é(-è_çà)=^$ù*! Ω^^\",\n" + " \"ts\" : 1471800234\n" + "}"; - Assert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); - + final String serializedJson = DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()); + JsonNode expected = objectMapper.readTree(expectedJson); + JsonNode serialized = objectMapper.readTree(serializedJson); + Assert.assertTrue(expected.equals(serialized)); final String expectedJsonBinary = "{\n" + " \"field\" : \"aGV5IHRoaXMgaXMgJsOpKC3DqF/Dp8OgKT1eJMO5KiEgzqleXg==\",\n" + " \"ts\" : 1471800234\n" + "}"; - Assert.assertEquals( - expectedJsonBinary, - DEFAULT_JSON_WRITER.writeValueAsString(sampledAsBinary.get(0).getRawValues()) - ); + + final String serializedJsonBinary = DEFAULT_JSON_WRITER.writeValueAsString(sampledAsBinary.get(0).getRawValues()); + JsonNode expectedBinary = objectMapper.readTree(expectedJsonBinary); + JsonNode serializedBinary = objectMapper.readTree(serializedJsonBinary); + Assert.assertTrue(expectedBinary.equals(serializedBinary)); } @@ -140,10 +146,14 @@ public void testParquet1217() throws IOException flattenSpec ); List sampled = sampleAllRows(reader); + ObjectMapper objectMapper = new ObjectMapper(); final String expectedJson = "{\n" + " \"col\" : -1\n" + "}"; - Assert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); + final String serializedJson = DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()); + JsonNode expected = objectMapper.readTree(expectedJson); + JsonNode serialized = objectMapper.readTree(serializedJson); + Assert.assertTrue(expected.equals(serialized)); } @Test @@ -246,6 +256,7 @@ required group nestedIntsColumn (LIST) { flattenSpec ); List sampled = sampleAllRows(reader); + ObjectMapper objectMapper = new ObjectMapper(); final String expectedJson = "{\n" + " \"enumColumn\" : \"SPADES\",\n" + " \"maybeStringColumn\" : null,\n" @@ -305,7 +316,10 @@ required group nestedIntsColumn (LIST) { + " } ]\n" + " }\n" + "}"; - Assert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); + final String serializedJson = DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()); + JsonNode expected = objectMapper.readTree(expectedJson); + JsonNode serialized = objectMapper.readTree(serializedJson); + Assert.assertTrue(expected.equals(serialized)); } @Test @@ -339,10 +353,14 @@ public void testOldRepeatedInt() throws IOException flattenSpec ); List sampled = sampleAllRows(reader); + ObjectMapper objectMapper = new ObjectMapper(); final String expectedJson = "{\n" + " \"repeatedInt\" : [ 1, 2, 3 ]\n" + "}"; - Assert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); + final String serializedJson = DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()); + JsonNode expected = objectMapper.readTree(expectedJson); + JsonNode serialized = objectMapper.readTree(serializedJson); + Assert.assertTrue(expected.equals(serialized)); } @@ -378,6 +396,7 @@ public void testReadNestedArrayStruct() throws IOException flattenSpec ); List sampled = sampleAllRows(reader); + ObjectMapper objectMapper = new ObjectMapper(); final String expectedJson = "{\n" + " \"primitive\" : 2,\n" + " \"myComplex\" : [ {\n" @@ -385,7 +404,10 @@ public void testReadNestedArrayStruct() throws IOException + " \"repeatedMessage\" : [ 3 ]\n" + " } ]\n" + "}"; - Assert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); + final String serializedJson = DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()); + JsonNode expected = objectMapper.readTree(expectedJson); + JsonNode serialized = objectMapper.readTree(serializedJson); + Assert.assertTrue(expected.equals(serialized)); } @Test @@ -425,6 +447,7 @@ public void testProtoStructWithArray() throws IOException flattenSpec ); List sampled = sampleAllRows(reader); + ObjectMapper objectMapper = new ObjectMapper(); final String expectedJson = "{\n" + " \"optionalMessage\" : null,\n" + " \"requiredPrimitive\" : 9,\n" @@ -435,6 +458,9 @@ public void testProtoStructWithArray() throws IOException + " \"someId\" : 9\n" + " }\n" + "}"; - Assert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); + final String serializedJson = DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()); + JsonNode expected = objectMapper.readTree(expectedJson); + JsonNode serialized = objectMapper.readTree(serializedJson); + Assert.assertTrue(expected.equals(serialized)); } } From 111acaac0552887c9620ea7b4994dada2b1d7d58 Mon Sep 17 00:00:00 2001 From: Prathyush Reddy Lakkidi Date: Thu, 30 Nov 2023 17:47:04 -0600 Subject: [PATCH 6/8] resolve comments --- .../parquet/CompatParquetReaderTest.java | 48 +++++-------------- .../parquet/FlattenSpecParquetReaderTest.java | 8 +--- .../parquet/TimestampsParquetReaderTest.java | 12 ++--- 3 files changed, 17 insertions(+), 51 deletions(-) diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetReaderTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetReaderTest.java index 47d1935741e2..408100f9e36c 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetReaderTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetReaderTest.java @@ -19,7 +19,6 @@ package org.apache.druid.data.input.parquet; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import org.apache.druid.data.input.ColumnsFilter; @@ -44,6 +43,8 @@ */ public class CompatParquetReaderTest extends BaseParquetReaderTest { + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + @Test public void testBinaryAsString() throws IOException { @@ -92,24 +93,17 @@ public void testBinaryAsString() throws IOException ); List sampled = sampleAllRows(reader); List sampledAsBinary = sampleAllRows(readerNotAsString); - ObjectMapper objectMapper = new ObjectMapper(); final String expectedJson = "{\n" + " \"field\" : \"hey this is &é(-è_çà)=^$ù*! Ω^^\",\n" + " \"ts\" : 1471800234\n" + "}"; - final String serializedJson = DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()); - JsonNode expected = objectMapper.readTree(expectedJson); - JsonNode serialized = objectMapper.readTree(serializedJson); - Assert.assertTrue(expected.equals(serialized)); + Assert.assertTrue(JSON_MAPPER.readTree(expectedJson).equals(JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())))); + final String expectedJsonBinary = "{\n" + " \"field\" : \"aGV5IHRoaXMgaXMgJsOpKC3DqF/Dp8OgKT1eJMO5KiEgzqleXg==\",\n" + " \"ts\" : 1471800234\n" - + "}"; - - final String serializedJsonBinary = DEFAULT_JSON_WRITER.writeValueAsString(sampledAsBinary.get(0).getRawValues()); - JsonNode expectedBinary = objectMapper.readTree(expectedJsonBinary); - JsonNode serializedBinary = objectMapper.readTree(serializedJsonBinary); - Assert.assertTrue(expectedBinary.equals(serializedBinary)); + + "}"; + Assert.assertTrue(JSON_MAPPER.readTree(expectedJsonBinary).equals(JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampledAsBinary.get(0).getRawValues())))); } @@ -146,14 +140,10 @@ public void testParquet1217() throws IOException flattenSpec ); List sampled = sampleAllRows(reader); - ObjectMapper objectMapper = new ObjectMapper(); final String expectedJson = "{\n" + " \"col\" : -1\n" + "}"; - final String serializedJson = DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()); - JsonNode expected = objectMapper.readTree(expectedJson); - JsonNode serialized = objectMapper.readTree(serializedJson); - Assert.assertTrue(expected.equals(serialized)); + Assert.assertTrue(JSON_MAPPER.readTree(expectedJson).equals(JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())))); } @Test @@ -256,7 +246,6 @@ required group nestedIntsColumn (LIST) { flattenSpec ); List sampled = sampleAllRows(reader); - ObjectMapper objectMapper = new ObjectMapper(); final String expectedJson = "{\n" + " \"enumColumn\" : \"SPADES\",\n" + " \"maybeStringColumn\" : null,\n" @@ -316,10 +305,7 @@ required group nestedIntsColumn (LIST) { + " } ]\n" + " }\n" + "}"; - final String serializedJson = DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()); - JsonNode expected = objectMapper.readTree(expectedJson); - JsonNode serialized = objectMapper.readTree(serializedJson); - Assert.assertTrue(expected.equals(serialized)); + Assert.assertTrue(JSON_MAPPER.readTree(expectedJson).equals(JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())))); } @Test @@ -353,14 +339,10 @@ public void testOldRepeatedInt() throws IOException flattenSpec ); List sampled = sampleAllRows(reader); - ObjectMapper objectMapper = new ObjectMapper(); final String expectedJson = "{\n" + " \"repeatedInt\" : [ 1, 2, 3 ]\n" + "}"; - final String serializedJson = DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()); - JsonNode expected = objectMapper.readTree(expectedJson); - JsonNode serialized = objectMapper.readTree(serializedJson); - Assert.assertTrue(expected.equals(serialized)); + Assert.assertTrue(JSON_MAPPER.readTree(expectedJson).equals(JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())))); } @@ -396,7 +378,6 @@ public void testReadNestedArrayStruct() throws IOException flattenSpec ); List sampled = sampleAllRows(reader); - ObjectMapper objectMapper = new ObjectMapper(); final String expectedJson = "{\n" + " \"primitive\" : 2,\n" + " \"myComplex\" : [ {\n" @@ -404,10 +385,7 @@ public void testReadNestedArrayStruct() throws IOException + " \"repeatedMessage\" : [ 3 ]\n" + " } ]\n" + "}"; - final String serializedJson = DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()); - JsonNode expected = objectMapper.readTree(expectedJson); - JsonNode serialized = objectMapper.readTree(serializedJson); - Assert.assertTrue(expected.equals(serialized)); + Assert.assertTrue(JSON_MAPPER.readTree(expectedJson).equals(JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())))); } @Test @@ -447,7 +425,6 @@ public void testProtoStructWithArray() throws IOException flattenSpec ); List sampled = sampleAllRows(reader); - ObjectMapper objectMapper = new ObjectMapper(); final String expectedJson = "{\n" + " \"optionalMessage\" : null,\n" + " \"requiredPrimitive\" : 9,\n" @@ -458,9 +435,6 @@ public void testProtoStructWithArray() throws IOException + " \"someId\" : 9\n" + " }\n" + "}"; - final String serializedJson = DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()); - JsonNode expected = objectMapper.readTree(expectedJson); - JsonNode serialized = objectMapper.readTree(serializedJson); - Assert.assertTrue(expected.equals(serialized)); + Assert.assertTrue(JSON_MAPPER.readTree(expectedJson).equals(JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())))); } } diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetReaderTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetReaderTest.java index 3a01c558159b..d6c104124538 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetReaderTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetReaderTest.java @@ -19,7 +19,6 @@ package org.apache.druid.data.input.parquet; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import org.apache.druid.data.input.ColumnsFilter; @@ -95,7 +94,6 @@ public void testFlat1NoFlattenSpec() throws IOException flattenSpec ); List sampled = sampleAllRows(reader); - Assert.assertEquals(FLAT_JSON, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); } @@ -208,11 +206,9 @@ public void testFlat1FlattenSelectListItem() throws IOException flattenSpec ); List sampled = sampleAllRows(reader); - ObjectMapper obj = new ObjectMapper(); - JsonNode expectedJson = obj.readTree(FLAT_JSON); + ObjectMapper mapper = new ObjectMapper(); - JsonNode sampledAsStringJson = obj.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())); - Assert.assertEquals(expectedJson, sampledAsStringJson); + Assert.assertEquals(mapper.readTree(FLAT_JSON), mapper.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); } diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java index 0b151c0016f6..def6cdd7061d 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java @@ -19,7 +19,6 @@ package org.apache.druid.data.input.parquet; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import org.apache.druid.data.input.ColumnsFilter; @@ -86,7 +85,7 @@ public void testDateHandling() throws IOException ); List sampledAsString = sampleAllRows(readerAsString); List sampledAsDate = sampleAllRows(readerAsDate); - final String expectedJsonString = "{\n" + final String expectedJson = "{\n" + " \"date_as_string\" : \"2017-06-18\",\n" + " \"timestamp_as_timestamp\" : 1497702471815,\n" + " \"timestamp_as_string\" : \"2017-06-17 14:27:51.815\",\n" @@ -94,13 +93,10 @@ public void testDateHandling() throws IOException + " \"date_as_date\" : 1497744000000\n" + "}"; - ObjectMapper obj = new ObjectMapper(); - JsonNode expectedJson = obj.readTree(expectedJsonString); + ObjectMapper mapper = new ObjectMapper(); - JsonNode sampledAsStringJson = obj.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampledAsString.get(0).getRawValues())); - JsonNode sampledAsDateJson = obj.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampledAsDate.get(0).getRawValues())); - Assert.assertEquals(expectedJson, sampledAsStringJson); - Assert.assertEquals(expectedJson, sampledAsDateJson); + Assert.assertEquals(mapper.readTree(expectedJson), mapper.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampledAsString.get(0).getRawValues()))); + Assert.assertEquals(mapper.readTree(expectedJson), mapper.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampledAsDate.get(0).getRawValues()))); } @Test From b803fac08de3402f97e50023f4096872b0e02056 Mon Sep 17 00:00:00 2001 From: Prathyush Reddy Lakkidi Date: Thu, 30 Nov 2023 17:49:49 -0600 Subject: [PATCH 7/8] remove space --- .../druid/data/input/parquet/TimestampsParquetReaderTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java index def6cdd7061d..5a4dec40c120 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java @@ -92,9 +92,7 @@ public void testDateHandling() throws IOException + " \"idx\" : 1,\n" + " \"date_as_date\" : 1497744000000\n" + "}"; - ObjectMapper mapper = new ObjectMapper(); - Assert.assertEquals(mapper.readTree(expectedJson), mapper.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampledAsString.get(0).getRawValues()))); Assert.assertEquals(mapper.readTree(expectedJson), mapper.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampledAsDate.get(0).getRawValues()))); } From 9bea28083024895fef873f2dcdeee5fb8e392d50 Mon Sep 17 00:00:00 2001 From: Prathyush Reddy Lakkidi Date: Thu, 30 Nov 2023 18:04:56 -0600 Subject: [PATCH 8/8] change asserttrue to assertequals --- .../parquet/CompatParquetReaderTest.java | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetReaderTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetReaderTest.java index 408100f9e36c..ccd48cfa3987 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetReaderTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetReaderTest.java @@ -44,7 +44,7 @@ public class CompatParquetReaderTest extends BaseParquetReaderTest { private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); - + @Test public void testBinaryAsString() throws IOException { @@ -97,16 +97,14 @@ public void testBinaryAsString() throws IOException + " \"field\" : \"hey this is &é(-è_çà)=^$ù*! Ω^^\",\n" + " \"ts\" : 1471800234\n" + "}"; - Assert.assertTrue(JSON_MAPPER.readTree(expectedJson).equals(JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())))); - + Assert.assertEquals(JSON_MAPPER.readTree(expectedJson), JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); final String expectedJsonBinary = "{\n" + " \"field\" : \"aGV5IHRoaXMgaXMgJsOpKC3DqF/Dp8OgKT1eJMO5KiEgzqleXg==\",\n" + " \"ts\" : 1471800234\n" + "}"; - Assert.assertTrue(JSON_MAPPER.readTree(expectedJsonBinary).equals(JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampledAsBinary.get(0).getRawValues())))); + Assert.assertEquals(JSON_MAPPER.readTree(expectedJsonBinary), JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampledAsBinary.get(0).getRawValues()))); } - @Test public void testParquet1217() throws IOException { @@ -143,7 +141,7 @@ public void testParquet1217() throws IOException final String expectedJson = "{\n" + " \"col\" : -1\n" + "}"; - Assert.assertTrue(JSON_MAPPER.readTree(expectedJson).equals(JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())))); + Assert.assertEquals(JSON_MAPPER.readTree(expectedJson), JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); } @Test @@ -305,7 +303,7 @@ required group nestedIntsColumn (LIST) { + " } ]\n" + " }\n" + "}"; - Assert.assertTrue(JSON_MAPPER.readTree(expectedJson).equals(JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())))); + Assert.assertEquals(JSON_MAPPER.readTree(expectedJson), JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); } @Test @@ -342,10 +340,9 @@ public void testOldRepeatedInt() throws IOException final String expectedJson = "{\n" + " \"repeatedInt\" : [ 1, 2, 3 ]\n" + "}"; - Assert.assertTrue(JSON_MAPPER.readTree(expectedJson).equals(JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())))); + Assert.assertEquals(JSON_MAPPER.readTree(expectedJson), JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); } - @Test public void testReadNestedArrayStruct() throws IOException { @@ -385,7 +382,7 @@ public void testReadNestedArrayStruct() throws IOException + " \"repeatedMessage\" : [ 3 ]\n" + " } ]\n" + "}"; - Assert.assertTrue(JSON_MAPPER.readTree(expectedJson).equals(JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())))); + Assert.assertEquals(JSON_MAPPER.readTree(expectedJson), JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); } @Test @@ -435,6 +432,6 @@ public void testProtoStructWithArray() throws IOException + " \"someId\" : 9\n" + " }\n" + "}"; - Assert.assertTrue(JSON_MAPPER.readTree(expectedJson).equals(JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues())))); + Assert.assertEquals(JSON_MAPPER.readTree(expectedJson), JSON_MAPPER.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()))); } }