From e2eb7b5754bec7260a7ab6503a85a40ee06cf710 Mon Sep 17 00:00:00 2001 From: ycp2 Date: Tue, 26 Sep 2023 23:14:43 -0500 Subject: [PATCH 1/4] fix flaky TimestampsParquetReaderTest with JSONAssert --- extensions-core/parquet-extensions/pom.xml | 11 +++++++++++ .../input/parquet/TimestampsParquetReaderTest.java | 11 +++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/extensions-core/parquet-extensions/pom.xml b/extensions-core/parquet-extensions/pom.xml index d5b882d60e49..3d43e7359306 100644 --- a/extensions-core/parquet-extensions/pom.xml +++ b/extensions-core/parquet-extensions/pom.xml @@ -194,6 +194,17 @@ ${hadoop.compile.version} runtime + + org.json + json + 20210307 + + + org.skyscreamer + jsonassert + 1.5.1 + test + 1.13.0 diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java index c0189fe8bc19..fb38a5ec798b 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java @@ -34,6 +34,9 @@ import java.io.IOException; import java.util.List; +import org.json.JSONException; +import org.skyscreamer.jsonassert.JSONAssert; + /** * Duplicate of {@link TimestampsParquetInputTest} but for {@link ParquetReader} instead of Hadoop */ @@ -91,8 +94,12 @@ public void testDateHandling() throws IOException + " \"idx\" : 1,\n" + " \"date_as_date\" : 1497744000000\n" + "}"; - Assert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampledAsString.get(0).getRawValues())); - Assert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampledAsDate.get(0).getRawValues())); + try{ + JSONAssert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampledAsString.get(0).getRawValues()), false); + JSONAssert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampledAsDate.get(0).getRawValues()), false); + } catch (JSONException jse){ + Assert.fail("Not comparing JSON strings"); + } } @Test From e0d06071297b805111bd35a676dc99bd8588799e Mon Sep 17 00:00:00 2001 From: ycp2 Date: Thu, 28 Sep 2023 17:09:27 -0500 Subject: [PATCH 2/4] Matching java format --- .../data/input/parquet/TimestampsParquetReaderTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java index fb38a5ec798b..7d517dac9751 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java @@ -94,11 +94,11 @@ public void testDateHandling() throws IOException + " \"idx\" : 1,\n" + " \"date_as_date\" : 1497744000000\n" + "}"; - try{ + try { JSONAssert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampledAsString.get(0).getRawValues()), false); JSONAssert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampledAsDate.get(0).getRawValues()), false); - } catch (JSONException jse){ - Assert.fail("Not comparing JSON strings"); + } catch (JSONException jse) { + Assert.fail("Not comparing JSON strings. Error: " + jse.getMessage()); } } From a895ef1fa109e74f13e3c652c1b8e39392191f59 Mon Sep 17 00:00:00 2001 From: ycp2 Date: Thu, 26 Oct 2023 22:21:50 -0500 Subject: [PATCH 3/4] TimestampsParquetReaderTest checkstyle format changed --- .../input/parquet/TimestampsParquetReaderTest.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java index 7d517dac9751..263408e245ec 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java @@ -28,15 +28,14 @@ import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.java.util.common.parsers.JSONPathSpec; +import org.json.JSONException; import org.junit.Assert; import org.junit.Test; +import org.skyscreamer.jsonassert.JSONAssert; import java.io.IOException; import java.util.List; -import org.json.JSONException; -import org.skyscreamer.jsonassert.JSONAssert; - /** * Duplicate of {@link TimestampsParquetInputTest} but for {@link ParquetReader} instead of Hadoop */ @@ -95,10 +94,11 @@ public void testDateHandling() throws IOException + " \"date_as_date\" : 1497744000000\n" + "}"; try { - JSONAssert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampledAsString.get(0).getRawValues()), false); - JSONAssert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampledAsDate.get(0).getRawValues()), false); - } catch (JSONException jse) { - Assert.fail("Not comparing JSON strings. Error: " + jse.getMessage()); + JSONAssert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampledAsString.get(0).getRawValues()), false); + JSONAssert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampledAsDate.get(0).getRawValues()), false); + } + catch (JSONException jse) { + throw new RuntimeException(jse); } } From 00a37479b4e1c9a0e8db7a3626865fc3bf64eaef Mon Sep 17 00:00:00 2001 From: ycp2 Date: Tue, 7 Nov 2023 21:42:24 -0600 Subject: [PATCH 4/4] change fix with ObjectMapper instead of JSONAssert --- extensions-core/parquet-extensions/pom.xml | 11 ----------- .../input/parquet/TimestampsParquetReaderTest.java | 14 +++++--------- 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/extensions-core/parquet-extensions/pom.xml b/extensions-core/parquet-extensions/pom.xml index 3d43e7359306..d5b882d60e49 100644 --- a/extensions-core/parquet-extensions/pom.xml +++ b/extensions-core/parquet-extensions/pom.xml @@ -194,17 +194,6 @@ ${hadoop.compile.version} runtime - - org.json - json - 20210307 - - - org.skyscreamer - jsonassert - 1.5.1 - test - 1.13.0 diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java index 263408e245ec..56b4319c89fc 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetReaderTest.java @@ -19,6 +19,7 @@ package org.apache.druid.data.input.parquet; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import org.apache.druid.data.input.ColumnsFilter; import org.apache.druid.data.input.InputEntityReader; @@ -28,10 +29,8 @@ import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.java.util.common.parsers.JSONPathSpec; -import org.json.JSONException; import org.junit.Assert; import org.junit.Test; -import org.skyscreamer.jsonassert.JSONAssert; import java.io.IOException; import java.util.List; @@ -84,6 +83,8 @@ public void testDateHandling() throws IOException schemaAsDate, JSONPathSpec.DEFAULT ); + ObjectMapper objectMapperString = new ObjectMapper(); + ObjectMapper objectMapperDate = new ObjectMapper(); List sampledAsString = sampleAllRows(readerAsString); List sampledAsDate = sampleAllRows(readerAsDate); final String expectedJson = "{\n" @@ -93,13 +94,8 @@ public void testDateHandling() throws IOException + " \"idx\" : 1,\n" + " \"date_as_date\" : 1497744000000\n" + "}"; - try { - JSONAssert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampledAsString.get(0).getRawValues()), false); - JSONAssert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampledAsDate.get(0).getRawValues()), false); - } - catch (JSONException jse) { - throw new RuntimeException(jse); - } + Assert.assertEquals(objectMapperString.readTree(expectedJson), objectMapperString.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampledAsString.get(0).getRawValues()))); + Assert.assertEquals(objectMapperDate.readTree(expectedJson), objectMapperDate.readTree(DEFAULT_JSON_WRITER.writeValueAsString(sampledAsDate.get(0).getRawValues()))); } @Test