From 5c7fbc206ae1826452a469f6492bfe9755104cb2 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 29 Jul 2026 15:15:37 -0700 Subject: [PATCH] Detect integral SUM overflow and fix BIGINT AVG Integral SUM used Calcite's unchecked long accumulator and could silently wrap. BIGINT AVG could likewise overflow its intermediate long sum before division. Use CHECKED_LONG_SUM for integral inputs and Math.addExact during enumerable accumulation. Preserve SqlKind.SUM so planner rewrites and native OpenSearch sum pushdown remain unchanged. Range-check and narrow pushed results while retaining the backend's double semantics. Use a double sum and long count for enumerable BIGINT AVG. Preserve SqlKind.AVG and native avg pushdown. Add unit, integration, REST, documentation, and explain coverage for both execution paths. Signed-off-by: Kai Huang --- .../sql/api/UnifiedQueryPlannerSqlV2Test.java | 6 +- .../udf/udaf/BigintAvgAggFunction.java | 31 +++ .../udf/udaf/CheckedLongSumAggFunction.java | 22 ++ .../utils/UserDefinedFunctionUtils.java | 25 +- .../function/PPLBuiltinOperators.java | 18 +- .../expression/function/PPLFuncImpTable.java | 38 ++- .../udf/udaf/BigintAvgAggFunctionTest.java | 44 ++++ .../udaf/CheckedLongSumAggFunctionTest.java | 42 ++++ docs/user/ppl/functions/expressions.md | 10 +- .../remote/CalcitePPLAggregationIT.java | 175 ++++++++++++- .../opensearch/sql/ppl/StatsCommandIT.java | 5 - .../calcite/chart_null_str.yaml | 8 +- .../calcite/clickbench/q10.yaml | 4 +- .../expectedOutput/calcite/clickbench/q3.yaml | 4 +- .../calcite/clickbench/q30.yaml | 4 +- .../calcite/clickbench/q31.yaml | 4 +- .../calcite/clickbench/q32.yaml | 4 +- .../calcite/clickbench/q33.yaml | 4 +- .../calcite/explain_agg_sort_on_measure2.yaml | 4 +- .../calcite/explain_agg_sort_on_measure4.yaml | 4 +- .../explain_agg_sort_on_measure_complex1.yaml | 4 +- .../explain_agg_sort_on_measure_complex2.yaml | 4 +- ...t_on_measure_multi_buckets_not_pushed.yaml | 4 +- .../calcite/explain_agg_with_script.yaml | 4 +- .../explain_agg_with_sum_enhancement.yaml | 4 +- .../calcite/explain_bin_minspan.json | 7 +- ...gg_with_sort_on_one_measure_not_push1.yaml | 4 +- ...gg_with_sort_on_one_measure_not_push2.yaml | 4 +- .../calcite/explain_streamstats_global.yaml | 4 +- ...xplain_streamstats_global_null_bucket.yaml | 4 +- .../calcite/explain_streamstats_reset.yaml | 23 +- ...explain_streamstats_reset_null_bucket.yaml | 25 +- .../agg_case_composite_cannot_push.yaml | 4 +- .../agg_composite2_range_count_push.yaml | 4 +- ...agg_composite2_range_range_count_push.yaml | 4 +- .../agg_composite_range_metric_push.yaml | 4 +- .../agg_range_count_push.yaml | 2 +- .../agg_range_metric_complex_push.yaml | 4 +- .../agg_range_metric_push.yaml | 2 +- .../agg_range_range_metric_push.yaml | 6 +- .../chart_multiple_group_keys.yaml | 8 +- .../calcite_no_pushdown/chart_null_str.yaml | 8 +- .../chart_single_group_key.yaml | 7 +- .../calcite_no_pushdown/chart_with_limit.yaml | 4 +- .../explain_agg_with_script.yaml | 6 +- .../explain_agg_with_sum_enhancement.yaml | 4 +- .../explain_bin_minspan.json | 7 +- .../explain_filter_agg_push.yaml | 4 +- .../calcite_no_pushdown/explain_output.yaml | 4 +- .../explain_sort_agg_push.json | 2 +- .../explain_sort_then_agg_push.json | 2 +- .../explain_streamstats_global.yaml | 4 +- ...xplain_streamstats_global_null_bucket.yaml | 4 +- .../explain_streamstats_reset.yaml | 23 +- ...explain_streamstats_reset_null_bucket.yaml | 25 +- .../rest-api-spec/test/issues/5164_agg.yml | 230 ++++++++++++++++++ .../opensearch/request/AggregateAnalyzer.java | 17 ++ .../response/agg/CheckedLongSumParser.java | 47 ++++ .../request/AggregateAnalyzerTest.java | 60 +++++ .../agg/CheckedLongSumParserTest.java | 70 ++++++ .../calcite/OpenSearchSparkSqlDialect.java | 3 +- .../sql/ppl/calcite/CalcitePPLJoinTest.java | 2 +- .../ppl/calcite/CalcitePPLTimewrapTest.java | 5 +- 63 files changed, 966 insertions(+), 157 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/calcite/udf/udaf/BigintAvgAggFunction.java create mode 100644 core/src/main/java/org/opensearch/sql/calcite/udf/udaf/CheckedLongSumAggFunction.java create mode 100644 core/src/test/java/org/opensearch/sql/calcite/udf/udaf/BigintAvgAggFunctionTest.java create mode 100644 core/src/test/java/org/opensearch/sql/calcite/udf/udaf/CheckedLongSumAggFunctionTest.java create mode 100644 integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5164_agg.yml create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/CheckedLongSumParser.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/response/agg/CheckedLongSumParserTest.java diff --git a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java index 3320391c0d2..ca0c524b4a1 100644 --- a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java +++ b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java @@ -286,7 +286,7 @@ SELECT department, SUM(age) AS total FROM catalog.employees GROUP BY department .assertPlan( """ LogicalProject(department=[$0], total=[$1]) - LogicalAggregate(group=[{0}], SUM(age)=[SUM($1)]) + LogicalAggregate(group=[{0}], SUM(age)=[CHECKED_LONG_SUM($1)]) LogicalProject(department=[$3], age=[$2]) LogicalTableScan(table=[[catalog, employees]]) """); @@ -366,7 +366,7 @@ SELECT department, SUM(age) FILTER(WHERE age > 30) FROM catalog.employees """) .assertPlan( """ - LogicalAggregate(group=[{0}], SUM(age) FILTER(WHERE age > 30)=[SUM($1) FILTER $2]) + LogicalAggregate(group=[{0}], SUM(age) FILTER(WHERE age > 30)=[CHECKED_LONG_SUM($1) FILTER $2]) LogicalProject(department=[$3], age=[$2], $f3=[>($2, 30)]) LogicalTableScan(table=[[catalog, employees]]) """); @@ -487,7 +487,7 @@ SELECT name, SUM(age) OVER(PARTITION BY department ORDER BY age) FROM catalog.em """) .assertPlan( """ - LogicalProject(name=[$1], SUM(age) OVER(PARTITION BY department ORDER BY age)=[SUM($2) OVER (PARTITION BY $3 ORDER BY $2 NULLS FIRST)]) + LogicalProject(name=[$1], SUM(age) OVER(PARTITION BY department ORDER BY age)=[CHECKED_LONG_SUM($2) OVER (PARTITION BY $3 ORDER BY $2 NULLS FIRST)]) LogicalTableScan(table=[[catalog, employees]]) """); } diff --git a/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/BigintAvgAggFunction.java b/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/BigintAvgAggFunction.java new file mode 100644 index 00000000000..4b33b9a0ffa --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/BigintAvgAggFunction.java @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.udf.udaf; + +/** BIGINT average aggregate that accumulates in double to avoid an intermediate long overflow. */ +public class BigintAvgAggFunction { + + public static Accumulator init() { + return new Accumulator(); + } + + public static Accumulator add(Accumulator accumulator, Long value) { + if (value != null) { + accumulator.sum += value; + accumulator.count++; + } + return accumulator; + } + + public static Double result(Accumulator accumulator) { + return accumulator.count == 0 ? null : accumulator.sum / accumulator.count; + } + + public static class Accumulator { + private double sum; + private long count; + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/CheckedLongSumAggFunction.java b/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/CheckedLongSumAggFunction.java new file mode 100644 index 00000000000..7e146eacedb --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/CheckedLongSumAggFunction.java @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.udf.udaf; + +/** BIGINT sum aggregate that throws when its running long accumulator overflows. */ +public class CheckedLongSumAggFunction { + + public static long init() { + return 0L; + } + + public static long add(long accumulator, long value) { + return Math.addExact(accumulator, value); + } + + public static long result(long accumulator) { + return accumulator; + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java index f619d966cc8..0b365fc98e6 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java @@ -94,9 +94,32 @@ public static SqlUserDefinedAggFunction createUserDefinedAggFunction( String functionName, SqlReturnTypeInference returnType, @Nullable UDFOperandMetadata operandMetadata) { + return createReflectiveAggFunction(udafClass, functionName, returnType, operandMetadata); + } + + /** Creates an aggregate function from a class following Calcite's reflective UDAF convention. */ + public static SqlUserDefinedAggFunction createReflectiveAggFunction( + Class udafClass, + String functionName, + SqlReturnTypeInference returnType, + @Nullable UDFOperandMetadata operandMetadata) { + return createReflectiveAggFunction( + udafClass, functionName, SqlKind.OTHER_FUNCTION, returnType, operandMetadata); + } + + /** + * Creates an aggregate function whose kind remains visible to planner rules while execution uses + * the supplied reflective UDAF. + */ + public static SqlUserDefinedAggFunction createReflectiveAggFunction( + Class udafClass, + String functionName, + SqlKind kind, + SqlReturnTypeInference returnType, + @Nullable UDFOperandMetadata operandMetadata) { return new SqlUserDefinedAggFunction( new SqlIdentifier(functionName, SqlParserPos.ZERO), - SqlKind.OTHER_FUNCTION, + kind, returnType, null, operandMetadata, diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 2a670af3fee..812b94967f7 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.adaptExprMethodToUDF; import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.adaptExprMethodWithPropertiesToUDF; import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.adaptMathFunctionToUDF; +import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.createReflectiveAggFunction; import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.createUserDefinedAggFunction; import com.google.common.base.Suppliers; @@ -29,6 +30,8 @@ import org.apache.calcite.sql.type.SqlTypeTransforms; import org.apache.calcite.sql.util.ReflectiveSqlOperatorTable; import org.apache.calcite.util.BuiltInMethod; +import org.opensearch.sql.calcite.udf.udaf.BigintAvgAggFunction; +import org.opensearch.sql.calcite.udf.udaf.CheckedLongSumAggFunction; import org.opensearch.sql.calcite.udf.udaf.DistinctCountApproxLogicalAggFunction; import org.opensearch.sql.calcite.udf.udaf.FirstAggFunction; import org.opensearch.sql.calcite.udf.udaf.LastAggFunction; @@ -449,7 +452,6 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { new NumberToStringFunction().toUDF("NUMBER_TO_STRING"); public static final SqlOperator TONUMBER = new ToNumberFunction().toUDF("TONUMBER"); public static final SqlOperator TOSTRING = new ToStringFunction().toUDF("TOSTRING"); - // PPL Convert command functions public static final SqlOperator AUTO = new AutoConvertFunction().toUDF("AUTO"); public static final SqlOperator NUM = new NumConvertFunction().toUDF("NUM"); @@ -488,6 +490,20 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { new NullableSqlAvgAggFunction(SqlKind.VAR_POP); public static final SqlAggFunction VAR_SAMP_NULLABLE = new NullableSqlAvgAggFunction(SqlKind.VAR_SAMP); + public static final SqlAggFunction CHECKED_LONG_SUM = + createReflectiveAggFunction( + CheckedLongSumAggFunction.class, + "CHECKED_LONG_SUM", + SqlKind.SUM, + ReturnTypes.BIGINT_FORCE_NULLABLE, + PPLOperandTypes.NUMERIC); + public static final SqlAggFunction BIGINT_AVG = + createReflectiveAggFunction( + BigintAvgAggFunction.class, + "AVG", + SqlKind.AVG, + ReturnTypes.DOUBLE_NULLABLE, + PPLOperandTypes.NUMERIC); public static final SqlAggFunction TAKE = createUserDefinedAggFunction( TakeAggFunction.class, diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index 64f29906829..a4f05dd675e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -283,6 +283,7 @@ import java.util.Optional; import java.util.StringJoiner; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; import java.util.stream.Collectors; @@ -1595,7 +1596,14 @@ void register( } void registerOperator(BuiltinFunctionName functionName, SqlAggFunction aggFunction) { - SqlOperandTypeChecker innerTypeChecker = extractTypeCheckerFromUDF(aggFunction); + registerOperator(functionName, aggFunction, field -> aggFunction); + } + + void registerOperator( + BuiltinFunctionName functionName, + SqlAggFunction typeCheckerSource, + Function aggFunctionSelector) { + SqlOperandTypeChecker innerTypeChecker = extractTypeCheckerFromUDF(typeCheckerSource); PPLTypeChecker typeChecker = wrapSqlOperandTypeChecker(innerTypeChecker, functionName.name(), true); AggHandler handler = @@ -1603,15 +1611,30 @@ void registerOperator(BuiltinFunctionName functionName, SqlAggFunction aggFuncti List newArgList = argList.stream().map(PlanUtils::derefMapCall).collect(Collectors.toList()); return UserDefinedFunctionUtils.makeAggregateCall( - aggFunction, List.of(field), newArgList, ctx.relBuilder); + aggFunctionSelector.apply(field), List.of(field), newArgList, ctx.relBuilder); }; register(functionName, handler, typeChecker); } + /** Registers checked integral sums while retaining standard SUM behavior for other types. */ + void registerSumOperator() { + registerOperator( + SUM, + SqlStdOperatorTable.SUM, + field -> + isIntegral(field.getType().getSqlTypeName()) + ? PPLBuiltinOperators.CHECKED_LONG_SUM + : SqlStdOperatorTable.SUM); + } + + private static boolean isIntegral(SqlTypeName typeName) { + return SqlTypeName.INT_TYPES.contains(typeName); + } + void populate() { registerOperator(MAX, SqlStdOperatorTable.MAX); registerOperator(MIN, SqlStdOperatorTable.MIN); - registerOperator(SUM, SqlStdOperatorTable.SUM); + registerSumOperator(); registerOperator(VARSAMP, PPLBuiltinOperators.VAR_SAMP_NULLABLE); registerOperator(VARPOP, PPLBuiltinOperators.VAR_POP_NULLABLE); registerOperator(STDDEV_SAMP, PPLBuiltinOperators.STDDEV_SAMP_NULLABLE); @@ -1630,7 +1653,14 @@ void populate() { register( AVG, - (distinct, field, argList, ctx) -> ctx.relBuilder.avg(distinct, null, field), + (distinct, field, argList, ctx) -> { + if (field.getType().getSqlTypeName() == SqlTypeName.BIGINT) { + return ctx.relBuilder + .aggregateCall(PPLBuiltinOperators.BIGINT_AVG, field) + .distinct(distinct); + } + return ctx.relBuilder.avg(distinct, null, field); + }, wrapSqlOperandTypeChecker( SqlStdOperatorTable.AVG.getOperandTypeChecker(), AVG.name(), false)); diff --git a/core/src/test/java/org/opensearch/sql/calcite/udf/udaf/BigintAvgAggFunctionTest.java b/core/src/test/java/org/opensearch/sql/calcite/udf/udaf/BigintAvgAggFunctionTest.java new file mode 100644 index 00000000000..ab70f912211 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/calcite/udf/udaf/BigintAvgAggFunctionTest.java @@ -0,0 +1,44 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.udf.udaf; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import org.apache.calcite.sql.SqlKind; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.expression.function.PPLBuiltinOperators; + +class BigintAvgAggFunctionTest { + + @Test + void retainsAvgKindForPushdownRules() { + assertEquals(SqlKind.AVG, PPLBuiltinOperators.BIGINT_AVG.getKind()); + } + + @Test + void averagesWithoutLongOverflow() { + BigintAvgAggFunction.Accumulator accumulator = BigintAvgAggFunction.init(); + accumulator = BigintAvgAggFunction.add(accumulator, Long.MAX_VALUE); + accumulator = BigintAvgAggFunction.add(accumulator, Long.MAX_VALUE); + + assertEquals((double) Long.MAX_VALUE, BigintAvgAggFunction.result(accumulator)); + } + + @Test + void ignoresNulls() { + BigintAvgAggFunction.Accumulator accumulator = BigintAvgAggFunction.init(); + accumulator = BigintAvgAggFunction.add(accumulator, null); + accumulator = BigintAvgAggFunction.add(accumulator, 10L); + + assertEquals(10D, BigintAvgAggFunction.result(accumulator)); + } + + @Test + void returnsNullForEmptyInput() { + assertNull(BigintAvgAggFunction.result(BigintAvgAggFunction.init())); + } +} diff --git a/core/src/test/java/org/opensearch/sql/calcite/udf/udaf/CheckedLongSumAggFunctionTest.java b/core/src/test/java/org/opensearch/sql/calcite/udf/udaf/CheckedLongSumAggFunctionTest.java new file mode 100644 index 00000000000..0128cb4aa67 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/calcite/udf/udaf/CheckedLongSumAggFunctionTest.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.udf.udaf; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.apache.calcite.sql.SqlKind; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.expression.function.PPLBuiltinOperators; + +class CheckedLongSumAggFunctionTest { + + @Test + void retainsSumKindForPlannerRules() { + assertEquals(SqlKind.SUM, PPLBuiltinOperators.CHECKED_LONG_SUM.getKind()); + } + + @Test + void sumsExactly() { + long accumulator = CheckedLongSumAggFunction.init(); + accumulator = CheckedLongSumAggFunction.add(accumulator, 1L << 62); + accumulator = CheckedLongSumAggFunction.add(accumulator, 1L); + + assertEquals((1L << 62) + 1L, CheckedLongSumAggFunction.result(accumulator)); + } + + @Test + void throwsOnPositiveOverflow() { + assertThrows( + ArithmeticException.class, () -> CheckedLongSumAggFunction.add(Long.MAX_VALUE, 1L)); + } + + @Test + void throwsOnNegativeOverflow() { + assertThrows( + ArithmeticException.class, () -> CheckedLongSumAggFunction.add(Long.MIN_VALUE, -1L)); + } +} diff --git a/docs/user/ppl/functions/expressions.md b/docs/user/ppl/functions/expressions.md index 427a0334b58..14d9606a0c2 100644 --- a/docs/user/ppl/functions/expressions.md +++ b/docs/user/ppl/functions/expressions.md @@ -13,7 +13,14 @@ Arithmetic expressions are formed by combining numeric literals and binary arith ### Overflow behavior -Integer and long arithmetic operations (`+`, `-`, `*`) in `eval` expressions detect overflow and return an error instead of silently wrapping. For example, `eval x = int_field + 1` where `int_field` is `2147483647` (integer max) returns an error rather than `-2147483648`. Floating-point (`float`, `double`) arithmetic follows IEEE 754 and does not produce overflow errors. +Long (`BIGINT`) arithmetic operations (`+`, `-`, `*`) in `eval` expressions detect overflow and return an error instead of silently wrapping. Narrower integer operands are widened before arithmetic, so crossing the 32-bit integer boundary does not overflow. Floating-point (`float`, `double`) arithmetic follows IEEE 754 and does not produce overflow errors. + +The accumulator used by `stats sum(integral_field)` depends on whether Calcite pushdown is enabled: + +- With `plugins.calcite.pushdown.enabled=true` (the default), OpenSearch uses its native double-based `sum`, then the result is checked and narrowed to `BIGINT`. Large in-range sums can lose low-order precision. A small overflow that rounds to a signed `BIGINT` boundary can also be indistinguishable from an in-range sum and may saturate at the boundary instead of returning an error. +- With `plugins.calcite.pushdown.enabled=false`, Calcite uses an exact `BIGINT` accumulator and `Math.addExact` for every addition. It returns an error as soon as the running sum exceeds the `BIGINT` range. + +For example, summing `4611686018427387904` (`2^62`) and `1` returns the exact `4611686018427387905` without pushdown. With pushdown enabled, the double accumulator cannot represent the low-order `1`, so the result is `4611686018427387904`. ### Precedence @@ -189,4 +196,3 @@ fetched rows / total rows = 2/2 | 28 | +-----+ ``` - \ No newline at end of file diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java index a2ab93b6599..f043d9b3afc 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java @@ -20,12 +20,16 @@ import static org.opensearch.sql.util.MatcherUtils.verifyErrorMessageContains; import static org.opensearch.sql.util.MatcherUtils.verifySchema; import static org.opensearch.sql.util.MatcherUtils.verifySchemaInOrder; +import static org.opensearch.sql.util.TestUtils.createIndexByRestClient; +import static org.opensearch.sql.util.TestUtils.isIndexExist; +import static org.opensearch.sql.util.TestUtils.performRequest; import java.io.IOException; import java.util.Arrays; import java.util.List; import org.json.JSONObject; import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.ppl.PPLIntegTestCase; @@ -93,6 +97,173 @@ public void testSumAvg() throws IOException { verifyDataRows(actual, rows(186973)); } + @Test + public void testSumAllIntegralTypes() throws IOException { + String stats = + "stats sum(byte_number), sum(short_number), sum(integer_number), sum(long_number)"; + String query = String.format("source=%s | %s", TEST_INDEX_DATATYPE_NUMERIC, stats); + + JSONObject actual = executeQuery(query); + verifySchema( + actual, + schema("sum(byte_number)", "bigint"), + schema("sum(short_number)", "bigint"), + schema("sum(integer_number)", "bigint"), + schema("sum(long_number)", "bigint")); + verifyDataRows(actual, rows(4L, 3L, 2L, 1L)); + + String explain = explainQueryToString(query); + assertAllIntegralSumsAreChecked(explain); + + // HEAD prevents aggregation pushdown while preserving each field's integral input type. + String fallbackQuery = + String.format("source=%s | head 1 | %s", TEST_INDEX_DATATYPE_NUMERIC, stats); + verifyDataRows(executeQuery(fallbackQuery), rows(4L, 3L, 2L, 1L)); + + String fallbackExplain = explainQueryToString(fallbackQuery); + assertTrue(fallbackExplain.contains("EnumerableAggregate")); + assertAllIntegralSumsAreChecked(fallbackExplain); + } + + private static void assertAllIntegralSumsAreChecked(String explain) { + assertTrue(explain.contains("sum(byte_number)=[CHECKED_LONG_SUM(")); + assertTrue(explain.contains("sum(short_number)=[CHECKED_LONG_SUM(")); + assertTrue(explain.contains("sum(integer_number)=[CHECKED_LONG_SUM(")); + assertTrue(explain.contains("sum(long_number)=[CHECKED_LONG_SUM(")); + } + + @Test + public void testSumAvgLongOverflow() throws IOException { + String overflowIndex = "test_sum_long_overflow"; + createLongIndex(overflowIndex, Long.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE); + String inRangeIndex = "test_sum_long_in_range"; + createLongIndex(inRangeIndex, 1000000000000L, 2000000000000L, 3000000000000L); + String boundaryIndex = "test_sum_long_boundary"; + createLongIndex(boundaryIndex, Long.MAX_VALUE); + String exactIndex = "test_sum_long_exact"; + createLongIndex(exactIndex, 4611686018427387904L, 1L); + + // SUM overflows the BIGINT range (3 * (2^63 - 1)); surfaced as a client error rather than + // silently wrapping to a negative value. + assertSumOverflow(String.format("source=%s | stats sum(v)", overflowIndex)); + + // HEAD forces enumerable execution even when global pushdown is enabled. + assertSumOverflow(String.format("source=%s | head 3 | stats sum(v)", overflowIndex)); + + // AVG is averaged in DOUBLE, so it holds the true average (the shared value) without wrapping. + JSONObject avg = executeQuery(String.format("source=%s | stats avg(v)", overflowIndex)); + verifySchema(avg, schema("avg(v)", "double")); + verifyDataRows(avg, rows(9.223372036854776e18)); + + JSONObject fallbackAvg = + executeQuery(String.format("source=%s | head 3 | stats avg(v)", overflowIndex)); + verifySchema(fallbackAvg, schema("avg(v)", "double")); + verifyDataRows(fallbackAvg, rows(9.223372036854776e18)); + + JSONObject expressionAvg = + executeQuery(String.format("source=%s | head 3 | stats avg(v + 0)", overflowIndex)); + verifySchema(expressionAvg, schema("avg(v + 0)", "double")); + verifyDataRows(expressionAvg, rows(9.223372036854776e18)); + + String pushedExpressionQuery = String.format("source=%s | stats avg(v + 0)", overflowIndex); + JSONObject pushedExpressionAvg = executeQuery(pushedExpressionQuery); + verifySchema(pushedExpressionAvg, schema("avg(v + 0)", "double")); + verifyDataRows(pushedExpressionAvg, rows(9.223372036854776e18)); + if (!isPushdownDisabled() && !isAnalyticsParquetIndicesEnabled()) { + assertTrue(explainQueryToString(pushedExpressionQuery).contains("AGGREGATION->")); + } + + // A sum well within the BIGINT range returns the exact value with no error. + JSONObject inRange = executeQuery(String.format("source=%s | stats sum(v)", inRangeIndex)); + verifySchema(inRange, schema("sum(v)", "bigint")); + verifyDataRows(inRange, rows(6000000000000L)); + + // A single Long.MAX_VALUE is a valid (non-overflowing) sum and must not error. + JSONObject boundary = executeQuery(String.format("source=%s | stats sum(v)", boundaryIndex)); + verifySchema(boundary, schema("sum(v)", "bigint")); + verifyDataRows(boundary, rows(9223372036854775807L)); + + // Native sum pushdown uses double and loses the low-order bit; the fallback and analytics + // backends retain it. + JSONObject exact = executeQuery(String.format("source=%s | stats sum(v)", exactIndex)); + verifySchema(exact, schema("sum(v)", "bigint")); + long expectedExact = + (isPushdownDisabled() || isAnalyticsParquetIndicesEnabled()) + ? 4611686018427387905L + : 4611686018427387904L; + verifyDataRows(exact, rows(expectedExact)); + + // HEAD prevents pushdown, so the checked long accumulator retains the low-order bit. + JSONObject exactFallback = + executeQuery(String.format("source=%s | head 2 | stats sum(v)", exactIndex)); + verifySchema(exactFallback, schema("sum(v)", "bigint")); + verifyDataRows(exactFallback, rows(4611686018427387905L)); + } + + @Test + public void testNegativeLongSumOverflowAndBoundary() throws IOException { + String overflowIndex = "test_sum_long_negative_overflow"; + createLongIndex(overflowIndex, Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE); + String boundaryIndex = "test_sum_long_negative_boundary"; + createLongIndex(boundaryIndex, Long.MIN_VALUE); + + assertSumOverflow(String.format("source=%s | stats sum(v)", overflowIndex)); + assertSumOverflow(String.format("source=%s | head 3 | stats sum(v)", overflowIndex)); + + JSONObject boundary = executeQuery(String.format("source=%s | stats sum(v)", boundaryIndex)); + verifySchema(boundary, schema("sum(v)", "bigint")); + verifyDataRows(boundary, rows(Long.MIN_VALUE)); + + JSONObject avg = executeQuery(String.format("source=%s | stats avg(v)", overflowIndex)); + verifySchema(avg, schema("avg(v)", "double")); + verifyDataRows(avg, rows((double) Long.MIN_VALUE)); + } + + @Test + public void testFallbackLongSumRejectsIntermediateOverflow() throws IOException { + String index = "test_sum_long_intermediate_overflow"; + createLongIndex(index, Long.MAX_VALUE, 1L, -1L); + + // The final mathematical result fits, but Math.addExact rejects the intermediate MAX + 1. + assertSumOverflow(String.format("source=%s | sort - v | head 3 | stats sum(v)", index)); + } + + @Test + public void testFloatingSumsDoNotUseCheckedLongAccumulator() throws IOException { + String stats = + "stats sum(double_number), sum(float_number)," + + " sum(half_float_number), sum(scaled_float_number)"; + String query = String.format("source=%s | %s", TEST_INDEX_DATATYPE_NUMERIC, stats); + String fallbackQuery = + String.format("source=%s | head 1 | %s", TEST_INDEX_DATATYPE_NUMERIC, stats); + + assertEquals(1, executeQuery(query).getInt("total")); + assertFalse(explainQueryToString(query).contains("CHECKED_LONG_SUM")); + assertEquals(1, executeQuery(fallbackQuery).getInt("total")); + assertFalse(explainQueryToString(fallbackQuery).contains("CHECKED_LONG_SUM")); + } + + private void createLongIndex(String index, long... values) throws IOException { + if (isIndexExist(client(), index)) { + return; + } + + createIndexByRestClient( + client(), index, "{\"mappings\":{\"properties\":{\"v\":{\"type\":\"long\"}}}}"); + StringBuilder body = new StringBuilder(); + for (long value : values) { + body.append("{\"index\":{}}\n").append("{\"v\":").append(value).append("}\n"); + } + Request bulk = new Request("POST", "/" + index + "/_bulk?refresh=true"); + bulk.setJsonEntity(body.toString()); + performRequest(client(), bulk); + } + + private void assertSumOverflow(String query) throws IOException { + Throwable error = assertThrowsWithReplace(RuntimeException.class, () -> executeQuery(query)); + verifyErrorMessageContains(error, "verflow"); + } + @Test public void testAsExistedField() throws IOException { JSONObject actual = @@ -993,9 +1164,7 @@ public void testSumGroupByNullValue() throws IOException { String.format( "source=%s | stats sum(balance) as a by age", TEST_INDEX_BANK_WITH_NULL_VALUES)); verifySchema(response, schema("a", null, "bigint"), schema("age", null, "int")); - // SUM of an all-null bucket is null per the SQL spec. The DSL-pushdown path returns 0 instead - // (a known pushdown quirk); the analytics-engine backend (DataFusion) follows the spec like - // Calcite-no-pushdown and returns null. See testSumNull and #3408. + // Native sum returns 0 for an all-null bucket; fallback and analytics backends return null. Object emptySum = (isPushdownDisabled() || isAnalyticsParquetIndicesEnabled()) ? null : 0; verifyDataRows( response, diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java index 7417fd112ec..4fd8aa7f852 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java @@ -512,11 +512,6 @@ public void testSumWithNull() throws IOException { "source=%s | where age = 36 | stats sum(balance)", TEST_INDEX_BANK_WITH_NULL_VALUES)); verifySchema(response, schema("sum(balance)", null, "bigint")); - // TODO: Fix -- temporary workaround for the pushdown issue: - // The current pushdown implementation will return 0 for sum when getting null values as input. - // Returning null should be the expected behavior. - // The analytics-engine backend (DataFusion) follows the SQL spec like Calcite-no-pushdown — - // SUM of all-null is null, not 0. Integer expectedValue = (isPushdownDisabled() || isAnalyticsParquetIndicesEnabled()) ? null : 0; verifyDataRows(response, rows(expectedValue)); } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml index 726eeedc429..f9196e3b597 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml @@ -25,15 +25,15 @@ calcite: EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['nil'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], gender=[$t0], age=[$t3], avg(balance)=[$t2]) + EnumerableAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], expr#11=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t10], $condition=[$t11]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], expr#4=[IS NOT NULL($t3)], age=[$t3], avg(balance)=[$t2], $condition=[$t4]) + EnumerableAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q10.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q10.yaml index f900b2ccbec..4e024b0c12d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q10.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q10.yaml @@ -3,9 +3,9 @@ calcite: LogicalSystemLimit(sort0=[$1], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$1], dir0=[DESC-nulls-last], fetch=[10]) LogicalProject(sum(AdvEngineID)=[$1], c=[$2], avg(ResolutionWidth)=[$3], dc(UserID)=[$4], RegionID=[$0]) - LogicalAggregate(group=[{0}], sum(AdvEngineID)=[SUM($1)], c=[COUNT()], avg(ResolutionWidth)=[AVG($2)], dc(UserID)=[COUNT(DISTINCT $3)]) + LogicalAggregate(group=[{0}], sum(AdvEngineID)=[CHECKED_LONG_SUM($1)], c=[COUNT()], avg(ResolutionWidth)=[AVG($2)], dc(UserID)=[COUNT(DISTINCT $3)]) LogicalProject(RegionID=[$68], AdvEngineID=[$19], ResolutionWidth=[$80], UserID=[$84]) LogicalFilter(condition=[IS NOT NULL($68)]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum(AdvEngineID)=SUM($0),c=COUNT(),avg(ResolutionWidth)=AVG($2),dc(UserID)=COUNT(DISTINCT $3)), PROJECT->[sum(AdvEngineID), c, avg(ResolutionWidth), dc(UserID), RegionID], SORT_AGG_METRICS->[1 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"RegionID":{"terms":{"field":"RegionID","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(AdvEngineID)":{"sum":{"field":"AdvEngineID"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"dc(UserID)":{"cardinality":{"field":"UserID"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum(AdvEngineID)=CHECKED_LONG_SUM($0),c=COUNT(),avg(ResolutionWidth)=AVG($2),dc(UserID)=COUNT(DISTINCT $3)), PROJECT->[sum(AdvEngineID), c, avg(ResolutionWidth), dc(UserID), RegionID], SORT_AGG_METRICS->[1 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"RegionID":{"terms":{"field":"RegionID","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(AdvEngineID)":{"sum":{"field":"AdvEngineID"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"dc(UserID)":{"cardinality":{"field":"UserID"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q3.yaml index ef93b63ee80..24ddecd3881 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q3.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q3.yaml @@ -1,8 +1,8 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{}], sum(AdvEngineID)=[SUM($0)], count()=[COUNT()], avg(ResolutionWidth)=[AVG($1)]) + LogicalAggregate(group=[{}], sum(AdvEngineID)=[CHECKED_LONG_SUM($0)], count()=[COUNT()], avg(ResolutionWidth)=[AVG($1)]) LogicalProject(AdvEngineID=[$19], ResolutionWidth=[$80]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},sum(AdvEngineID)=SUM($0),count()=COUNT(),avg(ResolutionWidth)=AVG($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"sum(AdvEngineID)":{"sum":{"field":"AdvEngineID"}},"count()":{"value_count":{"field":"_index"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},sum(AdvEngineID)=CHECKED_LONG_SUM($0),count()=COUNT(),avg(ResolutionWidth)=AVG($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"sum(AdvEngineID)":{"sum":{"field":"AdvEngineID"}},"count()":{"value_count":{"field":"_index"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml index d50a9ec47ce..189ab6349e1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml @@ -1,11 +1,11 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{}], sum(ResolutionWidth)=[SUM($0)], sum(ResolutionWidth+1)=[SUM($1)], sum(ResolutionWidth+2)=[SUM($2)], sum(ResolutionWidth+3)=[SUM($3)], sum(ResolutionWidth+4)=[SUM($4)], sum(ResolutionWidth+5)=[SUM($5)], sum(ResolutionWidth+6)=[SUM($6)], sum(ResolutionWidth+7)=[SUM($7)], sum(ResolutionWidth+8)=[SUM($8)], sum(ResolutionWidth+9)=[SUM($9)], sum(ResolutionWidth+10)=[SUM($10)], sum(ResolutionWidth+11)=[SUM($11)], sum(ResolutionWidth+12)=[SUM($12)], sum(ResolutionWidth+13)=[SUM($13)], sum(ResolutionWidth+14)=[SUM($14)], sum(ResolutionWidth+15)=[SUM($15)], sum(ResolutionWidth+16)=[SUM($16)], sum(ResolutionWidth+17)=[SUM($17)], sum(ResolutionWidth+18)=[SUM($18)], sum(ResolutionWidth+19)=[SUM($19)], sum(ResolutionWidth+20)=[SUM($20)], sum(ResolutionWidth+21)=[SUM($21)], sum(ResolutionWidth+22)=[SUM($22)], sum(ResolutionWidth+23)=[SUM($23)], sum(ResolutionWidth+24)=[SUM($24)], sum(ResolutionWidth+25)=[SUM($25)], sum(ResolutionWidth+26)=[SUM($26)], sum(ResolutionWidth+27)=[SUM($27)], sum(ResolutionWidth+28)=[SUM($28)], sum(ResolutionWidth+29)=[SUM($29)], sum(ResolutionWidth+30)=[SUM($30)], sum(ResolutionWidth+31)=[SUM($31)], sum(ResolutionWidth+32)=[SUM($32)], sum(ResolutionWidth+33)=[SUM($33)], sum(ResolutionWidth+34)=[SUM($34)], sum(ResolutionWidth+35)=[SUM($35)], sum(ResolutionWidth+36)=[SUM($36)], sum(ResolutionWidth+37)=[SUM($37)], sum(ResolutionWidth+38)=[SUM($38)], sum(ResolutionWidth+39)=[SUM($39)], sum(ResolutionWidth+40)=[SUM($40)], sum(ResolutionWidth+41)=[SUM($41)], sum(ResolutionWidth+42)=[SUM($42)], sum(ResolutionWidth+43)=[SUM($43)], sum(ResolutionWidth+44)=[SUM($44)], sum(ResolutionWidth+45)=[SUM($45)], sum(ResolutionWidth+46)=[SUM($46)], sum(ResolutionWidth+47)=[SUM($47)], sum(ResolutionWidth+48)=[SUM($48)], sum(ResolutionWidth+49)=[SUM($49)], sum(ResolutionWidth+50)=[SUM($50)], sum(ResolutionWidth+51)=[SUM($51)], sum(ResolutionWidth+52)=[SUM($52)], sum(ResolutionWidth+53)=[SUM($53)], sum(ResolutionWidth+54)=[SUM($54)], sum(ResolutionWidth+55)=[SUM($55)], sum(ResolutionWidth+56)=[SUM($56)], sum(ResolutionWidth+57)=[SUM($57)], sum(ResolutionWidth+58)=[SUM($58)], sum(ResolutionWidth+59)=[SUM($59)], sum(ResolutionWidth+60)=[SUM($60)], sum(ResolutionWidth+61)=[SUM($61)], sum(ResolutionWidth+62)=[SUM($62)], sum(ResolutionWidth+63)=[SUM($63)], sum(ResolutionWidth+64)=[SUM($64)], sum(ResolutionWidth+65)=[SUM($65)], sum(ResolutionWidth+66)=[SUM($66)], sum(ResolutionWidth+67)=[SUM($67)], sum(ResolutionWidth+68)=[SUM($68)], sum(ResolutionWidth+69)=[SUM($69)], sum(ResolutionWidth+70)=[SUM($70)], sum(ResolutionWidth+71)=[SUM($71)], sum(ResolutionWidth+72)=[SUM($72)], sum(ResolutionWidth+73)=[SUM($73)], sum(ResolutionWidth+74)=[SUM($74)], sum(ResolutionWidth+75)=[SUM($75)], sum(ResolutionWidth+76)=[SUM($76)], sum(ResolutionWidth+77)=[SUM($77)], sum(ResolutionWidth+78)=[SUM($78)], sum(ResolutionWidth+79)=[SUM($79)], sum(ResolutionWidth+80)=[SUM($80)], sum(ResolutionWidth+81)=[SUM($81)], sum(ResolutionWidth+82)=[SUM($82)], sum(ResolutionWidth+83)=[SUM($83)], sum(ResolutionWidth+84)=[SUM($84)], sum(ResolutionWidth+85)=[SUM($85)], sum(ResolutionWidth+86)=[SUM($86)], sum(ResolutionWidth+87)=[SUM($87)], sum(ResolutionWidth+88)=[SUM($88)], sum(ResolutionWidth+89)=[SUM($89)]) + LogicalAggregate(group=[{}], sum(ResolutionWidth)=[CHECKED_LONG_SUM($0)], sum(ResolutionWidth+1)=[CHECKED_LONG_SUM($1)], sum(ResolutionWidth+2)=[CHECKED_LONG_SUM($2)], sum(ResolutionWidth+3)=[CHECKED_LONG_SUM($3)], sum(ResolutionWidth+4)=[CHECKED_LONG_SUM($4)], sum(ResolutionWidth+5)=[CHECKED_LONG_SUM($5)], sum(ResolutionWidth+6)=[CHECKED_LONG_SUM($6)], sum(ResolutionWidth+7)=[CHECKED_LONG_SUM($7)], sum(ResolutionWidth+8)=[CHECKED_LONG_SUM($8)], sum(ResolutionWidth+9)=[CHECKED_LONG_SUM($9)], sum(ResolutionWidth+10)=[CHECKED_LONG_SUM($10)], sum(ResolutionWidth+11)=[CHECKED_LONG_SUM($11)], sum(ResolutionWidth+12)=[CHECKED_LONG_SUM($12)], sum(ResolutionWidth+13)=[CHECKED_LONG_SUM($13)], sum(ResolutionWidth+14)=[CHECKED_LONG_SUM($14)], sum(ResolutionWidth+15)=[CHECKED_LONG_SUM($15)], sum(ResolutionWidth+16)=[CHECKED_LONG_SUM($16)], sum(ResolutionWidth+17)=[CHECKED_LONG_SUM($17)], sum(ResolutionWidth+18)=[CHECKED_LONG_SUM($18)], sum(ResolutionWidth+19)=[CHECKED_LONG_SUM($19)], sum(ResolutionWidth+20)=[CHECKED_LONG_SUM($20)], sum(ResolutionWidth+21)=[CHECKED_LONG_SUM($21)], sum(ResolutionWidth+22)=[CHECKED_LONG_SUM($22)], sum(ResolutionWidth+23)=[CHECKED_LONG_SUM($23)], sum(ResolutionWidth+24)=[CHECKED_LONG_SUM($24)], sum(ResolutionWidth+25)=[CHECKED_LONG_SUM($25)], sum(ResolutionWidth+26)=[CHECKED_LONG_SUM($26)], sum(ResolutionWidth+27)=[CHECKED_LONG_SUM($27)], sum(ResolutionWidth+28)=[CHECKED_LONG_SUM($28)], sum(ResolutionWidth+29)=[CHECKED_LONG_SUM($29)], sum(ResolutionWidth+30)=[CHECKED_LONG_SUM($30)], sum(ResolutionWidth+31)=[CHECKED_LONG_SUM($31)], sum(ResolutionWidth+32)=[CHECKED_LONG_SUM($32)], sum(ResolutionWidth+33)=[CHECKED_LONG_SUM($33)], sum(ResolutionWidth+34)=[CHECKED_LONG_SUM($34)], sum(ResolutionWidth+35)=[CHECKED_LONG_SUM($35)], sum(ResolutionWidth+36)=[CHECKED_LONG_SUM($36)], sum(ResolutionWidth+37)=[CHECKED_LONG_SUM($37)], sum(ResolutionWidth+38)=[CHECKED_LONG_SUM($38)], sum(ResolutionWidth+39)=[CHECKED_LONG_SUM($39)], sum(ResolutionWidth+40)=[CHECKED_LONG_SUM($40)], sum(ResolutionWidth+41)=[CHECKED_LONG_SUM($41)], sum(ResolutionWidth+42)=[CHECKED_LONG_SUM($42)], sum(ResolutionWidth+43)=[CHECKED_LONG_SUM($43)], sum(ResolutionWidth+44)=[CHECKED_LONG_SUM($44)], sum(ResolutionWidth+45)=[CHECKED_LONG_SUM($45)], sum(ResolutionWidth+46)=[CHECKED_LONG_SUM($46)], sum(ResolutionWidth+47)=[CHECKED_LONG_SUM($47)], sum(ResolutionWidth+48)=[CHECKED_LONG_SUM($48)], sum(ResolutionWidth+49)=[CHECKED_LONG_SUM($49)], sum(ResolutionWidth+50)=[CHECKED_LONG_SUM($50)], sum(ResolutionWidth+51)=[CHECKED_LONG_SUM($51)], sum(ResolutionWidth+52)=[CHECKED_LONG_SUM($52)], sum(ResolutionWidth+53)=[CHECKED_LONG_SUM($53)], sum(ResolutionWidth+54)=[CHECKED_LONG_SUM($54)], sum(ResolutionWidth+55)=[CHECKED_LONG_SUM($55)], sum(ResolutionWidth+56)=[CHECKED_LONG_SUM($56)], sum(ResolutionWidth+57)=[CHECKED_LONG_SUM($57)], sum(ResolutionWidth+58)=[CHECKED_LONG_SUM($58)], sum(ResolutionWidth+59)=[CHECKED_LONG_SUM($59)], sum(ResolutionWidth+60)=[CHECKED_LONG_SUM($60)], sum(ResolutionWidth+61)=[CHECKED_LONG_SUM($61)], sum(ResolutionWidth+62)=[CHECKED_LONG_SUM($62)], sum(ResolutionWidth+63)=[CHECKED_LONG_SUM($63)], sum(ResolutionWidth+64)=[CHECKED_LONG_SUM($64)], sum(ResolutionWidth+65)=[CHECKED_LONG_SUM($65)], sum(ResolutionWidth+66)=[CHECKED_LONG_SUM($66)], sum(ResolutionWidth+67)=[CHECKED_LONG_SUM($67)], sum(ResolutionWidth+68)=[CHECKED_LONG_SUM($68)], sum(ResolutionWidth+69)=[CHECKED_LONG_SUM($69)], sum(ResolutionWidth+70)=[CHECKED_LONG_SUM($70)], sum(ResolutionWidth+71)=[CHECKED_LONG_SUM($71)], sum(ResolutionWidth+72)=[CHECKED_LONG_SUM($72)], sum(ResolutionWidth+73)=[CHECKED_LONG_SUM($73)], sum(ResolutionWidth+74)=[CHECKED_LONG_SUM($74)], sum(ResolutionWidth+75)=[CHECKED_LONG_SUM($75)], sum(ResolutionWidth+76)=[CHECKED_LONG_SUM($76)], sum(ResolutionWidth+77)=[CHECKED_LONG_SUM($77)], sum(ResolutionWidth+78)=[CHECKED_LONG_SUM($78)], sum(ResolutionWidth+79)=[CHECKED_LONG_SUM($79)], sum(ResolutionWidth+80)=[CHECKED_LONG_SUM($80)], sum(ResolutionWidth+81)=[CHECKED_LONG_SUM($81)], sum(ResolutionWidth+82)=[CHECKED_LONG_SUM($82)], sum(ResolutionWidth+83)=[CHECKED_LONG_SUM($83)], sum(ResolutionWidth+84)=[CHECKED_LONG_SUM($84)], sum(ResolutionWidth+85)=[CHECKED_LONG_SUM($85)], sum(ResolutionWidth+86)=[CHECKED_LONG_SUM($86)], sum(ResolutionWidth+87)=[CHECKED_LONG_SUM($87)], sum(ResolutionWidth+88)=[CHECKED_LONG_SUM($88)], sum(ResolutionWidth+89)=[CHECKED_LONG_SUM($89)]) LogicalProject(ResolutionWidth=[$80], $f90=[+(CAST($80):BIGINT, 1)], $f91=[+(CAST($80):BIGINT, 2)], $f92=[+(CAST($80):BIGINT, 3)], $f93=[+(CAST($80):BIGINT, 4)], $f94=[+(CAST($80):BIGINT, 5)], $f95=[+(CAST($80):BIGINT, 6)], $f96=[+(CAST($80):BIGINT, 7)], $f97=[+(CAST($80):BIGINT, 8)], $f98=[+(CAST($80):BIGINT, 9)], $f99=[+(CAST($80):BIGINT, 10)], $f100=[+(CAST($80):BIGINT, 11)], $f101=[+(CAST($80):BIGINT, 12)], $f102=[+(CAST($80):BIGINT, 13)], $f103=[+(CAST($80):BIGINT, 14)], $f104=[+(CAST($80):BIGINT, 15)], $f105=[+(CAST($80):BIGINT, 16)], $f106=[+(CAST($80):BIGINT, 17)], $f107=[+(CAST($80):BIGINT, 18)], $f108=[+(CAST($80):BIGINT, 19)], $f109=[+(CAST($80):BIGINT, 20)], $f110=[+(CAST($80):BIGINT, 21)], $f111=[+(CAST($80):BIGINT, 22)], $f112=[+(CAST($80):BIGINT, 23)], $f113=[+(CAST($80):BIGINT, 24)], $f114=[+(CAST($80):BIGINT, 25)], $f115=[+(CAST($80):BIGINT, 26)], $f116=[+(CAST($80):BIGINT, 27)], $f117=[+(CAST($80):BIGINT, 28)], $f118=[+(CAST($80):BIGINT, 29)], $f119=[+(CAST($80):BIGINT, 30)], $f120=[+(CAST($80):BIGINT, 31)], $f121=[+(CAST($80):BIGINT, 32)], $f122=[+(CAST($80):BIGINT, 33)], $f123=[+(CAST($80):BIGINT, 34)], $f124=[+(CAST($80):BIGINT, 35)], $f125=[+(CAST($80):BIGINT, 36)], $f126=[+(CAST($80):BIGINT, 37)], $f127=[+(CAST($80):BIGINT, 38)], $f128=[+(CAST($80):BIGINT, 39)], $f129=[+(CAST($80):BIGINT, 40)], $f130=[+(CAST($80):BIGINT, 41)], $f131=[+(CAST($80):BIGINT, 42)], $f132=[+(CAST($80):BIGINT, 43)], $f133=[+(CAST($80):BIGINT, 44)], $f134=[+(CAST($80):BIGINT, 45)], $f135=[+(CAST($80):BIGINT, 46)], $f136=[+(CAST($80):BIGINT, 47)], $f137=[+(CAST($80):BIGINT, 48)], $f138=[+(CAST($80):BIGINT, 49)], $f139=[+(CAST($80):BIGINT, 50)], $f140=[+(CAST($80):BIGINT, 51)], $f141=[+(CAST($80):BIGINT, 52)], $f142=[+(CAST($80):BIGINT, 53)], $f143=[+(CAST($80):BIGINT, 54)], $f144=[+(CAST($80):BIGINT, 55)], $f145=[+(CAST($80):BIGINT, 56)], $f146=[+(CAST($80):BIGINT, 57)], $f147=[+(CAST($80):BIGINT, 58)], $f148=[+(CAST($80):BIGINT, 59)], $f149=[+(CAST($80):BIGINT, 60)], $f150=[+(CAST($80):BIGINT, 61)], $f151=[+(CAST($80):BIGINT, 62)], $f152=[+(CAST($80):BIGINT, 63)], $f153=[+(CAST($80):BIGINT, 64)], $f154=[+(CAST($80):BIGINT, 65)], $f155=[+(CAST($80):BIGINT, 66)], $f156=[+(CAST($80):BIGINT, 67)], $f157=[+(CAST($80):BIGINT, 68)], $f158=[+(CAST($80):BIGINT, 69)], $f159=[+(CAST($80):BIGINT, 70)], $f160=[+(CAST($80):BIGINT, 71)], $f161=[+(CAST($80):BIGINT, 72)], $f162=[+(CAST($80):BIGINT, 73)], $f163=[+(CAST($80):BIGINT, 74)], $f164=[+(CAST($80):BIGINT, 75)], $f165=[+(CAST($80):BIGINT, 76)], $f166=[+(CAST($80):BIGINT, 77)], $f167=[+(CAST($80):BIGINT, 78)], $f168=[+(CAST($80):BIGINT, 79)], $f169=[+(CAST($80):BIGINT, 80)], $f170=[+(CAST($80):BIGINT, 81)], $f171=[+(CAST($80):BIGINT, 82)], $f172=[+(CAST($80):BIGINT, 83)], $f173=[+(CAST($80):BIGINT, 84)], $f174=[+(CAST($80):BIGINT, 85)], $f175=[+(CAST($80):BIGINT, 86)], $f176=[+(CAST($80):BIGINT, 87)], $f177=[+(CAST($80):BIGINT, 88)], $f178=[+(CAST($80):BIGINT, 89)]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableAggregate(group=[{}], sum(ResolutionWidth)=[SUM($0)], sum(ResolutionWidth+1)=[SUM($1)], sum(ResolutionWidth+2)=[SUM($2)], sum(ResolutionWidth+3)=[SUM($3)], sum(ResolutionWidth+4)=[SUM($4)], sum(ResolutionWidth+5)=[SUM($5)], sum(ResolutionWidth+6)=[SUM($6)], sum(ResolutionWidth+7)=[SUM($7)], sum(ResolutionWidth+8)=[SUM($8)], sum(ResolutionWidth+9)=[SUM($9)], sum(ResolutionWidth+10)=[SUM($10)], sum(ResolutionWidth+11)=[SUM($11)], sum(ResolutionWidth+12)=[SUM($12)], sum(ResolutionWidth+13)=[SUM($13)], sum(ResolutionWidth+14)=[SUM($14)], sum(ResolutionWidth+15)=[SUM($15)], sum(ResolutionWidth+16)=[SUM($16)], sum(ResolutionWidth+17)=[SUM($17)], sum(ResolutionWidth+18)=[SUM($18)], sum(ResolutionWidth+19)=[SUM($19)], sum(ResolutionWidth+20)=[SUM($20)], sum(ResolutionWidth+21)=[SUM($21)], sum(ResolutionWidth+22)=[SUM($22)], sum(ResolutionWidth+23)=[SUM($23)], sum(ResolutionWidth+24)=[SUM($24)], sum(ResolutionWidth+25)=[SUM($25)], sum(ResolutionWidth+26)=[SUM($26)], sum(ResolutionWidth+27)=[SUM($27)], sum(ResolutionWidth+28)=[SUM($28)], sum(ResolutionWidth+29)=[SUM($29)], sum(ResolutionWidth+30)=[SUM($30)], sum(ResolutionWidth+31)=[SUM($31)], sum(ResolutionWidth+32)=[SUM($32)], sum(ResolutionWidth+33)=[SUM($33)], sum(ResolutionWidth+34)=[SUM($34)], sum(ResolutionWidth+35)=[SUM($35)], sum(ResolutionWidth+36)=[SUM($36)], sum(ResolutionWidth+37)=[SUM($37)], sum(ResolutionWidth+38)=[SUM($38)], sum(ResolutionWidth+39)=[SUM($39)], sum(ResolutionWidth+40)=[SUM($40)], sum(ResolutionWidth+41)=[SUM($41)], sum(ResolutionWidth+42)=[SUM($42)], sum(ResolutionWidth+43)=[SUM($43)], sum(ResolutionWidth+44)=[SUM($44)], sum(ResolutionWidth+45)=[SUM($45)], sum(ResolutionWidth+46)=[SUM($46)], sum(ResolutionWidth+47)=[SUM($47)], sum(ResolutionWidth+48)=[SUM($48)], sum(ResolutionWidth+49)=[SUM($49)], sum(ResolutionWidth+50)=[SUM($50)], sum(ResolutionWidth+51)=[SUM($51)], sum(ResolutionWidth+52)=[SUM($52)], sum(ResolutionWidth+53)=[SUM($53)], sum(ResolutionWidth+54)=[SUM($54)], sum(ResolutionWidth+55)=[SUM($55)], sum(ResolutionWidth+56)=[SUM($56)], sum(ResolutionWidth+57)=[SUM($57)], sum(ResolutionWidth+58)=[SUM($58)], sum(ResolutionWidth+59)=[SUM($59)], sum(ResolutionWidth+60)=[SUM($60)], sum(ResolutionWidth+61)=[SUM($61)], sum(ResolutionWidth+62)=[SUM($62)], sum(ResolutionWidth+63)=[SUM($63)], sum(ResolutionWidth+64)=[SUM($64)], sum(ResolutionWidth+65)=[SUM($65)], sum(ResolutionWidth+66)=[SUM($66)], sum(ResolutionWidth+67)=[SUM($67)], sum(ResolutionWidth+68)=[SUM($68)], sum(ResolutionWidth+69)=[SUM($69)], sum(ResolutionWidth+70)=[SUM($70)], sum(ResolutionWidth+71)=[SUM($71)], sum(ResolutionWidth+72)=[SUM($72)], sum(ResolutionWidth+73)=[SUM($73)], sum(ResolutionWidth+74)=[SUM($74)], sum(ResolutionWidth+75)=[SUM($75)], sum(ResolutionWidth+76)=[SUM($76)], sum(ResolutionWidth+77)=[SUM($77)], sum(ResolutionWidth+78)=[SUM($78)], sum(ResolutionWidth+79)=[SUM($79)], sum(ResolutionWidth+80)=[SUM($80)], sum(ResolutionWidth+81)=[SUM($81)], sum(ResolutionWidth+82)=[SUM($82)], sum(ResolutionWidth+83)=[SUM($83)], sum(ResolutionWidth+84)=[SUM($84)], sum(ResolutionWidth+85)=[SUM($85)], sum(ResolutionWidth+86)=[SUM($86)], sum(ResolutionWidth+87)=[SUM($87)], sum(ResolutionWidth+88)=[SUM($88)], sum(ResolutionWidth+89)=[SUM($89)]) + EnumerableAggregate(group=[{}], sum(ResolutionWidth)=[CHECKED_LONG_SUM($0)], sum(ResolutionWidth+1)=[CHECKED_LONG_SUM($1)], sum(ResolutionWidth+2)=[CHECKED_LONG_SUM($2)], sum(ResolutionWidth+3)=[CHECKED_LONG_SUM($3)], sum(ResolutionWidth+4)=[CHECKED_LONG_SUM($4)], sum(ResolutionWidth+5)=[CHECKED_LONG_SUM($5)], sum(ResolutionWidth+6)=[CHECKED_LONG_SUM($6)], sum(ResolutionWidth+7)=[CHECKED_LONG_SUM($7)], sum(ResolutionWidth+8)=[CHECKED_LONG_SUM($8)], sum(ResolutionWidth+9)=[CHECKED_LONG_SUM($9)], sum(ResolutionWidth+10)=[CHECKED_LONG_SUM($10)], sum(ResolutionWidth+11)=[CHECKED_LONG_SUM($11)], sum(ResolutionWidth+12)=[CHECKED_LONG_SUM($12)], sum(ResolutionWidth+13)=[CHECKED_LONG_SUM($13)], sum(ResolutionWidth+14)=[CHECKED_LONG_SUM($14)], sum(ResolutionWidth+15)=[CHECKED_LONG_SUM($15)], sum(ResolutionWidth+16)=[CHECKED_LONG_SUM($16)], sum(ResolutionWidth+17)=[CHECKED_LONG_SUM($17)], sum(ResolutionWidth+18)=[CHECKED_LONG_SUM($18)], sum(ResolutionWidth+19)=[CHECKED_LONG_SUM($19)], sum(ResolutionWidth+20)=[CHECKED_LONG_SUM($20)], sum(ResolutionWidth+21)=[CHECKED_LONG_SUM($21)], sum(ResolutionWidth+22)=[CHECKED_LONG_SUM($22)], sum(ResolutionWidth+23)=[CHECKED_LONG_SUM($23)], sum(ResolutionWidth+24)=[CHECKED_LONG_SUM($24)], sum(ResolutionWidth+25)=[CHECKED_LONG_SUM($25)], sum(ResolutionWidth+26)=[CHECKED_LONG_SUM($26)], sum(ResolutionWidth+27)=[CHECKED_LONG_SUM($27)], sum(ResolutionWidth+28)=[CHECKED_LONG_SUM($28)], sum(ResolutionWidth+29)=[CHECKED_LONG_SUM($29)], sum(ResolutionWidth+30)=[CHECKED_LONG_SUM($30)], sum(ResolutionWidth+31)=[CHECKED_LONG_SUM($31)], sum(ResolutionWidth+32)=[CHECKED_LONG_SUM($32)], sum(ResolutionWidth+33)=[CHECKED_LONG_SUM($33)], sum(ResolutionWidth+34)=[CHECKED_LONG_SUM($34)], sum(ResolutionWidth+35)=[CHECKED_LONG_SUM($35)], sum(ResolutionWidth+36)=[CHECKED_LONG_SUM($36)], sum(ResolutionWidth+37)=[CHECKED_LONG_SUM($37)], sum(ResolutionWidth+38)=[CHECKED_LONG_SUM($38)], sum(ResolutionWidth+39)=[CHECKED_LONG_SUM($39)], sum(ResolutionWidth+40)=[CHECKED_LONG_SUM($40)], sum(ResolutionWidth+41)=[CHECKED_LONG_SUM($41)], sum(ResolutionWidth+42)=[CHECKED_LONG_SUM($42)], sum(ResolutionWidth+43)=[CHECKED_LONG_SUM($43)], sum(ResolutionWidth+44)=[CHECKED_LONG_SUM($44)], sum(ResolutionWidth+45)=[CHECKED_LONG_SUM($45)], sum(ResolutionWidth+46)=[CHECKED_LONG_SUM($46)], sum(ResolutionWidth+47)=[CHECKED_LONG_SUM($47)], sum(ResolutionWidth+48)=[CHECKED_LONG_SUM($48)], sum(ResolutionWidth+49)=[CHECKED_LONG_SUM($49)], sum(ResolutionWidth+50)=[CHECKED_LONG_SUM($50)], sum(ResolutionWidth+51)=[CHECKED_LONG_SUM($51)], sum(ResolutionWidth+52)=[CHECKED_LONG_SUM($52)], sum(ResolutionWidth+53)=[CHECKED_LONG_SUM($53)], sum(ResolutionWidth+54)=[CHECKED_LONG_SUM($54)], sum(ResolutionWidth+55)=[CHECKED_LONG_SUM($55)], sum(ResolutionWidth+56)=[CHECKED_LONG_SUM($56)], sum(ResolutionWidth+57)=[CHECKED_LONG_SUM($57)], sum(ResolutionWidth+58)=[CHECKED_LONG_SUM($58)], sum(ResolutionWidth+59)=[CHECKED_LONG_SUM($59)], sum(ResolutionWidth+60)=[CHECKED_LONG_SUM($60)], sum(ResolutionWidth+61)=[CHECKED_LONG_SUM($61)], sum(ResolutionWidth+62)=[CHECKED_LONG_SUM($62)], sum(ResolutionWidth+63)=[CHECKED_LONG_SUM($63)], sum(ResolutionWidth+64)=[CHECKED_LONG_SUM($64)], sum(ResolutionWidth+65)=[CHECKED_LONG_SUM($65)], sum(ResolutionWidth+66)=[CHECKED_LONG_SUM($66)], sum(ResolutionWidth+67)=[CHECKED_LONG_SUM($67)], sum(ResolutionWidth+68)=[CHECKED_LONG_SUM($68)], sum(ResolutionWidth+69)=[CHECKED_LONG_SUM($69)], sum(ResolutionWidth+70)=[CHECKED_LONG_SUM($70)], sum(ResolutionWidth+71)=[CHECKED_LONG_SUM($71)], sum(ResolutionWidth+72)=[CHECKED_LONG_SUM($72)], sum(ResolutionWidth+73)=[CHECKED_LONG_SUM($73)], sum(ResolutionWidth+74)=[CHECKED_LONG_SUM($74)], sum(ResolutionWidth+75)=[CHECKED_LONG_SUM($75)], sum(ResolutionWidth+76)=[CHECKED_LONG_SUM($76)], sum(ResolutionWidth+77)=[CHECKED_LONG_SUM($77)], sum(ResolutionWidth+78)=[CHECKED_LONG_SUM($78)], sum(ResolutionWidth+79)=[CHECKED_LONG_SUM($79)], sum(ResolutionWidth+80)=[CHECKED_LONG_SUM($80)], sum(ResolutionWidth+81)=[CHECKED_LONG_SUM($81)], sum(ResolutionWidth+82)=[CHECKED_LONG_SUM($82)], sum(ResolutionWidth+83)=[CHECKED_LONG_SUM($83)], sum(ResolutionWidth+84)=[CHECKED_LONG_SUM($84)], sum(ResolutionWidth+85)=[CHECKED_LONG_SUM($85)], sum(ResolutionWidth+86)=[CHECKED_LONG_SUM($86)], sum(ResolutionWidth+87)=[CHECKED_LONG_SUM($87)], sum(ResolutionWidth+88)=[CHECKED_LONG_SUM($88)], sum(ResolutionWidth+89)=[CHECKED_LONG_SUM($89)]) EnumerableCalc(expr#0=[{inputs}], expr#1=[CAST($t0):BIGINT], expr#2=[1:BIGINT], expr#3=[+($t1, $t2)], expr#4=[2:BIGINT], expr#5=[+($t1, $t4)], expr#6=[3:BIGINT], expr#7=[+($t1, $t6)], expr#8=[4:BIGINT], expr#9=[+($t1, $t8)], expr#10=[5:BIGINT], expr#11=[+($t1, $t10)], expr#12=[6:BIGINT], expr#13=[+($t1, $t12)], expr#14=[7:BIGINT], expr#15=[+($t1, $t14)], expr#16=[8:BIGINT], expr#17=[+($t1, $t16)], expr#18=[9:BIGINT], expr#19=[+($t1, $t18)], expr#20=[10:BIGINT], expr#21=[+($t1, $t20)], expr#22=[11:BIGINT], expr#23=[+($t1, $t22)], expr#24=[12:BIGINT], expr#25=[+($t1, $t24)], expr#26=[13:BIGINT], expr#27=[+($t1, $t26)], expr#28=[14:BIGINT], expr#29=[+($t1, $t28)], expr#30=[15:BIGINT], expr#31=[+($t1, $t30)], expr#32=[16:BIGINT], expr#33=[+($t1, $t32)], expr#34=[17:BIGINT], expr#35=[+($t1, $t34)], expr#36=[18:BIGINT], expr#37=[+($t1, $t36)], expr#38=[19:BIGINT], expr#39=[+($t1, $t38)], expr#40=[20:BIGINT], expr#41=[+($t1, $t40)], expr#42=[21:BIGINT], expr#43=[+($t1, $t42)], expr#44=[22:BIGINT], expr#45=[+($t1, $t44)], expr#46=[23:BIGINT], expr#47=[+($t1, $t46)], expr#48=[24:BIGINT], expr#49=[+($t1, $t48)], expr#50=[25:BIGINT], expr#51=[+($t1, $t50)], expr#52=[26:BIGINT], expr#53=[+($t1, $t52)], expr#54=[27:BIGINT], expr#55=[+($t1, $t54)], expr#56=[28:BIGINT], expr#57=[+($t1, $t56)], expr#58=[29:BIGINT], expr#59=[+($t1, $t58)], expr#60=[30:BIGINT], expr#61=[+($t1, $t60)], expr#62=[31:BIGINT], expr#63=[+($t1, $t62)], expr#64=[32:BIGINT], expr#65=[+($t1, $t64)], expr#66=[33:BIGINT], expr#67=[+($t1, $t66)], expr#68=[34:BIGINT], expr#69=[+($t1, $t68)], expr#70=[35:BIGINT], expr#71=[+($t1, $t70)], expr#72=[36:BIGINT], expr#73=[+($t1, $t72)], expr#74=[37:BIGINT], expr#75=[+($t1, $t74)], expr#76=[38:BIGINT], expr#77=[+($t1, $t76)], expr#78=[39:BIGINT], expr#79=[+($t1, $t78)], expr#80=[40:BIGINT], expr#81=[+($t1, $t80)], expr#82=[41:BIGINT], expr#83=[+($t1, $t82)], expr#84=[42:BIGINT], expr#85=[+($t1, $t84)], expr#86=[43:BIGINT], expr#87=[+($t1, $t86)], expr#88=[44:BIGINT], expr#89=[+($t1, $t88)], expr#90=[45:BIGINT], expr#91=[+($t1, $t90)], expr#92=[46:BIGINT], expr#93=[+($t1, $t92)], expr#94=[47:BIGINT], expr#95=[+($t1, $t94)], expr#96=[48:BIGINT], expr#97=[+($t1, $t96)], expr#98=[49:BIGINT], expr#99=[+($t1, $t98)], expr#100=[50:BIGINT], expr#101=[+($t1, $t100)], expr#102=[51:BIGINT], expr#103=[+($t1, $t102)], expr#104=[52:BIGINT], expr#105=[+($t1, $t104)], expr#106=[53:BIGINT], expr#107=[+($t1, $t106)], expr#108=[54:BIGINT], expr#109=[+($t1, $t108)], expr#110=[55:BIGINT], expr#111=[+($t1, $t110)], expr#112=[56:BIGINT], expr#113=[+($t1, $t112)], expr#114=[57:BIGINT], expr#115=[+($t1, $t114)], expr#116=[58:BIGINT], expr#117=[+($t1, $t116)], expr#118=[59:BIGINT], expr#119=[+($t1, $t118)], expr#120=[60:BIGINT], expr#121=[+($t1, $t120)], expr#122=[61:BIGINT], expr#123=[+($t1, $t122)], expr#124=[62:BIGINT], expr#125=[+($t1, $t124)], expr#126=[63:BIGINT], expr#127=[+($t1, $t126)], expr#128=[64:BIGINT], expr#129=[+($t1, $t128)], expr#130=[65:BIGINT], expr#131=[+($t1, $t130)], expr#132=[66:BIGINT], expr#133=[+($t1, $t132)], expr#134=[67:BIGINT], expr#135=[+($t1, $t134)], expr#136=[68:BIGINT], expr#137=[+($t1, $t136)], expr#138=[69:BIGINT], expr#139=[+($t1, $t138)], expr#140=[70:BIGINT], expr#141=[+($t1, $t140)], expr#142=[71:BIGINT], expr#143=[+($t1, $t142)], expr#144=[72:BIGINT], expr#145=[+($t1, $t144)], expr#146=[73:BIGINT], expr#147=[+($t1, $t146)], expr#148=[74:BIGINT], expr#149=[+($t1, $t148)], expr#150=[75:BIGINT], expr#151=[+($t1, $t150)], expr#152=[76:BIGINT], expr#153=[+($t1, $t152)], expr#154=[77:BIGINT], expr#155=[+($t1, $t154)], expr#156=[78:BIGINT], expr#157=[+($t1, $t156)], expr#158=[79:BIGINT], expr#159=[+($t1, $t158)], expr#160=[80:BIGINT], expr#161=[+($t1, $t160)], expr#162=[81:BIGINT], expr#163=[+($t1, $t162)], expr#164=[82:BIGINT], expr#165=[+($t1, $t164)], expr#166=[83:BIGINT], expr#167=[+($t1, $t166)], expr#168=[84:BIGINT], expr#169=[+($t1, $t168)], expr#170=[85:BIGINT], expr#171=[+($t1, $t170)], expr#172=[86:BIGINT], expr#173=[+($t1, $t172)], expr#174=[87:BIGINT], expr#175=[+($t1, $t174)], expr#176=[88:BIGINT], expr#177=[+($t1, $t176)], expr#178=[89:BIGINT], expr#179=[+($t1, $t178)], ResolutionWidth=[$t0], $f90=[$t3], $f91=[$t5], $f92=[$t7], $f93=[$t9], $f94=[$t11], $f95=[$t13], $f96=[$t15], $f97=[$t17], $f98=[$t19], $f99=[$t21], $f100=[$t23], $f101=[$t25], $f102=[$t27], $f103=[$t29], $f104=[$t31], $f105=[$t33], $f106=[$t35], $f107=[$t37], $f108=[$t39], $f109=[$t41], $f110=[$t43], $f111=[$t45], $f112=[$t47], $f113=[$t49], $f114=[$t51], $f115=[$t53], $f116=[$t55], $f117=[$t57], $f118=[$t59], $f119=[$t61], $f120=[$t63], $f121=[$t65], $f122=[$t67], $f123=[$t69], $f124=[$t71], $f125=[$t73], $f126=[$t75], $f127=[$t77], $f128=[$t79], $f129=[$t81], $f130=[$t83], $f131=[$t85], $f132=[$t87], $f133=[$t89], $f134=[$t91], $f135=[$t93], $f136=[$t95], $f137=[$t97], $f138=[$t99], $f139=[$t101], $f140=[$t103], $f141=[$t105], $f142=[$t107], $f143=[$t109], $f144=[$t111], $f145=[$t113], $f146=[$t115], $f147=[$t117], $f148=[$t119], $f149=[$t121], $f150=[$t123], $f151=[$t125], $f152=[$t127], $f153=[$t129], $f154=[$t131], $f155=[$t133], $f156=[$t135], $f157=[$t137], $f158=[$t139], $f159=[$t141], $f160=[$t143], $f161=[$t145], $f162=[$t147], $f163=[$t149], $f164=[$t151], $f165=[$t153], $f166=[$t155], $f167=[$t157], $f168=[$t159], $f169=[$t161], $f170=[$t163], $f171=[$t165], $f172=[$t167], $f173=[$t169], $f174=[$t171], $f175=[$t173], $f176=[$t175], $f177=[$t177], $f178=[$t179]) CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[ResolutionWidth]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["ResolutionWidth"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q31.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q31.yaml index bf40fe857ed..12fc0646da6 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q31.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q31.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) LogicalProject(c=[$2], sum(IsRefresh)=[$3], avg(ResolutionWidth)=[$4], SearchEngineID=[$0], ClientIP=[$1]) - LogicalAggregate(group=[{0, 1}], c=[COUNT()], sum(IsRefresh)=[SUM($2)], avg(ResolutionWidth)=[AVG($3)]) + LogicalAggregate(group=[{0, 1}], c=[COUNT()], sum(IsRefresh)=[CHECKED_LONG_SUM($2)], avg(ResolutionWidth)=[AVG($3)]) LogicalProject(SearchEngineID=[$65], ClientIP=[$76], IsRefresh=[$72], ResolutionWidth=[$80]) LogicalFilter(condition=[AND(IS NOT NULL($65), IS NOT NULL($76))]) LogicalFilter(condition=[<>($63, '')]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(<>($0, ''), IS NOT NULL($1), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1, 3},c=COUNT(),sum(IsRefresh)=SUM($2),avg(ResolutionWidth)=AVG($4)), PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), SearchEngineID, ClientIP], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"SearchEngineID","boost":1.0}},{"exists":{"field":"ClientIP","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"SearchEngineID|ClientIP":{"multi_terms":{"terms":[{"field":"SearchEngineID"},{"field":"ClientIP"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(IsRefresh)":{"sum":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(<>($0, ''), IS NOT NULL($1), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1, 3},c=COUNT(),sum(IsRefresh)=CHECKED_LONG_SUM($2),avg(ResolutionWidth)=AVG($4)), PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), SearchEngineID, ClientIP], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"SearchEngineID","boost":1.0}},{"exists":{"field":"ClientIP","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"SearchEngineID|ClientIP":{"multi_terms":{"terms":[{"field":"SearchEngineID"},{"field":"ClientIP"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(IsRefresh)":{"sum":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q32.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q32.yaml index 81236b33d51..9cdd38482bc 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q32.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q32.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) LogicalProject(c=[$2], sum(IsRefresh)=[$3], avg(ResolutionWidth)=[$4], WatchID=[$0], ClientIP=[$1]) - LogicalAggregate(group=[{0, 1}], c=[COUNT()], sum(IsRefresh)=[SUM($2)], avg(ResolutionWidth)=[AVG($3)]) + LogicalAggregate(group=[{0, 1}], c=[COUNT()], sum(IsRefresh)=[CHECKED_LONG_SUM($2)], avg(ResolutionWidth)=[AVG($3)]) LogicalProject(WatchID=[$41], ClientIP=[$76], IsRefresh=[$72], ResolutionWidth=[$80]) LogicalFilter(condition=[AND(IS NOT NULL($41), IS NOT NULL($76))]) LogicalFilter(condition=[<>($63, '')]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(<>($1, ''), IS NOT NULL($0), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 3},c=COUNT(),sum(IsRefresh)=SUM($2),avg(ResolutionWidth)=AVG($4)), PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), WatchID, ClientIP], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"WatchID","boost":1.0}},{"exists":{"field":"ClientIP","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"WatchID|ClientIP":{"multi_terms":{"terms":[{"field":"WatchID"},{"field":"ClientIP"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(IsRefresh)":{"sum":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(<>($1, ''), IS NOT NULL($0), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 3},c=COUNT(),sum(IsRefresh)=CHECKED_LONG_SUM($2),avg(ResolutionWidth)=AVG($4)), PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), WatchID, ClientIP], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"WatchID","boost":1.0}},{"exists":{"field":"ClientIP","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"WatchID|ClientIP":{"multi_terms":{"terms":[{"field":"WatchID"},{"field":"ClientIP"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(IsRefresh)":{"sum":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q33.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q33.yaml index ccda84ba38a..a64c682196a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q33.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q33.yaml @@ -3,9 +3,9 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) LogicalProject(c=[$2], sum(IsRefresh)=[$3], avg(ResolutionWidth)=[$4], WatchID=[$0], ClientIP=[$1]) - LogicalAggregate(group=[{0, 1}], c=[COUNT()], sum(IsRefresh)=[SUM($2)], avg(ResolutionWidth)=[AVG($3)]) + LogicalAggregate(group=[{0, 1}], c=[COUNT()], sum(IsRefresh)=[CHECKED_LONG_SUM($2)], avg(ResolutionWidth)=[AVG($3)]) LogicalProject(WatchID=[$41], ClientIP=[$76], IsRefresh=[$72], ResolutionWidth=[$80]) LogicalFilter(condition=[AND(IS NOT NULL($41), IS NOT NULL($76))]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},c=COUNT(),sum(IsRefresh)=SUM($1),avg(ResolutionWidth)=AVG($3)), PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), WatchID, ClientIP], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"WatchID|ClientIP":{"multi_terms":{"terms":[{"field":"WatchID"},{"field":"ClientIP"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(IsRefresh)":{"sum":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},c=COUNT(),sum(IsRefresh)=CHECKED_LONG_SUM($1),avg(ResolutionWidth)=AVG($3)), PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), WatchID, ClientIP], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"WatchID|ClientIP":{"multi_terms":{"terms":[{"field":"WatchID"},{"field":"ClientIP"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(IsRefresh)":{"sum":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure2.yaml index 9c41efa9139..48debf64773 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure2.yaml @@ -3,9 +3,9 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[DESC-nulls-last]) LogicalProject(sum=[$1], state=[$0]) - LogicalAggregate(group=[{0}], sum=[SUM($1)]) + LogicalAggregate(group=[{0}], sum=[CHECKED_LONG_SUM($1)]) LogicalProject(state=[$7], balance=[$3]) LogicalFilter(condition=[IS NOT NULL($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum=SUM($0)), PROJECT->[sum, state], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"state":{"terms":{"field":"state.keyword","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"sum":"desc"},{"_key":"asc"}]},"aggregations":{"sum":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum=CHECKED_LONG_SUM($0)), PROJECT->[sum, state], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"state":{"terms":{"field":"state.keyword","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"sum":"desc"},{"_key":"asc"}]},"aggregations":{"sum":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml index f2105ce0d3c..5dad5e945b4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[DESC-nulls-last]) LogicalProject(sum(balance)=[$1], span(age,5)=[$0]) - LogicalAggregate(group=[{1}], sum(balance)=[SUM($0)]) + LogicalAggregate(group=[{1}], sum(balance)=[CHECKED_LONG_SUM($0)]) LogicalProject(balance=[$7], span(age,5)=[SPAN($10, 5, null:NULL)]) LogicalFilter(condition=[IS NOT NULL($10)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum(balance)=SUM($0)), PROJECT->[sum(balance), span(age,5)], SORT_AGG_METRICS->[0 DESC LAST]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"span(age,5)":{"histogram":{"field":"age","interval":5.0,"offset":0.0,"order":[{"sum(balance)":"desc"},{"_key":"asc"}],"keyed":false,"min_doc_count":1},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum(balance)=CHECKED_LONG_SUM($0)), PROJECT->[sum(balance), span(age,5)], SORT_AGG_METRICS->[0 DESC LAST]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"span(age,5)":{"histogram":{"field":"age","interval":5.0,"offset":0.0,"order":[{"sum(balance)":"desc"},{"_key":"asc"}],"keyed":false,"min_doc_count":1},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex1.yaml index cd0355241fe..4e87192da1d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex1.yaml @@ -3,9 +3,9 @@ calcite: LogicalSystemLimit(sort0=[$1], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$1], dir0=[DESC-nulls-last]) LogicalProject(sum(balance)=[$1], c=[$2], dc(employer)=[$3], state=[$0]) - LogicalAggregate(group=[{0}], sum(balance)=[SUM($1)], c=[COUNT()], dc(employer)=[COUNT(DISTINCT $2)]) + LogicalAggregate(group=[{0}], sum(balance)=[CHECKED_LONG_SUM($1)], c=[COUNT()], dc(employer)=[COUNT(DISTINCT $2)]) LogicalProject(state=[$7], balance=[$3], employer=[$6]) LogicalFilter(condition=[IS NOT NULL($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={2},sum(balance)=SUM($0),c=COUNT(),dc(employer)=COUNT(DISTINCT $1)), PROJECT->[sum(balance), c, dc(employer), state], SORT_AGG_METRICS->[1 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"state":{"terms":{"field":"state.keyword","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"dc(employer)":{"cardinality":{"field":"employer.keyword"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={2},sum(balance)=CHECKED_LONG_SUM($0),c=COUNT(),dc(employer)=COUNT(DISTINCT $1)), PROJECT->[sum(balance), c, dc(employer), state], SORT_AGG_METRICS->[1 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"state":{"terms":{"field":"state.keyword","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"dc(employer)":{"cardinality":{"field":"employer.keyword"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex2.yaml index 59cd137ca59..3e3a45b6386 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex2.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$2], dir0=[DESC-nulls-last]) LogicalProject(sum(balance)=[$2], count()=[$3], d=[$4], gender=[$0], new_state=[$1]) - LogicalAggregate(group=[{0, 1}], sum(balance)=[SUM($2)], count()=[COUNT()], d=[COUNT(DISTINCT $3)]) + LogicalAggregate(group=[{0, 1}], sum(balance)=[CHECKED_LONG_SUM($2)], count()=[COUNT()], d=[COUNT(DISTINCT $3)]) LogicalProject(gender=[$4], new_state=[$17], balance=[$3], employer=[$6]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($17))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], new_state=[LOWER($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},sum(balance)=SUM($2),count()=COUNT(),d=COUNT(DISTINCT $3)), PROJECT->[sum(balance), count(), d, gender, new_state], SORT_AGG_METRICS->[2 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"gender|new_state":{"multi_terms":{"terms":[{"field":"gender.keyword"},{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}}}],"size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"d":"desc"},{"_key":"asc"}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"d":{"cardinality":{"field":"employer.keyword"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},sum(balance)=CHECKED_LONG_SUM($2),count()=COUNT(),d=COUNT(DISTINCT $3)), PROJECT->[sum(balance), count(), d, gender, new_state], SORT_AGG_METRICS->[2 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"gender|new_state":{"multi_terms":{"terms":[{"field":"gender.keyword"},{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}}}],"size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"d":"desc"},{"_key":"asc"}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"d":{"cardinality":{"field":"employer.keyword"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_buckets_not_pushed.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_buckets_not_pushed.yaml index 68cb12a49dd..c4c572ce9e5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_buckets_not_pushed.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_buckets_not_pushed.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[ASC-nulls-first]) LogicalProject(c=[$2], s=[$3], span(age,5)=[$1], state=[$0]) - LogicalAggregate(group=[{0, 2}], c=[COUNT()], s=[SUM($1)]) + LogicalAggregate(group=[{0, 2}], c=[COUNT()], s=[CHECKED_LONG_SUM($1)]) LogicalProject(state=[$7], balance=[$3], span(age,5)=[SPAN($8, 5, null:NULL)]) LogicalFilter(condition=[AND(IS NOT NULL($8), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableTopK(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},c=COUNT(),s=SUM($1)), PROJECT->[c, s, span(age,5), state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}},{"span(age,5)":{"histogram":{"field":"age","missing_bucket":false,"order":"asc","interval":5.0}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},c=COUNT(),s=CHECKED_LONG_SUM($1)), PROJECT->[c, s, span(age,5), state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}},{"span(age,5)":{"histogram":{"field":"age","missing_bucket":false,"order":"asc","interval":5.0}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml index bc65d5c4c29..a1d7c439d9e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(sum=[$2], len=[$0], gender=[$1]) - LogicalAggregate(group=[{0, 1}], sum=[SUM($2)]) + LogicalAggregate(group=[{0, 1}], sum=[CHECKED_LONG_SUM($2)]) LogicalProject(len=[CHAR_LENGTH($4)], gender=[$4], $f3=[+($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..2=[{inputs}], expr#3=[100:BIGINT], expr#4=[*($t2, $t3)], expr#5=[+($t1, $t4)], expr#6=[CHAR_LENGTH($t0)], sum=[$t5], len=[$t6], gender=[$t0]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum_SUM=SUM($1),sum_COUNT=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum_SUM":{"sum":{"field":"balance"}},"sum_COUNT":{"value_count":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum_SUM=CHECKED_LONG_SUM($1),sum_COUNT=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum_SUM":{"sum":{"field":"balance"}},"sum_COUNT":{"value_count":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml index 1d664d5cd43..f6402d44d63 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(sum(balance)=[$1], sum(balance + 100)=[$2], sum(balance - 100)=[$3], sum(balance * 100)=[$4], sum(balance / 100)=[$5], gender=[$0]) - LogicalAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)=[SUM($2)], sum(balance - 100)=[SUM($3)], sum(balance * 100)=[SUM($4)], sum(balance / 100)=[SUM($5)]) + LogicalAggregate(group=[{0}], sum(balance)=[CHECKED_LONG_SUM($1)], sum(balance + 100)=[CHECKED_LONG_SUM($2)], sum(balance - 100)=[CHECKED_LONG_SUM($3)], sum(balance * 100)=[CHECKED_LONG_SUM($4)], sum(balance / 100)=[CHECKED_LONG_SUM($5)]) LogicalProject(gender=[$4], balance=[$7], $f6=[+($7, 100)], $f7=[-($7, 100)], $f8=[*($7, 100)], $f9=[DIVIDE($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100:BIGINT], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCEHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJESVZJREUiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",100]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=CHECKED_LONG_SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=CHECKED_LONG_SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCEHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJESVZJREUiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",100]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_minspan.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_minspan.json index 064aa294a2d..f265a37f292 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_minspan.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_minspan.json @@ -1 +1,6 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[MINSPAN_BUCKET($8, 5.0E0:DOUBLE, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[5.0E0:DOUBLE], expr#14=[-($t11, $t12)], expr#15=[MINSPAN_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15])\n EnumerableLimit(fetch=[5])\n EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n"}} \ No newline at end of file +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[MINSPAN_BUCKET($8, 5.0E0:DOUBLE, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[5.0E0:DOUBLE], expr#14=[-($t11, $t12)], expr#15=[MINSPAN_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15])\n EnumerableLimit(fetch=[5])\n EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push1.yaml index 3767a38b3a9..2fb92e7c652 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push1.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$0], sort1=[$2], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], sort1=[$2], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) LogicalProject(c=[$1], s=[$2], state=[$0]) - LogicalAggregate(group=[{0}], c=[COUNT()], s=[SUM($1)]) + LogicalAggregate(group=[{0}], c=[COUNT()], s=[CHECKED_LONG_SUM($1)]) LogicalProject(state=[$7], balance=[$3]) LogicalFilter(condition=[IS NOT NULL($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableTopK(sort0=[$0], sort1=[$2], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT(),s=SUM($0)), PROJECT->[c, s, state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT(),s=CHECKED_LONG_SUM($0)), PROJECT->[c, s, state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push2.yaml index 520f729b7f9..b5192ed01ce 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push2.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) LogicalProject(c=[$1], s=[$2], state=[$0]) - LogicalAggregate(group=[{0}], c=[COUNT()], s=[SUM($1)]) + LogicalAggregate(group=[{0}], c=[COUNT()], s=[CHECKED_LONG_SUM($1)]) LogicalProject(state=[$7], balance=[$3]) LogicalFilter(condition=[IS NOT NULL($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableTopK(sort0=[$0], sort1=[$1], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT(),s=SUM($0)), PROJECT->[c, s, state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT(),s=CHECKED_LONG_SUM($0)), PROJECT->[c, s, state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml index 0478b24369c..9955ef801c2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml @@ -10,9 +10,9 @@ calcite: LogicalProject(__r_seq__=[ROW_NUMBER() OVER ()], __r_gender__=[$4], __r_age__=[$8]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..19=[{inputs}], expr#20=[0], expr#21=[=($t19, $t20)], expr#22=[null:BIGINT], expr#23=[CASE($t21, $t22, $t18)], expr#24=[CAST($t23):DOUBLE], expr#25=[/($t24, $t19)], proj#0..10=[{exprs}], avg_age=[$t25]) + EnumerableCalc(expr#0..18=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t18]) CalciteEnumerableTopK(sort0=[$17], dir0=[ASC], fetch=[10000]) - EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], agg#0=[$SUM0($20)], agg#1=[COUNT($20)]) + EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], avg_age=[AVG($20)]) EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($4, $19), >=($18, -($17, 1)), <=($18, $17))], joinType=[left]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global_null_bucket.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global_null_bucket.yaml index a1cf6ae00e9..902af30e1d7 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global_null_bucket.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global_null_bucket.yaml @@ -10,9 +10,9 @@ calcite: LogicalProject(__r_seq__=[ROW_NUMBER() OVER ()], __r_gender__=[$4], __r_age__=[$8]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..19=[{inputs}], expr#20=[0], expr#21=[=($t19, $t20)], expr#22=[null:BIGINT], expr#23=[CASE($t21, $t22, $t18)], expr#24=[CAST($t23):DOUBLE], expr#25=[/($t24, $t19)], proj#0..10=[{exprs}], avg_age=[$t25]) + EnumerableCalc(expr#0..18=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t18]) CalciteEnumerableTopK(sort0=[$17], dir0=[ASC], fetch=[10000]) - EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], agg#0=[$SUM0($20)], agg#1=[COUNT($20)]) + EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], avg_age=[AVG($20)]) EnumerableMergeJoin(condition=[AND(=($4, $19), >=($18, -($17, 1)), <=($18, $17))], joinType=[left]) EnumerableSort(sort0=[$4], dir0=[ASC]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml index 324960f28dd..d0f762c426d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml @@ -22,17 +22,16 @@ calcite: EnumerableCalc(expr#0..11=[{inputs}], expr#12=[34], expr#13=[>($t8, $t12)], expr#14=[1], expr#15=[0], expr#16=[CASE($t13, $t14, $t15)], expr#17=[25], expr#18=[<($t8, $t17)], expr#19=[CASE($t18, $t14, $t15)], expr#20=[IS NULL($t4)], proj#0..11=[{exprs}], __reset_before_flag__=[$t16], __reset_after_flag__=[$t19], $14=[$t20]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t5, $t6)], expr#8=[null:BIGINT], expr#9=[CASE($t7, $t8, $t4)], expr#10=[CAST($t9):DOUBLE], expr#11=[/($t10, $t5)], proj#0..3=[{exprs}], avg_age=[$t11]) - EnumerableAggregate(group=[{0, 1, 2, 3}], agg#0=[$SUM0($5)], agg#1=[COUNT($5)]) - EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) - EnumerableAggregate(group=[{0, 1, 2, 3}]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], expr#11=[IS NULL($t0)], gender=[$t0], __stream_seq__=[$t2], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10], $4=[$t11]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], proj#0..2=[{exprs}], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) + EnumerableAggregate(group=[{0, 1, 2, 3}], avg_age=[AVG($5)]) + EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) + EnumerableAggregate(group=[{0, 1, 2, 3}]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], expr#11=[IS NULL($t0)], gender=[$t0], __stream_seq__=[$t2], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10], $4=[$t11]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], proj#0..2=[{exprs}], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset_null_bucket.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset_null_bucket.yaml index 42b50e7eb5f..213eef91aa2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset_null_bucket.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset_null_bucket.yaml @@ -24,17 +24,16 @@ calcite: EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], sort1=[$1], sort2=[$2], dir0=[ASC], dir1=[ASC], dir2=[ASC]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t4, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t3)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t4)], proj#0..2=[{exprs}], avg_age=[$t10]) - EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($4)], agg#1=[COUNT($4)]) - EnumerableHashJoin(condition=[AND(=($2, $6), =($0, $3), <($5, $1))], joinType=[inner]) - EnumerableAggregate(group=[{0, 1, 2}]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[COALESCE($t5, $t6)], expr#8=[+($t4, $t7)], proj#0..1=[{exprs}], __seg_id__=[$t8]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $4 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], gender=[$t0], __stream_seq__=[$t2], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], proj#0..2=[{exprs}], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) + EnumerableAggregate(group=[{0, 1, 2}], avg_age=[AVG($4)]) + EnumerableHashJoin(condition=[AND(=($2, $6), =($0, $3), <($5, $1))], joinType=[inner]) + EnumerableAggregate(group=[{0, 1, 2}]) + EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[COALESCE($t5, $t6)], expr#8=[+($t4, $t7)], proj#0..1=[{exprs}], __seg_id__=[$t8]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $4 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], gender=[$t0], __stream_seq__=[$t2], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], proj#0..2=[{exprs}], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_case_composite_cannot_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_case_composite_cannot_push.yaml index 059caa2e2d2..79252ca2148 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_case_composite_cannot_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_case_composite_cannot_push.yaml @@ -7,7 +7,7 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg_balance=[$t9], age_range=[$t0], state=[$t1]) - EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..2=[{inputs}], avg_balance=[$t2], age_range=[$t0], state=[$t1]) + EnumerableAggregate(group=[{0, 1}], avg_balance=[AVG($2)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[35], expr#20=[<($t10, $t19)], expr#21=['u35':VARCHAR], expr#22=[CASE($t20, $t21, $t11)], age_range=[$t22], state=[$t9], balance=[$t7]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_count_push.yaml index 43e27cd2d5d..b3ec99de2cf 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_count_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_count_push.yaml @@ -7,7 +7,7 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t4, $t6)], expr#8=[null:BIGINT], expr#9=[CASE($t7, $t8, $t3)], expr#10=[CAST($t9):DOUBLE], expr#11=[/($t10, $t4)], avg(balance)=[$t11], count()=[$t5], age_range=[$t0], state=[$t1], gender=[$t2]) - EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($3)], agg#1=[COUNT($3)], count()=[COUNT()]) + EnumerableCalc(expr#0..4=[{inputs}], avg(balance)=[$t3], count()=[$t4], age_range=[$t0], state=[$t1], gender=[$t2]) + EnumerableAggregate(group=[{0, 1, 2}], avg(balance)=[AVG($3)], count()=[COUNT()]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=['a30':VARCHAR], expr#23=[CASE($t20, $t21, $t22)], age_range=[$t23], state=[$t9], gender=[$t4], balance=[$t7]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_range_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_range_count_push.yaml index 6dfa7cd65a3..e49dee2d350 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_range_count_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_range_count_push.yaml @@ -7,7 +7,7 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t4, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t3)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t4)], avg_balance=[$t10], age_range=[$t0], balance_range=[$t1], state=[$t2]) - EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($3)], agg#1=[COUNT($3)]) + EnumerableCalc(expr#0..3=[{inputs}], avg_balance=[$t3], age_range=[$t0], balance_range=[$t1], state=[$t2]) + EnumerableAggregate(group=[{0, 1, 2}], avg_balance=[AVG($3)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[35], expr#20=[<($t10, $t19)], expr#21=['u35':VARCHAR], expr#22=['a35':VARCHAR], expr#23=[CASE($t20, $t21, $t22)], expr#24=[20000], expr#25=[<($t7, $t24)], expr#26=['medium':VARCHAR], expr#27=['high':VARCHAR], expr#28=[CASE($t25, $t26, $t27)], age_range=[$t23], balance_range=[$t28], state=[$t9], balance=[$t7]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_range_metric_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_range_metric_push.yaml index 41ed8ba61fc..1dd00811008 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_range_metric_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_range_metric_push.yaml @@ -7,7 +7,7 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg(balance)=[$t9], state=[$t0], age_range=[$t1]) - EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..2=[{inputs}], avg(balance)=[$t2], state=[$t0], age_range=[$t1]) + EnumerableAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=['a30':VARCHAR], expr#23=[CASE($t20, $t21, $t22)], state=[$t9], age_range=[$t23], balance=[$t7]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_count_push.yaml index 67ad0f0fd07..1ae9205fa10 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_count_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_count_push.yaml @@ -10,4 +10,4 @@ calcite: EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg(age)=[$t8], age_range=[$t0]) EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[Sarg[[30..40)]], expr#23=[SEARCH($t10, $t22)], expr#24=['u40':VARCHAR], expr#25=['u100':VARCHAR], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], age_range=[$t26], age=[$t10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_complex_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_complex_push.yaml index 10ead7ad449..14664cb5df6 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_complex_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_complex_push.yaml @@ -7,7 +7,7 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg(balance)=[$t8], age_range=[$t0]) - EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..1=[{inputs}], avg(balance)=[$t1], age_range=[$t0]) + EnumerableAggregate(group=[{0}], avg(balance)=[AVG($1)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[Sarg[[35..40), [80..+∞)]], expr#23=[SEARCH($t10, $t22)], expr#24=['30-40 or >=80':VARCHAR], expr#25=[null:NULL], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], age_range=[$t26], balance=[$t7]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_push.yaml index a81e208bdbf..6a33abfd5df 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_push.yaml @@ -10,4 +10,4 @@ calcite: EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg_age=[$t8], age_range=[$t0]) EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[40], expr#23=[<($t10, $t22)], expr#24=['u40':VARCHAR], expr#25=['u100':VARCHAR], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], age_range=[$t26], age=[$t10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_range_metric_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_range_metric_push.yaml index 404726f6083..1aee0d0ced4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_range_metric_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_range_metric_push.yaml @@ -7,7 +7,7 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg_balance=[$t9], age_range=[$t0], balance_range=[$t1]) - EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..2=[{inputs}], avg_balance=[$t2], age_range=[$t0], balance_range=[$t1]) + EnumerableAggregate(group=[{0, 1}], avg_balance=[AVG($2)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[40], expr#23=[<($t10, $t22)], expr#24=['u40':VARCHAR], expr#25=['u100':VARCHAR], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], expr#27=[20000], expr#28=[<($t7, $t27)], expr#29=['medium':VARCHAR], expr#30=['high':VARCHAR], expr#31=[CASE($t28, $t29, $t30)], age_range=[$t26], balance_range=[$t31], balance=[$t7]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml index fe925e0a80a..1d1b9bfec33 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml @@ -26,15 +26,15 @@ calcite: EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{4, 10}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], gender=[$t0], age=[$t3], avg(balance)=[$t2]) + EnumerableAggregate(group=[{4, 10}], avg(balance)=[AVG($7)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t19, $t20)], proj#0..18=[{exprs}], $condition=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{4, 10}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], age=[$t3], avg(balance)=[$t2]) + EnumerableAggregate(group=[{4, 10}], avg(balance)=[AVG($7)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], expr#20=[IS NOT NULL($t7)], expr#21=[SAFE_CAST($t10)], expr#22=[IS NOT NULL($t21)], expr#23=[AND($t19, $t20, $t22)], proj#0..18=[{exprs}], $condition=[$t23]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml index beb3275a6c6..1876916cb25 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml @@ -26,15 +26,15 @@ calcite: EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['nil'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], gender=[$t0], age=[$t3], avg(balance)=[$t2]) + EnumerableAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], expr#16=[IS NOT NULL($t4)], expr#17=[IS NOT NULL($t3)], expr#18=[AND($t16, $t17)], gender=[$t4], balance=[$t3], age0=[$t15], $condition=[$t18]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], age=[$t3], avg(balance)=[$t2]) + EnumerableAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], expr#16=[IS NOT NULL($t4)], expr#17=[IS NOT NULL($t3)], expr#18=[SAFE_CAST($t15)], expr#19=[IS NOT NULL($t18)], expr#20=[AND($t16, $t17, $t19)], gender=[$t4], balance=[$t3], age0=[$t15], $condition=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_single_group_key.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_single_group_key.yaml index 8224f075819..d6fd5118aa5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_single_group_key.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_single_group_key.yaml @@ -9,7 +9,6 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], gender=[$t0], avg(balance)=[$t8]) - EnumerableAggregate(group=[{4}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t19, $t20)], proj#0..18=[{exprs}], $condition=[$t21]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + EnumerableAggregate(group=[{4}], avg(balance)=[AVG($7)]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t19, $t20)], proj#0..18=[{exprs}], $condition=[$t21]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_limit.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_limit.yaml index 16aa3871687..52fb3848a8d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_limit.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_limit.yaml @@ -9,7 +9,7 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], state=[$t1], gender=[$t0], avg(balance)=[$t9]) - EnumerableAggregate(group=[{4, 9}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) + EnumerableCalc(expr#0..2=[{inputs}], state=[$t1], gender=[$t0], avg(balance)=[$t2]) + EnumerableAggregate(group=[{4, 9}], avg(balance)=[AVG($7)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t9)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t19, $t20)], proj#0..18=[{exprs}], $condition=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml index 1db12fc013f..a5d14f85dcf 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml @@ -2,11 +2,11 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(sum=[$2], len=[$0], gender=[$1]) - LogicalAggregate(group=[{0, 1}], sum=[SUM($2)]) + LogicalAggregate(group=[{0, 1}], sum=[CHECKED_LONG_SUM($2)]) LogicalProject(len=[CHAR_LENGTH($4)], gender=[$4], $f3=[+($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[100:BIGINT], expr#8=[*($t2, $t7)], expr#9=[+($t6, $t8)], expr#10=[CHAR_LENGTH($t0)], sum=[$t9], len=[$t10], gender=[$t0]) - EnumerableAggregate(group=[{4}], sum_SUM=[$SUM0($7)], agg#1=[COUNT($7)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[100:BIGINT], expr#4=[*($t2, $t3)], expr#5=[+($t1, $t4)], expr#6=[CHAR_LENGTH($t0)], sum=[$t5], len=[$t6], gender=[$t0]) + EnumerableAggregate(group=[{4}], sum_SUM=[CHECKED_LONG_SUM($7)], sum_COUNT=[COUNT($7)]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml index 655e16839ed..0a06b733276 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml @@ -2,12 +2,12 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(sum(balance)=[$1], sum(balance + 100)=[$2], sum(balance - 100)=[$3], sum(balance * 100)=[$4], sum(balance / 100)=[$5], gender=[$0]) - LogicalAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)=[SUM($2)], sum(balance - 100)=[SUM($3)], sum(balance * 100)=[SUM($4)], sum(balance / 100)=[SUM($5)]) + LogicalAggregate(group=[{0}], sum(balance)=[CHECKED_LONG_SUM($1)], sum(balance + 100)=[CHECKED_LONG_SUM($2)], sum(balance - 100)=[CHECKED_LONG_SUM($3)], sum(balance * 100)=[CHECKED_LONG_SUM($4)], sum(balance / 100)=[CHECKED_LONG_SUM($5)]) LogicalProject(gender=[$4], balance=[$7], $f6=[+($7, 100)], $f7=[-($7, 100)], $f8=[*($7, 100)], $f9=[DIVIDE($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100:BIGINT], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) - EnumerableAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)_COUNT=[COUNT($1)], sum(balance / 100)=[SUM($2)]) + EnumerableAggregate(group=[{0}], sum(balance)=[CHECKED_LONG_SUM($1)], sum(balance + 100)_COUNT=[COUNT($1)], sum(balance / 100)=[CHECKED_LONG_SUM($2)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[100], expr#20=[DIVIDE($t7, $t19)], gender=[$t4], balance=[$t7], $f5=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_minspan.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_minspan.json index a31d2acfc61..0b9e873d52c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_minspan.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_minspan.json @@ -1 +1,6 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[MINSPAN_BUCKET($8, 5.0E0:DOUBLE, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[5.0E0:DOUBLE], expr#14=[-($t11, $t12)], expr#15=[MINSPAN_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15])\n EnumerableLimit(fetch=[5])\n EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])])\n EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n"}} \ No newline at end of file +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[MINSPAN_BUCKET($8, 5.0E0:DOUBLE, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[5.0E0:DOUBLE], expr#14=[-($t11, $t12)], expr#15=[MINSPAN_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15])\n EnumerableLimit(fetch=[5])\n EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])])\n EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml index ac3728eacb9..a283b195883 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml @@ -8,7 +8,7 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg_age=[$t9], state=[$t1], city=[$t0]) - EnumerableAggregate(group=[{5, 7}], agg#0=[$SUM0($8)], agg#1=[COUNT($8)]) + EnumerableCalc(expr#0..2=[{inputs}], avg_age=[$t2], state=[$t1], city=[$t0]) + EnumerableAggregate(group=[{5, 7}], avg_age=[AVG($8)]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], proj#0..16=[{exprs}], $condition=[$t18]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml index f781995261c..5acfa39c392 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml @@ -19,7 +19,7 @@ calcite: EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) EnumerableWindow(window#0=[window(partition {1} order by [1 ASC-nulls-first] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[2], expr#11=[+($t9, $t10)], expr#12=[IS NOT NULL($t11)], state=[$t1], age2=[$t11], $condition=[$t12]) - EnumerableAggregate(group=[{5, 7}], agg#0=[$SUM0($8)], agg#1=[COUNT($8)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[2], expr#4=[+($t2, $t3)], expr#5=[IS NOT NULL($t2)], state=[$t1], age2=[$t4], $condition=[$t5]) + EnumerableAggregate(group=[{5, 7}], avg_age=[AVG($8)]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], proj#0..16=[{exprs}], $condition=[$t18]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_agg_push.json index 028a56cb020..126f559f4d0 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_agg_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_agg_push.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$0], dir0=[ASC-nulls-first])\n LogicalProject(avg_age=[$2], state=[$0], city=[$1])\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)])\n LogicalProject(state=[$7], city=[$5], age=[$8])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg_age=[$t9], state=[$t1], city=[$t0])\n EnumerableAggregate(group=[{5, 7}], agg#0=[$SUM0($8)], agg#1=[COUNT($8)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..2=[{inputs}], avg_age=[$t2], state=[$t1], city=[$t0])\n EnumerableAggregate(group=[{5, 7}], avg_age=[AVG($8)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_agg_push.json index 9d64b554b18..1be834a39ba 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_agg_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_agg_push.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(avg(balance)=[$1], state=[$0])\n LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)])\n LogicalProject(state=[$7], balance=[$3])\n LogicalSort(sort0=[$3], sort1=[$8], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg(balance)=[$t8], state=[$t0])\n EnumerableAggregate(group=[{7}], agg#0=[$SUM0($3)], agg#1=[COUNT($3)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], avg(balance)=[$t1], state=[$t0])\n EnumerableAggregate(group=[{7}], avg(balance)=[AVG($3)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml index 0bf9a2c50ce..f901475ef8d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml @@ -10,10 +10,10 @@ calcite: LogicalProject(__r_seq__=[ROW_NUMBER() OVER ()], __r_gender__=[$4], __r_age__=[$8]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..19=[{inputs}], expr#20=[0], expr#21=[=($t19, $t20)], expr#22=[null:BIGINT], expr#23=[CASE($t21, $t22, $t18)], expr#24=[CAST($t23):DOUBLE], expr#25=[/($t24, $t19)], proj#0..10=[{exprs}], avg_age=[$t25]) + EnumerableCalc(expr#0..18=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t18]) EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$17], dir0=[ASC]) - EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], agg#0=[$SUM0($20)], agg#1=[COUNT($20)]) + EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], avg_age=[AVG($20)]) EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($4, $19), >=($18, -($17, 1)), <=($18, $17))], joinType=[left]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global_null_bucket.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global_null_bucket.yaml index d72bf7b429f..2816bf68f20 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global_null_bucket.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global_null_bucket.yaml @@ -10,10 +10,10 @@ calcite: LogicalProject(__r_seq__=[ROW_NUMBER() OVER ()], __r_gender__=[$4], __r_age__=[$8]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..19=[{inputs}], expr#20=[0], expr#21=[=($t19, $t20)], expr#22=[null:BIGINT], expr#23=[CASE($t21, $t22, $t18)], expr#24=[CAST($t23):DOUBLE], expr#25=[/($t24, $t19)], proj#0..10=[{exprs}], avg_age=[$t25]) + EnumerableCalc(expr#0..18=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t18]) EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$17], dir0=[ASC]) - EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], agg#0=[$SUM0($20)], agg#1=[COUNT($20)]) + EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], avg_age=[AVG($20)]) EnumerableMergeJoin(condition=[AND(=($4, $19), >=($18, -($17, 1)), <=($18, $17))], joinType=[left]) EnumerableSort(sort0=[$4], dir0=[ASC]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml index 3ec98ba9382..b131d24ba2c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml @@ -23,17 +23,16 @@ calcite: EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], expr#26=[IS NULL($t4)], proj#0..10=[{exprs}], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25], $14=[$t26]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t5, $t6)], expr#8=[null:BIGINT], expr#9=[CASE($t7, $t8, $t4)], expr#10=[CAST($t9):DOUBLE], expr#11=[/($t10, $t5)], proj#0..3=[{exprs}], avg_age=[$t11]) - EnumerableAggregate(group=[{0, 1, 2, 3}], agg#0=[$SUM0($5)], agg#1=[COUNT($5)]) - EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) - EnumerableAggregate(group=[{0, 1, 2, 3}]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], expr#26=[IS NULL($t4)], gender=[$t4], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25], $4=[$t26]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], age=[$t8], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) + EnumerableAggregate(group=[{0, 1, 2, 3}], avg_age=[AVG($5)]) + EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) + EnumerableAggregate(group=[{0, 1, 2, 3}]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], expr#26=[IS NULL($t4)], gender=[$t4], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25], $4=[$t26]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], age=[$t8], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset_null_bucket.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset_null_bucket.yaml index 40fb4087001..e0ceaba3192 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset_null_bucket.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset_null_bucket.yaml @@ -23,17 +23,16 @@ calcite: EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], proj#0..10=[{exprs}], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t4, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t3)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t4)], proj#0..2=[{exprs}], avg_age=[$t10]) - EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($4)], agg#1=[COUNT($4)]) - EnumerableHashJoin(condition=[AND(=($2, $6), =($0, $3), <($5, $1))], joinType=[inner]) - EnumerableAggregate(group=[{0, 1, 2}]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[COALESCE($t5, $t6)], expr#8=[+($t4, $t7)], proj#0..1=[{exprs}], __seg_id__=[$t8]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $4 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], age=[$t8], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) + EnumerableAggregate(group=[{0, 1, 2}], avg_age=[AVG($4)]) + EnumerableHashJoin(condition=[AND(=($2, $6), =($0, $3), <($5, $1))], joinType=[inner]) + EnumerableAggregate(group=[{0, 1, 2}]) + EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[COALESCE($t5, $t6)], expr#8=[+($t4, $t7)], proj#0..1=[{exprs}], __seg_id__=[$t8]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $4 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], age=[$t8], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5164_agg.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5164_agg.yml new file mode 100644 index 00000000000..d638ed4495f --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5164_agg.yml @@ -0,0 +1,230 @@ +# Issue: https://github.com/opensearch-project/sql/issues/5164 +# SUM/AVG over a BIGINT (long) column near 2^63 must not silently wrap to a negative value. +# +# The enumerable SUM accumulator for a long argument is a plain long, so a running sum past 2^63 +# wraps to a negative result (e.g. SUM(long) returned -5594372458244005145). AVG reduces to +# SUM(field)/COUNT(field), so its intermediate long SUM wraps the same way (AVG returned a negative +# average). SUM(long) now uses checked BIGINT accumulation with Math.addExact in the Calcite +# fallback. The pushdown path uses OpenSearch's native double-based sum and checks its final value +# before narrowing to BIGINT. AVG(long) is averaged in DOUBLE, which holds the true average without +# wrapping. + +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: true + - do: + indices.create: + index: test_agg_overflow + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + long_field: + type: long + - do: + bulk: + index: test_agg_overflow + refresh: true + body: + - '{"index": {}}' + - '{"long_field": 9223372036854775807}' + - '{"index": {}}' + - '{"long_field": 9223372036854775807}' + - '{"index": {}}' + - '{"long_field": 9223372036854775807}' + - do: + indices.create: + index: test_agg_in_range + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + long_field: + type: long + - do: + bulk: + index: test_agg_in_range + refresh: true + body: + - '{"index": {}}' + - '{"long_field": 1000000000000}' + - '{"index": {}}' + - '{"long_field": 2000000000000}' + - '{"index": {}}' + - '{"long_field": 3000000000000}' + - do: + indices.create: + index: test_agg_exact + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + long_field: + type: long + - do: + bulk: + index: test_agg_exact + refresh: true + body: + - '{"index": {}}' + - '{"long_field": 4611686018427387904}' + - '{"index": {}}' + - '{"long_field": 1}' + - do: + indices.create: + index: test_agg_reduce_left + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + long_field: + type: long + - do: + index: + index: test_agg_reduce_left + refresh: true + body: + long_field: 9223372036854775807 + - do: + indices.create: + index: test_agg_reduce_right + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + long_field: + type: long + - do: + index: + index: test_agg_reduce_right + refresh: true + body: + long_field: 4096 + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: false + - do: + indices.delete: + index: test_agg_overflow + ignore_unavailable: true + - do: + indices.delete: + index: test_agg_in_range + ignore_unavailable: true + - do: + indices.delete: + index: test_agg_exact + ignore_unavailable: true + - do: + indices.delete: + index: test_agg_reduce_* + ignore_unavailable: true + +--- +"SUM of large longs overflowing BIGINT throws error": + - skip: + features: + - headers + # 3 * (2^63 - 1) far exceeds the BIGINT range; historically this wrapped to a negative value. + - do: + catch: bad_request + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_agg_overflow | stats sum(long_field) + - match: { "$body": "/[Oo]verflow/" } + +--- +"Pushed SUM whose final value exceeds BIGINT throws error": + - skip: + features: + - headers + # Each one-shard index has an in-range partial. The final result is far enough beyond 2^63 to be + # distinguishable from Long.MAX_VALUE after native double accumulation. + - do: + catch: bad_request + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_agg_reduce_* | stats sum(long_field) + - match: { "$body": "/[Oo]verflow/" } + +--- +"AVG of large longs does not overflow or error": + - skip: + features: + - headers + # The average of three identical values is that value; averaging in DOUBLE avoids the wrap that + # a long intermediate SUM would cause. 9223372036854775807 is returned as a double (9.223...E18). + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_agg_overflow | stats avg(long_field) + - match: { total: 1 } + - match: { datarows: [[9.223372036854776e18]] } + +--- +"SUM of longs within BIGINT range does not error": + - skip: + features: + - headers + # 1e12 + 2e12 + 3e12 = 6e12, well within long range; must return the exact sum, no error. + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_agg_in_range | stats sum(long_field) + - match: { total: 1 } + - match: { datarows: [[6000000000000]] } + +--- +"Pushed SUM of large in-range longs uses native double precision": + - skip: + features: + - headers + # 2^62 + 1 fits in BIGINT but cannot be represented exactly by a double accumulator. + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_agg_exact | stats sum(long_field) + - match: { total: 1 } + - match: { datarows: [[4611686018427387904]] } + +--- +"Fallback SUM of large in-range longs retains low-order precision": + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_agg_exact | head 2 | stats sum(long_field) + - match: { total: 1 } + - match: { datarows: [[4611686018427387905]] } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java index 775b0278683..5a336a2e500 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java @@ -89,11 +89,13 @@ import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.PPLBuiltinOperators; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.request.PredicateAnalyzer.NamedFieldExpression; import org.opensearch.sql.opensearch.request.PredicateAnalyzer.ScriptQueryExpression; import org.opensearch.sql.opensearch.response.agg.ArgMaxMinParser; import org.opensearch.sql.opensearch.response.agg.BucketAggregationParser; +import org.opensearch.sql.opensearch.response.agg.CheckedLongSumParser; import org.opensearch.sql.opensearch.response.agg.CountAsTotalHitsParser; import org.opensearch.sql.opensearch.response.agg.MetricParser; import org.opensearch.sql.opensearch.response.agg.NoBucketAggregationParser; @@ -477,6 +479,10 @@ private static Pair createRegularAggregation( AggregateBuilderHelper helper, List dedupSortKeys) { + if (aggCall.getAggregation() == PPLBuiltinOperators.CHECKED_LONG_SUM) { + return createCheckedLongSumAggregation(args, aggName, helper); + } + return switch (aggCall.getAggregation().kind) { case AVG -> Pair.of( @@ -651,6 +657,17 @@ yield switch (functionName) { }; } + private static Pair createCheckedLongSumAggregation( + List> args, String aggName, AggregateBuilderHelper helper) { + if (args.size() != 1) { + throw new AggregateAnalyzerException("CHECKED_LONG_SUM requires exactly one argument"); + } + + return Pair.of( + helper.build(args.getFirst().getKey(), AggregationBuilders.sum(aggName)), + new CheckedLongSumParser(aggName)); + } + private static boolean supportsMaxMinAggregation(ExprType fieldType) { ExprType coreType = (fieldType instanceof OpenSearchDataType) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/CheckedLongSumParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/CheckedLongSumParser.java new file mode 100644 index 00000000000..7ea18298b13 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/CheckedLongSumParser.java @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.response.agg; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.opensearch.search.aggregations.Aggregation; +import org.opensearch.search.aggregations.metrics.NumericMetricsAggregation; + +/** + * Narrows OpenSearch's double-based native sum to the BIGINT type declared by CHECKED_LONG_SUM. + * + *

OpenSearch may already have lost low-order precision before this parser receives the result. A + * double at the positive BIGINT boundary is also ambiguous because {@code Long.MAX_VALUE} rounds to + * {@code 2^63}; Java's narrowing conversion saturates that value to {@code Long.MAX_VALUE}. + */ +@EqualsAndHashCode +@RequiredArgsConstructor +public class CheckedLongSumParser implements MetricParser { + + private static final double TWO_POW_63 = 0x1p63; + + @Getter private final String name; + + @Override + public List> parse(Aggregation aggregation) { + double value = ((NumericMetricsAggregation.SingleValue) aggregation).value(); + Long narrowed = Double.isNaN(value) ? null : narrow(value); + return Collections.singletonList( + new HashMap<>(Collections.singletonMap(aggregation.getName(), narrowed))); + } + + static long narrow(double value) { + if (!Double.isFinite(value) || value > TWO_POW_63 || value < -TWO_POW_63) { + throw new ArithmeticException("BIGINT overflow in SUM"); + } + return (long) value; + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java index 4779332abac..0995a60adeb 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java @@ -41,12 +41,14 @@ import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.Test; import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.metrics.SumAggregationBuilder; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.function.PPLBuiltinOperators; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType.MappingType; import org.opensearch.sql.opensearch.request.AggregateAnalyzer.ExpressionNotAnalyzableException; import org.opensearch.sql.opensearch.response.agg.BucketAggregationParser; +import org.opensearch.sql.opensearch.response.agg.CheckedLongSumParser; import org.opensearch.sql.opensearch.response.agg.FilterParser; import org.opensearch.sql.opensearch.response.agg.MetricParserHelper; import org.opensearch.sql.opensearch.response.agg.NoBucketAggregationParser; @@ -176,6 +178,64 @@ void analyze_aggCall_simple() throws ExpressionNotAnalyzableException { }); } + @Test + void analyze_checkedLongSum() throws ExpressionNotAnalyzableException { + AggregateCall checkedLongSumCall = + AggregateCall.create( + PPLBuiltinOperators.CHECKED_LONG_SUM, + false, + false, + false, + ImmutableList.of(), + ImmutableList.of(0), + -1, + null, + RelCollations.EMPTY, + typeFactory.createSqlType(SqlTypeName.BIGINT), + "checked_sum"); + Aggregate aggregate = createMockAggregate(List.of(checkedLongSumCall), ImmutableBitSet.of()); + Project project = createMockProject(List.of(0)); + AggregateAnalyzer.AggregateBuilderHelper helper = + new AggregateAnalyzer.AggregateBuilderHelper(rowType, fieldTypes, null, true, BUCKET_SIZE); + + Pair, OpenSearchAggregationResponseParser> result = + AggregateAnalyzer.analyze(aggregate, project, List.of("checked_sum"), helper); + + SumAggregationBuilder builder = + assertInstanceOf(SumAggregationBuilder.class, result.getLeft().getFirst()); + assertEquals("a", builder.field()); + NoBucketAggregationParser parser = + assertInstanceOf(NoBucketAggregationParser.class, result.getRight()); + assertInstanceOf( + CheckedLongSumParser.class, + parser.getMetricsParser().getMetricParserMap().get("checked_sum")); + } + + @Test + void analyze_checkedLongSumExpressionUsesNativeScriptedSum() + throws ExpressionNotAnalyzableException { + buildAggregation("checked_sum") + .withAggCall( + b -> + b.aggregateCall( + PPLBuiltinOperators.CHECKED_LONG_SUM, + b.call(SqlStdOperatorTable.PLUS, b.field("a"), b.literal(1))) + .as("checked_sum")) + .expectDslTemplate("[{\"checked_sum\":{\"sum\":{\"script\":*}}}]") + .expectResponseParser( + new MetricParserHelper(List.of(new CheckedLongSumParser("checked_sum")))) + .verify(); + } + + @Test + void analyze_bigintAvgUsesNativeField() throws ExpressionNotAnalyzableException { + buildAggregation("avg") + .withAggCall(b -> b.aggregateCall(PPLBuiltinOperators.BIGINT_AVG, b.field("a")).as("avg")) + .expectDslQuery("[{\"avg\":{\"avg\":{\"field\":\"a\"}}}]") + .expectResponseParser(new MetricParserHelper(List.of(new SingleValueParser("avg")))) + .verify(); + } + @Test void analyze_aggCall_extended() throws ExpressionNotAnalyzableException { AggregateCall varSampCall = diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/agg/CheckedLongSumParserTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/agg/CheckedLongSumParserTest.java new file mode 100644 index 00000000000..8a5c940ada6 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/agg/CheckedLongSumParserTest.java @@ -0,0 +1,70 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.response.agg; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.opensearch.search.aggregations.metrics.NumericMetricsAggregation; + +class CheckedLongSumParserTest { + + private final CheckedLongSumParser parser = new CheckedLongSumParser("sum"); + + @Test + void narrowsNativeSumToLong() { + assertEquals(42L, value(parser.parse(aggregation(42d)))); + } + + @Test + void preservesNativeDoublePrecisionBehavior() { + double rounded = (double) ((1L << 62) + 1L); + assertEquals(1L << 62, value(parser.parse(aggregation(rounded)))); + } + + @Test + void saturatesAmbiguousPositiveBoundary() { + assertEquals(Long.MAX_VALUE, value(parser.parse(aggregation((double) Long.MAX_VALUE)))); + } + + @Test + void rejectsValueClearlyOutsideLongRange() { + assertThrows(ArithmeticException.class, () -> parser.parse(aggregation(Math.nextUp(0x1p63)))); + assertThrows( + ArithmeticException.class, () -> parser.parse(aggregation(Math.nextDown(-0x1p63)))); + } + + @Test + void rejectsInfiniteValue() { + assertThrows( + ArithmeticException.class, () -> parser.parse(aggregation(Double.POSITIVE_INFINITY))); + assertThrows( + ArithmeticException.class, () -> parser.parse(aggregation(Double.NEGATIVE_INFINITY))); + } + + @Test + void convertsNanToNull() { + assertNull(value(parser.parse(aggregation(Double.NaN)))); + } + + private static NumericMetricsAggregation.SingleValue aggregation(double value) { + NumericMetricsAggregation.SingleValue aggregation = + mock(NumericMetricsAggregation.SingleValue.class); + when(aggregation.getName()).thenReturn("sum"); + when(aggregation.value()).thenReturn(value); + return aggregation; + } + + private static Object value(List> rows) { + return rows.getFirst().get("sum"); + } +} diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/calcite/OpenSearchSparkSqlDialect.java b/ppl/src/main/java/org/opensearch/sql/ppl/calcite/OpenSearchSparkSqlDialect.java index 2d044da58e6..c25e4655bf8 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/calcite/OpenSearchSparkSqlDialect.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/calcite/OpenSearchSparkSqlDialect.java @@ -25,7 +25,8 @@ public class OpenSearchSparkSqlDialect extends SparkSqlDialect { ImmutableMap.of( "ARG_MIN", "MIN_BY", "ARG_MAX", "MAX_BY", - "SAFE_CAST", "TRY_CAST"); + "SAFE_CAST", "TRY_CAST", + "CHECKED_LONG_SUM", "SUM"); private static final Map CALL_SEPARATOR = ImmutableMap.of("SAFE_CAST", "AS"); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java index 415acc5558b..850cefd7308 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java @@ -482,7 +482,7 @@ public void testJoinWithRelationSubquery() { RelNode root = getRelNode(ppl); String expectedLogical = "LogicalProject(sum=[$1], JOB=[$0])\n" - + " LogicalAggregate(group=[{0}], sum=[SUM($1)])\n" + + " LogicalAggregate(group=[{0}], sum=[CHECKED_LONG_SUM($1)])\n" + " LogicalProject(JOB=[$2], MGR=[$3])\n" + " LogicalJoin(condition=[=($7, $8)], joinType=[inner])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimewrapTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimewrapTest.java index 66027839f8e..63cc0572453 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimewrapTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimewrapTest.java @@ -86,7 +86,7 @@ public void testTimewrapDayProducesUnpivotedPlan() { + ":BIGINT NOT NULL) OVER (), CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL), 86400), 1)])\n" + " LogicalSort(sort0=[$0], dir0=[ASC])\n" + " LogicalProject(@timestamp=[$0], sum(value)=[$1])\n" - + " LogicalAggregate(group=[{1}], sum(value)=[SUM($0)])\n" + + " LogicalAggregate(group=[{1}], sum(value)=[CHECKED_LONG_SUM($0)])\n" + " LogicalProject(value=[$1], @timestamp0=[SPAN($0, 6, 'h')])\n" + " LogicalFilter(condition=[AND(>=($0, TIMESTAMP('2024-07-01" + " 00:00:00':VARCHAR)), <=($0, TIMESTAMP('2024-07-03 18:00:00':VARCHAR)), IS NOT" @@ -109,7 +109,8 @@ public void testTimewrapDaySparkSql() { + " `__base_offset__`, ((MAX(CAST(UNIX_TIMESTAMP(`@timestamp`) AS BIGINT)) OVER (RANGE" + " BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) -" + " CAST(UNIX_TIMESTAMP(`@timestamp`) AS BIGINT)) / 86400 + 1 `__period__`\n" - + "FROM (SELECT SPAN(`@timestamp`, 6, 'h') `@timestamp`, SUM(`value`) `sum(value)`\n" + + "FROM (SELECT SPAN(`@timestamp`, 6, 'h') `@timestamp`," + + " SUM(`value`) `sum(value)`\n" + "FROM `scott`.`events`\n" + "WHERE `@timestamp` >= TIMESTAMP('2024-07-01 00:00:00') AND `@timestamp` <=" + " TIMESTAMP('2024-07-03 18:00:00') AND `value` IS NOT NULL\n"