Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions api/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,13 @@ plugins {
dependencies {
api project(':ppl')
api project(':sql')
api group: 'org.apache.calcite', name: 'calcite-babel', version: '1.41.0'
api group: 'org.apache.calcite', name: 'calcite-babel', version: "${calcite_version}"

testImplementation testFixtures(project(':api'))
testImplementation group: 'junit', name: 'junit', version: '4.13.2'
testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: "${hamcrest_version}"
testImplementation group: 'org.mockito', name: 'mockito-core', version: "${mockito_version}"
testImplementation group: 'org.apache.calcite', name: 'calcite-testkit', version: '1.41.0'
testImplementation group: 'org.apache.calcite', name: 'calcite-testkit', version: "${calcite_version}"

testFixturesApi group: 'junit', name: 'junit', version: '4.13.2'
testFixturesApi group: 'org.hamcrest', name: 'hamcrest', version: "${hamcrest_version}"
Expand Down
4 changes: 4 additions & 0 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ buildscript {
// TODO: Migrate following to Gradle version catalog || Read from OpenSearch BOM in the future.
// See: https://github.com/opensearch-project/sql/issues/3257
aws_java_sdk_version = "1.12.651"
calcite_version = "1.42.0"
guava_version = "33.3.0-jre"
resilience4j_version = "1.5.0"
hamcrest_version = "2.1"
Expand Down Expand Up @@ -162,6 +163,9 @@ allprojects {
resolutionStrategy.force 'org.apache.commons:commons-text:1.11.0'
resolutionStrategy.force 'commons-io:commons-io:2.15.0'
resolutionStrategy.force 'org.yaml:snakeyaml:2.2'
// Align Calcite 1.42.0's bumped transitive deps under strict conflict resolution
resolutionStrategy.force 'com.jayway.jsonpath:json-path:2.10.0'
resolutionStrategy.force 'org.jooq:joou-java-6:0.9.5'
resolutionStrategy.dependencySubstitution {
substitute module('commons-lang:commons-lang') using module('org.apache.commons:commons-lang3:3.18.0') because 'CVE-2025-48924: commons-lang 2.x vulnerable to StackOverflowError'
}
Expand Down
6 changes: 3 additions & 3 deletions core/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,10 @@ dependencies {
api group: 'com.google.code.gson', name: 'gson', version: '2.8.9'
api group: 'com.tdunning', name: 't-digest', version: '3.3'
api "net.minidev:json-smart:${versions.json_smart}"
api('org.apache.calcite:calcite-core:1.41.0') {
api("org.apache.calcite:calcite-core:${calcite_version}") {
exclude group: 'net.minidev', module: 'json-smart'
}
api 'org.apache.calcite:calcite-linq4j:1.41.0'
api "org.apache.calcite:calcite-linq4j:${calcite_version}"
api project(':common')
compileOnly 'org.opensearch.sandbox:analytics-api:3.8.0-SNAPSHOT'
// Needed because analytics-api's QueryPlanExecutor signature uses
Expand All @@ -72,7 +72,7 @@ dependencies {
testImplementation 'org.opensearch.sandbox:analytics-api:3.8.0-SNAPSHOT'
testImplementation group: 'org.opensearch', name: 'opensearch-core', version: "${opensearch_version}"
implementation "com.github.seancfoley:ipaddress:5.4.2"
implementation "com.jayway.jsonpath:json-path:2.9.0"
implementation "com.jayway.jsonpath:json-path:2.10.0"

annotationProcessor('org.immutables:value:2.8.8')
compileOnly 'org.immutables:value-annotations:2.8.8'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,10 @@ public class OpenSearchTypeSystem extends RelDataTypeSystemImpl {

private OpenSearchTypeSystem() {}

@Override
public int getMaxNumericPrecision() {
return MAX_PRECISION;
}

@Override
public int getMaxPrecision(SqlTypeName typeName) {
return switch (typeName) {
case DECIMAL -> MAX_PRECISION;
case TIME,
TIME_WITH_LOCAL_TIME_ZONE,
TIME_TZ,
Expand All @@ -47,8 +43,11 @@ public int getMaxPrecision(SqlTypeName typeName) {
}

@Override
public int getMaxNumericScale() {
return MAX_SCALE;
public int getMaxScale(SqlTypeName typeName) {
return switch (typeName) {
case DECIMAL -> MAX_SCALE;
default -> super.getMaxScale(typeName);
};
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
import org.apache.calcite.runtime.SqlFunctions;
import org.apache.calcite.sql.*;
import org.apache.calcite.sql.type.SqlReturnTypeInference;
import org.apache.calcite.sql.type.SqlTypeUtil;
import org.opensearch.sql.calcite.utils.MathUtils;
import org.opensearch.sql.calcite.utils.PPLOperandTypes;
import org.opensearch.sql.calcite.utils.PPLReturnTypes;
import org.opensearch.sql.expression.function.ImplementorUDF;
Expand Down Expand Up @@ -67,6 +69,11 @@ public Expression implement(
RexToLixTranslator translator, RexCall call, List<Expression> translatedOperands) {
Expression fieldValue = translatedOperands.get(0);
Expression format = translatedOperands.get(1);
// Box numeric operands and pass them as Number so method resolution succeeds whether the
// upstream expression yields a primitive or a boxed value (e.g. a nullable long/double).
if (SqlTypeUtil.isNumeric(call.getOperands().get(0).getType())) {
fieldValue = Expressions.convert_(Expressions.box(fieldValue), Number.class);
}
return Expressions.call(ToStringFunction.class, "toString", fieldValue, format);
}
}
Expand Down Expand Up @@ -97,13 +104,16 @@ public static String toString(BigDecimal num, String format) {
}

@Strict
public static String toString(double num, String format) {
return toString(BigDecimal.valueOf(num), format);
}

@Strict
public static String toString(int num, String format) {
return toString(BigDecimal.valueOf(num), format);
public static String toString(Number num, String format) {
BigDecimal bd;
if (num instanceof BigDecimal decimal) {
bd = decimal;
} else if (MathUtils.isIntegral(num)) {
bd = BigDecimal.valueOf(num.longValue());
} else {
bd = BigDecimal.valueOf(num.doubleValue());
}
return toString(bd, format);
}

@Strict
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import static org.opensearch.sql.expression.datetime.DateTimeFunctions.exprFromUnixTime;
import static org.opensearch.sql.expression.datetime.DateTimeFunctions.exprFromUnixTimeFormat;

import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.List;
import org.apache.calcite.adapter.enumerable.NotNullImplementor;
import org.apache.calcite.adapter.enumerable.NullPolicy;
Expand Down Expand Up @@ -61,18 +61,18 @@ public static class FromUnixTimeImplementor implements NotNullImplementor {
@Override
public Expression implement(
RexToLixTranslator translator, RexCall call, List<Expression> translatedOperands) {
return Expressions.call(FromUnixTimeImplementor.class, "fromUnixTime", translatedOperands);
// Box the numeric operand and pass it as Number so method resolution succeeds whether the
// upstream expression yields a primitive or a boxed value (e.g. a nullable double).
List<Expression> operands = new ArrayList<>(translatedOperands);
operands.set(0, Expressions.convert_(Expressions.box(operands.getFirst()), Number.class));
return Expressions.call(FromUnixTimeImplementor.class, "fromUnixTime", operands);
}

public static String fromUnixTime(double unixTime) {
public static String fromUnixTime(Number unixTime) {
return (String) exprFromUnixTime(new ExprDoubleValue(unixTime)).valueForCalcite();
}

public static String fromUnixTime(BigDecimal unixTime) {
return (String) exprFromUnixTime(new ExprDoubleValue(unixTime)).valueForCalcite();
}

public static String fromUnixTime(double unixTime, String format) {
public static String fromUnixTime(Number unixTime, String format) {
return (String)
exprFromUnixTimeFormat(new ExprDoubleValue(unixTime), new ExprStringValue(format))
.valueForCalcite();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ calcite:
EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], gender=[$t0], age=[$t3], avg(balance)=[$t2])
CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->AND(IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(balance)=AVG($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])
EnumerableSort(sort0=[$0], dir0=[ASC])
EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2])
EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], _row_number_chart_=[$t2])
EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])])
EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)])
EnumerableCalc(expr#0..1=[{inputs}], expr#2=[SAFE_CAST($t0)], expr#3=[IS NOT NULL($t2)], age=[$t2], avg(balance)=[$t1], $condition=[$t3])
CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->AND(IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(balance)=AVG($1)), PROJECT->[age, avg(balance)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])
CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->AND(IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(balance)=AVG($1)), PROJECT->[age, avg(balance)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,10 @@ calcite:
EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5])
CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])
EnumerableSort(sort0=[$0], dir0=[ASC])
EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2])
EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], _row_number_chart_=[$t2])
EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])])
EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)])
EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], expr#11=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t10], $condition=[$t11])
EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)])
EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5])
CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])
CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ calcite:
EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left])
CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->AND(IS NOT NULL($2), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},max(value)=MAX($1)), PROJECT->[timestamp0, category, max(value)], SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"timestamp","boost":1.0}},{"exists":{"field":"value","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"timestamp0":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])
EnumerableSort(sort0=[$0], dir0=[ASC])
EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], $1=[$t2])
EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], _row_number_chart_=[$t2])
EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])])
EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)])
CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->AND(IS NOT NULL($2), IS NOT NULL($1)), FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},max(value)=MAX($1)), PROJECT->[category, max(value)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"bool":{"must":[{"exists":{"field":"timestamp","boost":1.0}},{"exists":{"field":"value","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"category","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"timestamp0":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])
Loading
Loading