diff --git a/backend.Dockerfile b/backend.Dockerfile index 25a6f69eae..35272b989a 100644 --- a/backend.Dockerfile +++ b/backend.Dockerfile @@ -1,5 +1,5 @@ # Version Extractor -FROM alpine/git:2.52.0 AS version-extractor +FROM alpine/git:v2.52.0 AS version-extractor WORKDIR /app COPY .git . diff --git a/backend/pom.xml b/backend/pom.xml index b31ca86ca3..5d5b69d9f8 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -373,6 +373,11 @@ testcontainers-postgresql test + + org.testcontainers + testcontainers-clickhouse + test + org.testcontainers testcontainers-solr @@ -383,6 +388,12 @@ ngdbc 2.17.10 + + com.clickhouse + clickhouse-jdbc + 0.9.8 + all + io.prometheus simpleclient_dropwizard diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/ArrayConceptQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ArrayConceptQuery.java index 05884083f2..ed1cac3541 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/ArrayConceptQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ArrayConceptQuery.java @@ -26,6 +26,7 @@ import lombok.NoArgsConstructor; import lombok.NonNull; import lombok.Setter; +import lombok.ToString; import lombok.extern.slf4j.Slf4j; /** @@ -38,6 +39,7 @@ @CPSType(id = "ARRAY_CONCEPT_QUERY", base = QueryDescription.class) @Slf4j @NoArgsConstructor(access = AccessLevel.PRIVATE, onConstructor_ = {@JsonCreator}) +@ToString public class ArrayConceptQuery extends Query { @NotEmpty diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/ResultHeaders.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ResultHeaders.java index 53aec081b8..3394c7e101 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/ResultHeaders.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ResultHeaders.java @@ -14,6 +14,7 @@ import com.bakdata.conquery.models.query.resultinfo.printers.common.LocalizedEnumPrinter; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import lombok.experimental.UtilityClass; @UtilityClass @@ -27,6 +28,11 @@ public static ResultInfo datesInfo() { public String userColumnName(PrintSettings printSettings) { return C10nCache.getLocalized(ResultHeadersC10n.class, printSettings.getLocale()).dates(); } + + @Override + public ResultSetProcessor.Reader createReader(ResultSetProcessor resultSetProcessor) { + return resultSetProcessor::getDateRangeList; + } }; } @@ -39,6 +45,11 @@ public static ResultInfo historyDatesInfo() { public String userColumnName(PrintSettings printSettings) { return C10nCache.getLocalized(ResultHeadersC10n.class, printSettings.getLocale()).dates(); } + + @Override + public ResultSetProcessor.Reader createReader(ResultSetProcessor resultSetProcessor) { + return resultSetProcessor::getDateRangeList; + } }; } @@ -48,6 +59,11 @@ public static ResultInfo sourceInfo() { public String userColumnName(PrintSettings printSettings) { return C10nCache.getLocalized(ResultHeadersC10n.class, printSettings.getLocale()).source(); } + + @Override + public ResultSetProcessor.Reader createReader(ResultSetProcessor resultSetProcessor) { + return resultSetProcessor::getString; + } }; } @@ -58,6 +74,11 @@ public static ResultInfo formContextInfo() { public String userColumnName(PrintSettings printSettings) { return C10nCache.getLocalized(ResultHeadersC10n.class, printSettings.getLocale()).index(); } + + @Override + public ResultSetProcessor.Reader createReader(ResultSetProcessor resultSetProcessor) { + return resultSetProcessor::getInteger; + } }; } @@ -68,6 +89,11 @@ public static ResultInfo formDateRangeInfo() { public String userColumnName(PrintSettings printSettings) { return C10nCache.getLocalized(ResultHeadersC10n.class, printSettings.getLocale()).dateRange(); } + + @Override + public ResultSetProcessor.Reader createReader(ResultSetProcessor resultSetProcessor) { + return resultSetProcessor::getDateRange; + } }; } @@ -83,6 +109,11 @@ public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printS public String userColumnName(PrintSettings printSettings) { return C10nCache.getLocalized(ResultHeadersC10n.class, printSettings.getLocale()).resolution(); } + + @Override + public ResultSetProcessor.Reader createReader(ResultSetProcessor resultSetProcessor) { + return resultSetProcessor::getString; + } }; } @@ -93,6 +124,11 @@ public static ResultInfo formEventDateInfo() { public String userColumnName(PrintSettings printSettings) { return C10nCache.getLocalized(ResultHeadersC10n.class, printSettings.getLocale()).eventDate(); } + + @Override + public ResultSetProcessor.Reader createReader(ResultSetProcessor resultSetProcessor) { + return resultSetProcessor::getDate; + } }; } @@ -108,6 +144,11 @@ public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printS public String userColumnName(PrintSettings printSettings) { return C10nCache.getLocalized(ResultHeadersC10n.class, printSettings.getLocale()).observationScope(); } + + @Override + public ResultSetProcessor.Reader createReader(ResultSetProcessor resultSetProcessor) { + return resultSetProcessor::getString; + } }; } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java index 449dd899c7..beb1fe2ae0 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java @@ -68,9 +68,6 @@ public SqlFilters convertToSqlFilter(SqlIdColumns ids, ConversionContext context FilterContext filterContext = FilterContext.forConceptConversion(ids, readValue(), context, tables); final Filter resolve = (Filter) filter.resolve(); SqlFilters sqlFilters = resolve.createConverter().convertToSqlFilter(resolve, filterContext); - if (context.isNegation()) { - return new SqlFilters(sqlFilters.getSelects(), sqlFilters.getWhereClauses().negated()); - } return sqlFilters; } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java index acd27bf8d1..cf9577ff15 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java @@ -31,6 +31,7 @@ import com.bakdata.conquery.models.query.resultinfo.FixedLabelResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.types.ResultType; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.bakdata.conquery.util.QueryUtils; import com.fasterxml.jackson.annotation.JsonView; import com.google.common.base.Preconditions; @@ -129,6 +130,11 @@ public String userColumnName(PrintSettings printSettings) { public String defaultColumnName(PrintSettings printSettings) { return defaultLabel(printSettings.getLocale()); } + + @Override + public ResultSetProcessor.Reader createReader(ResultSetProcessor resultSetProcessor) { + return resultSetProcessor::getBoolean; + } }); } return resultInfos; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java index f164832c1d..190b021d50 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java @@ -10,6 +10,7 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import com.bakdata.conquery.models.query.DateAggregationMode; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.Visitable; @@ -50,12 +51,12 @@ public void collectRequiredQueries(Set requiredQueries) { public void resolve(QueryResolveContext context) { Preconditions.checkNotNull(context.getDateAggregationMode()); - dateAction = determineDateAction(context); + dateAction = determineDateAction(context.getDateAggregationMode()); child.resolve(context); } - private DateAggregationAction determineDateAction(QueryResolveContext context) { - return switch (context.getDateAggregationMode()) { + public static DateAggregationAction determineDateAction(DateAggregationMode dateAggregationMode) { + return switch (dateAggregationMode) { case MERGE, NONE, INTERSECT -> DateAggregationAction.BLOCK; case LOGICAL -> DateAggregationAction.NEGATE; }; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java index eea5f58c7f..cb019ee307 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java @@ -31,6 +31,7 @@ import com.bakdata.conquery.models.query.resultinfo.FixedLabelResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.types.ResultType; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.bakdata.conquery.util.QueryUtils; import com.fasterxml.jackson.annotation.JsonView; import com.google.common.base.Preconditions; @@ -134,6 +135,11 @@ public String userColumnName(PrintSettings printSettings) { public String defaultColumnName(PrintSettings printSettings) { return defaultLabel(printSettings.getLocale()); } + + @Override + public ResultSetProcessor.Reader createReader(ResultSetProcessor resultSetProcessor) { + return resultSetProcessor::getBoolean; + } }); } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQTemporal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQTemporal.java index 710ac89ba9..2b1c725aee 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQTemporal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQTemporal.java @@ -32,6 +32,7 @@ import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.printers.common.ListResultInfo; import com.bakdata.conquery.models.types.ResultType; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonView; import lombok.Data; import lombok.Setter; @@ -188,6 +189,11 @@ public List getResultInfos() { public String userColumnName(PrintSettings printSettings) { return C10N.get(ResultHeadersC10n.class, printSettings.getLocale()).temporalCompareLabel(compare.userLabel(printSettings.getLocale())); } + + @Override + public ResultSetProcessor.Reader createReader(ResultSetProcessor resultSetProcessor) { + return resultSetProcessor::getDateRangeList; + } }); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java index f7d7a9422b..4da2d4f7d8 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java @@ -36,8 +36,9 @@ public void toCSV(List idHeaders, List infos, Stream infos, Stream results, PrintSettings printSettings, - PrinterFactory printerFactory) { + private void createCSVBody( + PrintSettings cfg, List infos, Stream results, PrintSettings printSettings, + PrinterFactory printerFactory) { final Printer[] printers = infos.stream().map(info -> info.createPrinter(printerFactory, printSettings)).toArray(Printer[]::new); results.map(result -> Pair.of(cfg.getIdMapper().map(result), result)) @@ -52,20 +53,19 @@ private void createCSVBody(PrintSettings cfg, List infos, Stream resolveIds(String[][] values, List rowIndex = field(name(ROW_INDEX), Integer.class); Field externalPrimaryColumn = field(name(SharedAliases.PRIMARY_COLUMN.getAlias()), String.class); Field innerPrimaryColumn = field(name(idColumns.findPrimaryIdColumn().getField()), String.class); - Field isResolved = when(innerPrimaryColumn.isNotNull(), val(true)) - .otherwise(false) - .as(IS_RESOLVED_ALIAS); + // Would prefer this to be `is not null`, but hana does not support that for fields + Field isResolved = case_().when(innerPrimaryColumn.isNull(), inline(false)).otherwise(inline(true)).as(IS_RESOLVED_ALIAS); Table allIdsTable = table(name(idColumns.getTable())); + SelectConditionStep> resolveIdsQuery = context.with(unresolvedCte) .select(rowIndex, externalPrimaryColumn, isResolved) @@ -187,8 +181,8 @@ private CommonTableExpression createUnresolvedCte(String[][] values, List rowIndex = val(i).as(ROW_INDEX); - Field externalPrimaryColumn = val(resolvedId).as(SharedAliases.PRIMARY_COLUMN.getAlias()); + Field rowIndex = inline(i).as(ROW_INDEX); + Field externalPrimaryColumn = inline(resolvedId).as(SharedAliases.PRIMARY_COLUMN.getAlias()); Select> externalIdSelect = context.select(rowIndex, externalPrimaryColumn) // some dialects can't just select static values without FROM clause .from(dialect.getFunctionProvider().getNoOpTable()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java b/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java index c1effedb58..0bb1507f68 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java @@ -1,8 +1,9 @@ package com.bakdata.conquery.models.config; import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; -import com.bakdata.conquery.sql.conversion.dialect.HanaDialectBundle; -import com.bakdata.conquery.sql.conversion.dialect.PostgreDialectBundle; +import com.bakdata.conquery.sql.conversion.dialect.clickhouse.ClickhouseDialectBundle; +import com.bakdata.conquery.sql.conversion.dialect.hana.HanaDialectBundle; +import com.bakdata.conquery.sql.conversion.dialect.pg.PostgreDialectBundle; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -15,13 +16,8 @@ @Getter public enum Dialect { - /** - * Dialect for PostgreSQL database - */ POSTGRESQL(new PostgreDialectBundle()), - /** - * Dialect for SAP HANA database - */ + CLICKHOUSE(new ClickhouseDialectBundle()), HANA(new HanaDialectBundle()); private final DialectBundle dialectBundle; diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java index 59cfaf3022..cf94d9953e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java @@ -78,6 +78,10 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuil return Collections.emptyList(); } + if(singleExecution.getResultInfos() == null){ + return Collections.emptyList(); + } + // Save id column count to later check if xlsx dimensions are feasible idColumnsCount = exec.getConfig().getIdColumns().getIdResultInfos().size(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java index c757481494..28e0e7beb5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java @@ -17,6 +17,7 @@ import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.base.Functions; import com.google.common.collect.MoreCollectors; @@ -139,6 +140,11 @@ public String userColumnName(PrintSettings printSettings) { labels.size() == 1 ? labels.values().stream().collect(MoreCollectors.onlyElement()) : col.getField() ), col.getField()); } + + @Override + public ResultSetProcessor.Reader createReader(ResultSetProcessor resultSetProcessor) { + return resultSetProcessor::getString; + } }; }).collect(Collectors.toUnmodifiableList()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DurationSumFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DurationSumFilter.java index 4e2b0c9350..47d666535b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DurationSumFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DurationSumFilter.java @@ -1,11 +1,9 @@ package com.bakdata.conquery.models.datasets.concepts.filters.specific; import java.util.ArrayList; -import java.util.EnumSet; import java.util.List; import javax.annotation.Nullable; import jakarta.validation.Valid; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.frontend.FrontendFilterConfiguration; import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; @@ -13,6 +11,7 @@ import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.datasets.concepts.DaterangeSelectOrFilter; import com.bakdata.conquery.models.datasets.concepts.filters.Filter; import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.exceptions.ConceptConfigurationException; @@ -21,8 +20,10 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.ColumnAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.DistinctValuesWrapperAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.DurationSumAggregator; +import com.bakdata.conquery.models.query.queryplan.aggregators.specific.TwoColumnDurationSumAggregator; import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; -import com.fasterxml.jackson.annotation.JsonAlias; +import com.bakdata.conquery.sql.conversion.model.aggregator.DurationSumSqlAggregator; +import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; import lombok.Data; @@ -31,31 +32,39 @@ @Data @Slf4j @CPSType(id = "DURATION_SUM", base = Filter.class) -public class DurationSumFilter extends Filter { - - @Valid - @NotNull - @JsonAlias("column") - private final ColumnId dateRangeColumn; +public class DurationSumFilter extends Filter implements DaterangeSelectOrFilter { @Valid @Nullable - private final List distinctBy; + private List distinctBy; + @Nullable + private ColumnId column; + @Nullable + private ColumnId startColumn; + @Nullable + private ColumnId endColumn; @JsonIgnore - public EnumSet getAcceptedColumnTypes() { - return EnumSet.of(MajorTypeId.DATE_RANGE); + @Override + public List getRequiredColumns() { + List required = new ArrayList<>(); + + if (hasDistinct()) { + required.addAll(distinctBy); + } + if (column != null) { + required.add(column); + } + else { + required.add(startColumn); + required.add(endColumn); + } + return required; + } @Override public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig conqueryConfig) throws ConceptConfigurationException { - MajorTypeId type = getDateRangeColumn().resolve().getType(); - if (type != MajorTypeId.DATE_RANGE) { - throw new ConceptConfigurationException(getConnector(), "DURATION_SUM filter is incompatible with columns of type " - + type - ); - } - f.setType(FrontendFilterType.Fields.INTEGER_RANGE); f.setMin(0); } @@ -67,7 +76,8 @@ private boolean hasDistinct() { @Override public FilterNode createFilterNode(Range.LongRange value) { - ColumnAggregator aggregator = new DurationSumAggregator(getDateRangeColumn().resolve()); + ColumnAggregator aggregator = getColumn() != null ? new DurationSumAggregator(getColumn().resolve()) + : new TwoColumnDurationSumAggregator(startColumn.resolve(), endColumn.resolve()); if (hasDistinct()) { aggregator = new DistinctValuesWrapperAggregator<>(aggregator, distinctBy.stream().map(ColumnId::resolve).toList()); @@ -77,29 +87,36 @@ public FilterNode createFilterNode(Range.LongRange value) { } @Override - public List getRequiredColumns() { - List required = new ArrayList<>(); - - if (hasDistinct()) { - required.addAll(distinctBy); - } - - required.add(dateRangeColumn); - return required; - + public FilterConverter createConverter() { + return new DurationSumSqlAggregator(); } + @JsonIgnore @ValidationMethod(message = "Columns do not match required Type.") public boolean isValidColumnType() { - final Column resolved = getDateRangeColumn().resolve(); - final boolean acceptable = getAcceptedColumnTypes().contains(resolved.getType()); + if (column != null) { + final Column resolved = getColumn().resolve(); + + if (!(resolved.getType().equals(MajorTypeId.DATE) || resolved.getType().equals(MajorTypeId.DATE_RANGE))) { + log.error("Column {} of type {} is not date compatible", resolved.getId(), resolved.getType()); + return false; + } + return true; + } + + if (!startColumn.resolve().getType().equals(MajorTypeId.DATE)) { + log.error("startColumn {} is not of type DATE", startColumn); + return false; + } - if (!acceptable) { - log.error("Column[{}] is of Type[{}]. Not one of [{}]", resolved.getId(), resolved.getType(), getAcceptedColumnTypes()); + if (!endColumn.resolve().getType().equals(MajorTypeId.DATE)) { + log.error("startColumn {} is not of type DATE", endColumn); + return false; } - return acceptable; + + return true; } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java index d726ec4282..4a1828699e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java @@ -21,6 +21,7 @@ import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonBackReference; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; @@ -121,6 +122,8 @@ public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printS return printerFactory.printerFor(getResultType(), printSettings); } + public abstract ResultSetProcessor.Reader createResultSetReader(ResultSetProcessor processor); + @JsonIgnore public abstract ResultType getResultType(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java index bdb7bbe71a..c803aaea20 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.datasets.concepts.select.concept; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.Set; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; @@ -19,6 +21,7 @@ import com.bakdata.conquery.models.types.SemanticType; import com.bakdata.conquery.sql.conversion.model.select.ConceptColumnSelectConverter; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; import lombok.Data; @@ -79,4 +82,9 @@ public SelectConverter createConverter() { //TODO bind Select to converter here return new ConceptColumnSelectConverter(); } + + @Override + public ResultSetProcessor.Reader createResultSetReader(ResultSetProcessor processor) { + return processor::getString; + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDateUnionSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDateUnionSelect.java index 85ae7d8d34..1b7d1f43a4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDateUnionSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDateUnionSelect.java @@ -1,5 +1,8 @@ package com.bakdata.conquery.models.datasets.concepts.select.concept.specific; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; import java.util.stream.Collectors; import com.bakdata.conquery.io.cps.CPSType; @@ -12,6 +15,7 @@ import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.sql.conversion.model.select.EventDateUnionSelectConverter; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; /** * Collects the event dates that are corresponding to an enclosing {@link Connector} or {@link Concept} provided in a query. @@ -48,4 +52,9 @@ public SelectConverter createConverter() { public ResultType getResultType() { return new ResultType.ListT<>(ResultType.Primitive.DATE_RANGE); } + + @Override + public ResultSetProcessor.Reader>> createResultSetReader(ResultSetProcessor processor) { + return processor::getDateRangeList; + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDurationSumSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDurationSumSelect.java index 8511f43ead..d5bb67b5c5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDurationSumSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDurationSumSelect.java @@ -1,5 +1,8 @@ package com.bakdata.conquery.models.datasets.concepts.select.concept.specific; +import java.sql.ResultSet; +import java.sql.SQLException; + import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.datasets.concepts.select.concept.UniversalSelect; @@ -8,6 +11,7 @@ import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.sql.conversion.model.select.EventDurationSumSelectConverter; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.google.common.base.Preconditions; import lombok.AccessLevel; import lombok.NoArgsConstructor; @@ -35,6 +39,11 @@ public boolean isEventDateSelect() { return true; } + @Override + public ResultSetProcessor.Reader createResultSetReader(ResultSetProcessor processor) { + return processor::getInteger; + } + @Override public SelectConverter createConverter() { return new EventDurationSumSelectConverter(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/ExistsSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/ExistsSelect.java index 7e66cf24a9..f02a41fd63 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/ExistsSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/ExistsSelect.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.datasets.concepts.select.concept.specific; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.Set; import java.util.stream.Collectors; @@ -12,6 +14,7 @@ import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.sql.conversion.model.select.ExistsSelectConverter; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; @@ -27,6 +30,11 @@ public ExistsAggregator createAggregator() { return new ExistsAggregator(getRequiredTables()); } + @Override + public ResultSetProcessor.Reader createResultSetReader(ResultSetProcessor processor) { + return processor::getBoolean; + } + @Override public SelectConverter createConverter() { return new ExistsSelectConverter(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/QuarterSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/QuarterSelect.java index 26269c27e5..b46624ef86 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/QuarterSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/QuarterSelect.java @@ -1,5 +1,8 @@ package com.bakdata.conquery.models.datasets.concepts.select.concept.specific; +import java.sql.ResultSet; +import java.sql.SQLException; + import com.bakdata.conquery.apiv1.query.TemporalSamplerFactory; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.select.Select; @@ -7,6 +10,7 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.QuarterAggregator; import com.bakdata.conquery.models.types.ResultType; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonCreator; import jakarta.validation.constraints.NotNull; import lombok.Data; @@ -32,4 +36,9 @@ public Aggregator createAggregator() { public ResultType getResultType() { return ResultType.Primitive.STRING; } + + @Override + public ResultSetProcessor.Reader createResultSetReader(ResultSetProcessor processor) { + return processor::getString; + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java index fa7fa3061f..a014283044 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java @@ -19,6 +19,7 @@ import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.sql.conversion.model.select.DistinctSelectConverter; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonCreator; @CPSType(id = "DISTINCT", base = Select.class) @@ -31,7 +32,7 @@ public DistinctSelect(ColumnId column, InternToExternMapperId mapping, Range.Int @Override public Aggregator createAggregator() { - return new AllValuesAggregator<>(getColumn().resolve(), getSubstringRange()); + return new AllValuesAggregator(getColumn().resolve(), getSubstringRange()); } @Override @@ -50,14 +51,20 @@ public Printer createPrinter(PrinterFactory printerFactory, PrintSettings pri } @Override - public ResultType getResultType() { - if (getMapping() == null) { - return new ResultType.ListT<>(ResultType.resolveResultType(getColumn().resolve().getType())); + public ResultSetProcessor.Reader createResultSetReader(ResultSetProcessor processor) { + if (getMapping() != null) { + return processor::getStringList; } - return new ResultType.ListT<>(ResultType.Primitive.STRING); + return ResultSetProcessor.readerForType(getResultType(), processor); } + @Override + public ResultType getResultType() { + return new ResultType.ListT<>(ResultType.resolveResultType(getColumn().resolve().getType())); + } + + /** * Ensures that mapped values are still distinct. */ diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/FirstValueSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/FirstValueSelect.java index cf74afa985..aa43ede5ae 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/FirstValueSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/FirstValueSelect.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector; +import java.sql.ResultSet; + import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.datasets.concepts.select.Select; @@ -10,6 +12,7 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.FirstValueAggregator; import com.bakdata.conquery.sql.conversion.model.select.FirstValueSelectConverter; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonCreator; @CPSType(id = "FIRST", base = Select.class) @@ -29,6 +32,8 @@ public Aggregator createAggregator() { return new FirstValueAggregator<>(getColumn().resolve(), getSubstringRange()); } + + @Override public SelectConverter createConverter() { return new FirstValueSelectConverter(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/SingleColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/SingleColumnSelect.java index cd8109782a..237e2358da 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/SingleColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/SingleColumnSelect.java @@ -72,7 +72,7 @@ public boolean isValidColumnType() { return true; } - log.error("Column[{}] is of Type[{}]. Not one of [{}]", column, type, getAcceptedColumnTypes()); + log.error("Column[{}] is of Type[{}]. Not one of {} for {}", column, type, getAcceptedColumnTypes(), getClass().getSimpleName()); return false; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountQuartersSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountQuartersSelect.java index a03861dd07..e21b695ef7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountQuartersSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountQuartersSelect.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector.specific; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.List; import javax.annotation.Nullable; @@ -15,6 +17,7 @@ import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.sql.conversion.model.aggregator.CountQuartersSqlAggregator; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonCreator; import lombok.Getter; import lombok.NoArgsConstructor; @@ -61,6 +64,11 @@ public ResultType getResultType() { return ResultType.Primitive.INTEGER; } + @Override + public ResultSetProcessor.Reader createResultSetReader(ResultSetProcessor processor) { + return processor::getInteger; + } + @Override public SelectConverter createConverter() { return new CountQuartersSqlAggregator(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountSelect.java index 0c20e77498..6812eddb0a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountSelect.java @@ -15,6 +15,7 @@ import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.sql.conversion.model.aggregator.CountSqlAggregator; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import lombok.Data; import lombok.NoArgsConstructor; import org.jetbrains.annotations.Nullable; @@ -65,6 +66,10 @@ public SelectConverter createConverter() { return new CountSqlAggregator(); } + @Override + public ResultSetProcessor.Reader createResultSetReader(ResultSetProcessor processor) { + return processor::getInteger; + } @Override public ResultType getResultType() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateDistanceSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateDistanceSelect.java index 4747fef16c..76bb598163 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateDistanceSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateDistanceSelect.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector.specific; +import java.sql.ResultSet; +import java.sql.SQLException; import java.time.temporal.ChronoUnit; import java.util.EnumSet; import jakarta.validation.constraints.NotNull; @@ -14,6 +16,7 @@ import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.sql.conversion.model.aggregator.DateDistanceSqlAggregator; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonCreator; import lombok.Getter; import lombok.Setter; @@ -46,6 +49,11 @@ public SelectConverter createConverter() { return new DateDistanceSqlAggregator(); } + @Override + public ResultSetProcessor.Reader createResultSetReader(ResultSetProcessor processor) { + return processor::getInteger; + } + @Override public ResultType getResultType() { return ResultType.Primitive.INTEGER; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java index 127fa40e1e..43399ac6b2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector.specific; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.List; import javax.annotation.Nullable; @@ -12,6 +14,7 @@ import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.sql.conversion.model.select.DateUnionSelectConverter; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Getter; @@ -51,6 +54,11 @@ public ResultType getResultType() { return new ResultType.ListT<>(ResultType.Primitive.DATE_RANGE); } + @Override + public ResultSetProcessor.Reader>> createResultSetReader(ResultSetProcessor processor) { + return processor::getDateRangeList; + } + @Override public SelectConverter createConverter() { return new DateUnionSelectConverter(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java index f0397b6888..638e0738d3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector.specific; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import javax.annotation.Nullable; @@ -9,14 +11,18 @@ import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; +import com.bakdata.conquery.models.query.queryplan.aggregators.ColumnAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.DistinctValuesWrapperAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.DurationSumAggregator; +import com.bakdata.conquery.models.query.queryplan.aggregators.specific.TwoColumnDurationSumAggregator; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.sql.conversion.model.select.DurationSumSelectConverter; +import com.bakdata.conquery.sql.conversion.model.aggregator.DurationSumSqlAggregator; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Data; +import lombok.NoArgsConstructor; @Data @CPSType(id = "DURATION_SUM", base = Select.class) @@ -24,12 +30,12 @@ public class DurationSumSelect extends Select implements DaterangeSelectOrFilter { @Nullable - private final ColumnId column; + private ColumnId column; @Nullable - private final ColumnId startColumn, endColumn; + private ColumnId startColumn, endColumn; - private final List distinctBy; + private List distinctBy; @Override public List getRequiredColumns() { @@ -49,6 +55,11 @@ public List getRequiredColumns() { return out; } + @Override + public ResultSetProcessor.Reader createResultSetReader(ResultSetProcessor processor) { + return processor::getInteger; + } + @JsonIgnore private boolean hasDistinct() { return distinctBy != null && !distinctBy.isEmpty(); @@ -56,7 +67,8 @@ private boolean hasDistinct() { @Override public Aggregator createAggregator() { - DurationSumAggregator aggregator = new DurationSumAggregator(getColumn().resolve()); + ColumnAggregator aggregator = getColumn() != null ? new DurationSumAggregator(getColumn().resolve()) + : new TwoColumnDurationSumAggregator(startColumn.resolve(), endColumn.resolve()); if (!hasDistinct()) { return aggregator; @@ -73,6 +85,6 @@ public ResultType getResultType() { @Override public SelectConverter createConverter() { //TODO apply distinctBy (though needs to be done once other branches are merged) - return new DurationSumSelectConverter(); + return new DurationSumSqlAggregator(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/FlagSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/FlagSelect.java index 2f79121b10..2679f75364 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/FlagSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/FlagSelect.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector.specific; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -15,6 +17,7 @@ import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.sql.conversion.model.aggregator.FlagSqlAggregator; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; @@ -42,6 +45,11 @@ public List getRequiredColumns() { return flags.values().stream().toList(); } + @Override + public ResultSetProcessor.Reader> createResultSetReader(ResultSetProcessor processor) { + return processor::getStringList; + } + @Override public Aggregator createAggregator() { final Map collect = flags.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().resolve())); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java index fb2f0a8e1d..8900fa8d2f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java @@ -1,5 +1,9 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector.specific; +import static com.bakdata.conquery.models.types.ResultType.Primitive.STRING; +import static com.bakdata.conquery.models.types.ResultType.resolveResultType; +import static org.jooq.impl.DSL.*; + import java.util.Collections; import java.util.Set; import javax.annotation.Nullable; @@ -7,7 +11,9 @@ import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.io.jackson.View; +import com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider; import com.bakdata.conquery.models.common.Range; +import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.select.connector.SingleColumnSelect; import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; @@ -19,15 +25,21 @@ import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; +import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; +import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.bakdata.conquery.sql.conversion.model.select.SelectContext; +import com.bakdata.conquery.sql.conversion.model.select.SingleColumnSqlSelect; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; import lombok.Getter; +import org.jooq.Field; @Getter public abstract class MappableSingleColumnSelect extends SingleColumnSelect { /** - * If a mapping was provided the mapping changes the aggregator result before it is processed by a {@link com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider}. + * If a mapping was provided the mapping changes the aggregator result before it is processed by a {@link ResultRendererProvider}. */ @Valid @Nullable @@ -45,6 +57,31 @@ public MappableSingleColumnSelect(ColumnId column, @Nullable InternToExternMappe this.substringRange = substringRange; } + public static SingleColumnSqlSelect getSubstringSelect( + Column column, Range.IntegerRange substringRange, SelectContext selectContext, + String alias) { + + Field field = field(name(selectContext.getTables().getRootTable(), column.getName()), String.class); + + if (substringRange != null && !substringRange.isAll()) { + if (substringRange.isAtLeast()) { + field = substring(field, 1 + substringRange.getMin()); + } + else if (substringRange.isAtMost()) { + field = substring(field, 1, substringRange.getMax()); + } + else { + field = substring(field, 1 + substringRange.getMin(), substringRange.getMax() - substringRange.getMin()); + } + } + + if (alias != null) { + field = field.as(name(alias)); + } + + return new FieldWrapper<>(field, column.getName()); + } + @Override public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings) { if (mapping == null) { @@ -69,16 +106,16 @@ public SelectResultInfo getResultInfo(CQConcept cqConcept) { @Override public ResultType getResultType() { if (mapping == null) { - return ResultType.resolveResultType(getColumn().resolve().getType()); + return resolveResultType(getColumn().resolve().getType()); } InternToExternMapper resolved = mapping.resolve(); if (resolved.isAllowMultiple()) { - return new ResultType.ListT<>(ResultType.Primitive.STRING); + return new ResultType.ListT<>(STRING); } - return ResultType.Primitive.STRING; + return STRING; } public void loadMapping() { @@ -119,4 +156,14 @@ public boolean isMinPositive() { return getSubstringRange().getMin() >= 0; } + + @Override + public ResultSetProcessor.Reader createResultSetReader(ResultSetProcessor processor) { + if (mapping != null) { + return processor::getString; + } + + return ResultSetProcessor.readerForType(getResultType(), processor); + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/PrefixSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/PrefixSelect.java index 5fb858be2b..d098c9d796 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/PrefixSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/PrefixSelect.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector.specific; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.EnumSet; import com.bakdata.conquery.io.cps.CPSType; @@ -10,6 +12,7 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.PrefixTextAggregator; import com.bakdata.conquery.models.types.ResultType; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonCreator; import lombok.Getter; import lombok.Setter; @@ -37,6 +40,11 @@ public Aggregator createAggregator() { return new PrefixTextAggregator(getColumn().resolve(), prefix); } + @Override + public ResultSetProcessor.Reader createResultSetReader(ResultSetProcessor processor) { + throw new IllegalStateException("PREFIX select is not implemented for SQL mode."); + } + @Override public ResultType getResultType() { return new ResultType.ListT<>(ResultType.Primitive.STRING); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/QuartersInYearSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/QuartersInYearSelect.java index 1d3dbaf693..70f5385c5a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/QuartersInYearSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/QuartersInYearSelect.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector.specific; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.EnumSet; import com.bakdata.conquery.io.cps.CPSType; @@ -10,6 +12,7 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.QuartersInYearAggregator; import com.bakdata.conquery.models.types.ResultType; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonCreator; /** @@ -33,6 +36,11 @@ public Aggregator createAggregator() { return new QuartersInYearAggregator(getColumn().resolve()); } + @Override + public ResultSetProcessor.Reader createResultSetReader(ResultSetProcessor processor) { + return processor::getInteger; + } + @Override public ResultType getResultType() { return ResultType.Primitive.INTEGER; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/SumSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/SumSelect.java index bc50b24deb..c620a327a8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/SumSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/SumSelect.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector.specific; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; @@ -25,6 +27,7 @@ import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.sql.conversion.model.aggregator.SumSqlAggregator; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; @@ -110,6 +113,11 @@ public List getRequiredColumns() { return out; } + @Override + public ResultSetProcessor.Reader createResultSetReader(ResultSetProcessor processor) { + return processor::getInteger; + } + @Override public SelectConverter createConverter() { return new SumSqlAggregator<>(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/CBlock.java b/backend/src/main/java/com/bakdata/conquery/models/events/CBlock.java index 9e503b3b9d..981c48e622 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/CBlock.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/CBlock.java @@ -14,6 +14,7 @@ import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.datasets.concepts.Connector; +import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeCache; import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeChild; import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeConnector; @@ -95,7 +96,7 @@ public static CBlock createCBlock(ConceptTreeConnector connector, BucketId bucke final int[][] mostSpecificChildren = calculateSpecificChildrenPaths(bucket, connector, bucketManager); //TODO Object2LongMap final Map includedConcepts = calculateConceptElementPathBloomFilter(bucket, mostSpecificChildren); - final Map entitySpans = calculateEntityDateIndices(bucket); + final Map entitySpans = calculateEntityDateIndices(bucket, connector); final CBlock cBlock = new CBlock(bucketId, connector.getId(), includedConcepts, entitySpans, mostSpecificChildren); return cBlock; @@ -147,17 +148,10 @@ private static Map calculateConceptElementPathBloomFilter(Bucket b * * @implNote This is an unrolled implementation of {@link CDateRange#span(CDateRange)}. */ - private static Map calculateEntityDateIndices(Bucket bucket) { + private static Map calculateEntityDateIndices(Bucket bucket, ConceptTreeConnector connector) { final Map spans = new HashMap<>(); - final Table table = bucket.getTable().resolve(); - - - for (Column column : table.getColumns()) { - if (!column.getType().isDateCompatible()) { - continue; - } - + for (ValidityDate validityDate : connector.getValidityDates()) { for (String entity : bucket.entities()) { final int end = bucket.getEntityEnd(entity); @@ -166,14 +160,14 @@ private static Map calculateEntityDateIndices(Bucket bucket) int max = Integer.MIN_VALUE; int min = Integer.MAX_VALUE; - for (int event = bucket.getEntityStart(entity); event < end; event++) { - if (!bucket.has(event, column)) { + + final CDateRange range = validityDate.getValidityDate(event, bucket); + + if(range == null) { continue; } - final CDateRange range = bucket.getAsDateRange(event, column); - final int minValue = range.getMinValue(); min = Math.min(min, minValue); @@ -192,7 +186,6 @@ private static Map calculateEntityDateIndices(Bucket bucket) final CDateRange span = CDateRange.of(min, max); spans.compute(entity, (ignored, current) -> current == null ? span : current.span(span)); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/BooleanParser.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/BooleanParser.java index d974ed2c72..048127c22d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/BooleanParser.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/BooleanParser.java @@ -9,6 +9,7 @@ import com.bakdata.conquery.models.preproc.parser.ColumnValues; import com.bakdata.conquery.models.preproc.parser.Parser; import lombok.ToString; +import org.jetbrains.annotations.NotNull; @ToString(callSuper = true) public class BooleanParser extends Parser { @@ -19,6 +20,11 @@ public BooleanParser(ConqueryConfig config) { @Override protected Boolean parseValue(@Nonnull String value) throws ParsingException { + return parseBoolean(value); + } + + @NotNull + public static Boolean parseBoolean(@NotNull String value) { return switch (value) { case "J", "true", "1" -> true; case "N", "false", "0" -> false; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/DateAggregationMode.java b/backend/src/main/java/com/bakdata/conquery/models/query/DateAggregationMode.java index 604c0f0b6a..d54bf4242f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/DateAggregationMode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/DateAggregationMode.java @@ -24,5 +24,5 @@ public enum DateAggregationMode { * Merge or intersect the dates depending on certain nodes in the query plan (OR -> MERGE, AND -> INTERSECT, * NOT -> INVERT) */ - LOGICAL; + LOGICAL } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/TwoColumnDurationSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/TwoColumnDurationSumAggregator.java new file mode 100644 index 0000000000..a36bf62cdc --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/TwoColumnDurationSumAggregator.java @@ -0,0 +1,74 @@ +package com.bakdata.conquery.models.query.queryplan.aggregators.specific; + +import java.util.List; + +import com.bakdata.conquery.models.common.CDate; +import com.bakdata.conquery.models.common.CDateSet; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.events.Bucket; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; +import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; +import com.bakdata.conquery.models.query.queryplan.aggregators.ColumnAggregator; +import lombok.Data; +import lombok.ToString; + +/** + * Aggregator, counting the number of days present. + */ +@Data +@ToString(onlyExplicitlyIncluded = true) +public class TwoColumnDurationSumAggregator extends ColumnAggregator { + + @ToString.Include + private final Column startColumn, endColumn; + + private CDateSet set = CDateSet.createEmpty(); + private CDateSet dateRestriction; + + private int realUpperBound; + + + @Override + public void init(Entity entity, QueryExecutionContext context) { + set.clear(); + realUpperBound = context.getToday(); + } + + @Override + public void nextTable(QueryExecutionContext ctx, Table currentTable) { + dateRestriction = ctx.getDateRestriction(); + } + + @Override + public List getRequiredColumns() { + return List.of(startColumn, endColumn); + } + + @Override + public void consumeEvent(Bucket bucket, int event) { + if (!bucket.has(event, startColumn)) { + return; + } + + if (!bucket.has(event, endColumn)) { + return; + } + + final int begin = bucket.getDate(event, startColumn); + final int end = bucket.getDate(event, endColumn); + + set.maskedAdd(CDateRange.of(begin, end), dateRestriction, realUpperBound); + } + + @Override + public Long createAggregationResult() { + if (set.isEmpty() || CDate.isNegativeInfinity(set.getMinValue())) { + return null; + } + return set.countDays(); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/AllValuesAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/AllValuesAggregator.java index f6b99783e8..8aae5ba413 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/AllValuesAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/AllValuesAggregator.java @@ -3,7 +3,9 @@ import static com.bakdata.conquery.models.query.StringUtils.getSubstringFromRange; import java.util.HashSet; +import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.datasets.Column; @@ -11,19 +13,17 @@ import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; -import com.google.common.collect.ImmutableSet; +import joptsimple.internal.Strings; import lombok.ToString; /** * Aggregator gathering all unique values in a column, into a Set. - * - * @param Value type of the column. */ @ToString(callSuper = true, onlyExplicitlyIncluded = true) -public class AllValuesAggregator extends SingleColumnAggregator> { +public class AllValuesAggregator extends SingleColumnAggregator> { private final Range.IntegerRange substring; - private final Set entries = new HashSet<>(); + private final Set entries = new HashSet<>(); public AllValuesAggregator(Column column, Range.IntegerRange substring) { super(column); @@ -43,16 +43,23 @@ public void consumeEvent(Bucket bucket, int event) { if (substring != null) { String string = bucket.getString(event, getColumn()); - entries.add((VALUE) getSubstringFromRange(string, substring)); + String extract = getSubstringFromRange(string, substring); + + if (Strings.isNullOrEmpty(extract)) { + return; + } + + entries.add(extract); return; } - entries.add((VALUE) bucket.createScriptValue(event, getColumn())); + entries.add(bucket.createScriptValue(event, getColumn())); } @Override - public Set createAggregationResult() { - return entries.isEmpty() ? null : ImmutableSet.copyOf(entries); + public List createAggregationResult() { + List rendered = entries.stream().sorted().collect(Collectors.toList()); + return rendered.isEmpty() ? null : rendered; } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalQueryNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalQueryNode.java index 9659fdc01e..0c5987ca46 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalQueryNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalQueryNode.java @@ -133,7 +133,8 @@ public boolean evaluateTemporalQuery(QueryExecutionContext ctx, Entity entity) { return false; } - final CDateRange[] periods = indexSelector.sample(indexQueryPlan.getDateAggregator().createAggregationResult()); + CDateSet indexDates = indexQueryPlan.getDateAggregator().createAggregationResult(); + final CDateRange[] periods = indexSelector.sample(indexDates); final CDateRange[] indexPeriods = indexMode.convert(periods, indexSelector); final boolean[] results = new boolean[indexPeriods.length]; @@ -181,14 +182,14 @@ public boolean evaluateTemporalQuery(QueryExecutionContext ctx, Entity entity) { satisfies ); + results[current] = satisfies; + // If compare's selector is satisfied, we append current to the results and collect the aggregation results if (!satisfies) { continue; } - results[current] = true; indexDateResult.add(periods[current]); - addAggregationResults(compareContained, compareDates, compareAggregationResults); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java index 30047a847a..a55cae4aad 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java @@ -9,6 +9,7 @@ import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.query.resultinfo.printers.common.ConceptIdPrinter; import com.bakdata.conquery.models.types.ResultType; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; @@ -51,4 +52,9 @@ public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printS return printerFactory.printerFor(type, printSettings); } + @Override + public ResultSetProcessor.Reader createReader(ResultSetProcessor resultSetProcessor) { + return ResultSetProcessor.readerForType(getType(), resultSetProcessor); + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ExternalResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ExternalResultInfo.java index f5900aae5a..cac31827c6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ExternalResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ExternalResultInfo.java @@ -6,6 +6,7 @@ import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.types.ResultType; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -41,4 +42,13 @@ public String getDescription() { public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings) { return printerFactory.printerFor(type, printSettings); } + + @Override + public ResultSetProcessor.Reader createReader(ResultSetProcessor resultSetProcessor) { + if (type instanceof ResultType.ListT list && list.getElementType().equals(ResultType.Primitive.STRING)) { + return resultSetProcessor::getStringList; + } + + return resultSetProcessor::getString; + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java index 9774dbe390..e48f129540 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java @@ -11,6 +11,7 @@ import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.google.common.collect.ImmutableSet; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -63,4 +64,6 @@ public Set getSemantics() { public abstract String getDescription(); public abstract Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings); + + public abstract ResultSetProcessor.Reader createReader(ResultSetProcessor resultSetProcessor); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SecondaryIdResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SecondaryIdResultInfo.java index e38aa8a941..a41b03b6c1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SecondaryIdResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SecondaryIdResultInfo.java @@ -8,6 +8,7 @@ import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import lombok.Getter; import lombok.ToString; @@ -47,4 +48,9 @@ public String userColumnName(PrintSettings printSettings) { public String defaultColumnName(PrintSettings printSettings) { return userColumnName(printSettings); } + + @Override + public ResultSetProcessor.Reader createReader(ResultSetProcessor resultSetProcessor) { + return resultSetProcessor::getString; + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java index 1d8348764d..c40af179bf 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java @@ -9,6 +9,7 @@ import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.google.common.collect.Sets; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -39,6 +40,11 @@ public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printS return select.createPrinter(printerFactory, printSettings); } + @Override + public ResultSetProcessor.Reader createReader(ResultSetProcessor resultSetProcessor) { + return select.createResultSetReader(resultSetProcessor); + } + @Override public ResultType getType() { return select.getResultType(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateRangeStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateRangeStringPrinter.java index 8c4003ff5c..74c474d0b1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateRangeStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateRangeStringPrinter.java @@ -20,6 +20,10 @@ public DateRangeStringPrinter(PrintSettings printSettings, String negativeInf, S @Override public String apply(@NotNull List f) { + if (f.isEmpty()) { + return null; + } + Preconditions.checkArgument(f.size() == 2, "Expected a list with 2 elements, one min, one max. The list was: %s ", f); final Integer min = f.get(0); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListResultInfo.java index 11d7589f18..b138be9b08 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListResultInfo.java @@ -5,6 +5,7 @@ import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.types.ResultType; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; /** * Simple wrapper to turn results into Lists. @@ -42,4 +43,9 @@ public String getDescription() { public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings) { return printerFactory.getListPrinter(resultInfo.createPrinter(printerFactory, printSettings), printSettings); } + + @Override + public ResultSetProcessor.Reader createReader(ResultSetProcessor resultSetProcessor) { + return resultInfo.createReader(resultSetProcessor); //TODO not sure if this is correct + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java index 218a37d71f..0e62677566 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.models.query.resultinfo.printers.common; import java.util.Collection; +import java.util.Objects; import java.util.StringJoiner; import com.bakdata.conquery.models.config.LocaleConfig; @@ -24,7 +25,7 @@ public String apply(@NotNull Collection f) { continue; } - joiner.add(listFormat.escapeListElement(elementPrinter.apply(obj).toString())); + joiner.add(listFormat.escapeListElement(Objects.toString(elementPrinter.apply(obj)))); } return joiner.toString(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/Conversions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/Conversions.java index d8c64532bb..50116dea6e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/Conversions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/Conversions.java @@ -1,8 +1,8 @@ package com.bakdata.conquery.sql.conversion; import java.util.List; +import java.util.Optional; -import com.google.common.collect.MoreCollectors; import lombok.Getter; /** @@ -23,9 +23,24 @@ protected Conversions(List> converters) { } public R convert(C node, X context) { - return converters.stream() - .flatMap(converter -> converter.tryConvert(node, context).stream()) - .collect(MoreCollectors.onlyElement()); + R converted = null; + for (Converter converter : converters) { + Optional maybeConverted = converter.tryConvert(node, context); + if (maybeConverted.isPresent()) { + if (converted == null) { + converted = maybeConverted.get(); + } + else { + throw new IllegalStateException("Multiple Converters for %s".formatted(node)); + } + } + } + + if (converted == null) { + throw new IllegalStateException("No converter found for %s".formatted(node)); + } + + return converted; } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java index c3d69f1a3b..147fae21da 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java @@ -1,8 +1,12 @@ package com.bakdata.conquery.sql.conversion.cqelement; -import static com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider.SQL_UNIT_SEPARATOR; +import static org.jooq.impl.DSL.*; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; @@ -19,9 +23,11 @@ import com.google.common.base.Preconditions; import org.jooq.Field; import org.jooq.Name; +import org.jooq.Nullability; import org.jooq.Record; import org.jooq.Table; import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; public class CQExternalConverter implements NodeConverter { @@ -30,42 +36,29 @@ public class CQExternalConverter implements NodeConverter { private static final String UNDERSCORE = "_"; private static final String WHITESPACE = " "; - @Override - public Class getConversionClass() { - return CQExternal.class; - } - - @Override - public ConversionContext convert(CQExternal external, ConversionContext context) { - SqlFunctionProvider functionProvider = context.getFunctionProvider(); - QueryStep externalIdsCte = createExternalIdsCte(external, functionProvider); - ConversionContext withExternalIdCte = context.withQueryStep(externalIdsCte); - if (!external.isWithExtras()) { - return withExternalIdCte; - } - QueryStep externalExtrasCte = createExternalExtrasCte(external, functionProvider); - return withExternalIdCte.withExternalExtras(externalExtrasCte); - } - - private static QueryStep createExternalIdsCte(CQExternal external, SqlFunctionProvider functionProvider) { + private static QueryStep createExternalIdsCte(CQExternal external, SqlFunctionProvider functionProvider, ConversionContext context) { List unions = new ArrayList<>(); for (Map.Entry entry : external.getValuesResolved().entrySet()) { List rowSelects = createRowSelects(entry, functionProvider); unions.addAll(rowSelects); } - Preconditions.checkArgument(!unions.isEmpty(), "Expecting at least 1 converted resolved row when converting a CQExternal"); - return QueryStep.createUnionAllStep(unions, CQ_EXTERNAL_IDS_CTE_NAME, Collections.emptyList()); - } + Preconditions.checkArgument(!unions.isEmpty(), "Expecting at least 1 resolved row when converting a CQExternal"); + QueryStep allStep = QueryStep.createUnionAllStep(unions, CQ_EXTERNAL_IDS_CTE_NAME, Collections.emptyList(), context.isNegation()); - private QueryStep createExternalExtrasCte(CQExternal external, SqlFunctionProvider functionProvider) { - List unions = new ArrayList<>(); - for (Map.Entry entry : external.getValuesResolved().entrySet()) { - List>> extrasForId = external.getExtrasForId(entry.getKey()); - QueryStep rowSelects = createRowSelects(entry, extrasForId, functionProvider); - unions.add(rowSelects); + Optional maybeValidityDate = allStep.getSelects().getValidityDate(); + + if (maybeValidityDate.isEmpty() || !context.dateRestrictionActive()) { + return allStep; } - Preconditions.checkArgument(!unions.isEmpty(), "Expecting at least 1 converted resolved row when converting a CQExternal"); - return QueryStep.createUnionAllStep(unions, CQ_EXTERNAL_EXTRAS_CTE_NAME, Collections.emptyList()); + + return QueryStep.builder() + .predecessors(List.of(allStep)) + .conditions(List.of(functionProvider.dateRestriction(functionProvider.forCDateRange(context.getDateRestrictionRange()), maybeValidityDate.get()))) + .selects(allStep.getQualifiedSelects()) + .fromTable(table(name(allStep.getCteName()))) + .cteName(CQ_EXTERNAL_IDS_CTE_NAME + "_date_restriction") + .build(); + } /** @@ -73,38 +66,22 @@ private QueryStep createExternalExtrasCte(CQExternal external, SqlFunctionProvid * 1 row per ID is sufficient. For other dialects there can be multiple rows with the same pid -> date range from the date set. */ private static List createRowSelects(Map.Entry entry, SqlFunctionProvider functionProvider) { - SqlIdColumns ids = createIdSelect(entry); + SqlIdColumns ids = createIdSelect(entry, functionProvider); List validityDateEntries = functionProvider.forCDateSet(entry.getValue(), SharedAliases.DATES_COLUMN); return validityDateEntries.stream() .map(validityDateEntry -> createIdRowSelect(ids, validityDateEntry, functionProvider)) .toList(); } - /** - * For each entry, we need to create a SELECT statement of static values for each pid -> extras. - */ - private QueryStep createRowSelects( - Map.Entry entry, - List>> extra, - SqlFunctionProvider functionProvider - ) { - SqlIdColumns ids = createIdSelect(entry); - List extraSelects = extra.stream().map(CQExternalConverter::createExtraColumnValue).collect(Collectors.toList()); - return createExtraRowSelect(ids, extraSelects, functionProvider); - } - - private static SqlIdColumns createIdSelect(Map.Entry entry) { - Field primaryColumn = DSL.val(entry.getKey()).coerce(Object.class).as(SharedAliases.PRIMARY_COLUMN.getAlias()); + private static SqlIdColumns createIdSelect(Map.Entry entry, SqlFunctionProvider functionProvider) { + Field primaryColumn = functionProvider.externalId(entry.getKey()).as(SharedAliases.PRIMARY_COLUMN.getAlias()); return new SqlIdColumns(primaryColumn); } - private static FieldWrapper createExtraColumnValue(Map.Entry> extraEntry) { - String extraValues = extraEntry.getValue().stream() - .map(DSL::val) - .map(Field::toString) - .collect(Collectors.joining(SQL_UNIT_SEPARATOR)); - final Name alias = DSL.name(extraEntry.getKey().replace(WHITESPACE, UNDERSCORE)); - final Field withAlias = DSL.field(extraValues).as(alias); + private static FieldWrapper createExtraColumnValue(Map.Entry> extraEntry, SqlFunctionProvider functionProvider) { + Field extraValues = functionProvider.asArrayRepr(extraEntry.getValue()); + final Name alias = name(extraEntry.getKey().replace(WHITESPACE, UNDERSCORE)); + final Field withAlias = extraValues.as(alias); return new FieldWrapper<>(withAlias); } @@ -152,4 +129,48 @@ private static QueryStep wrapInQueryStep(Selects selects, SqlFunctionProvider fu .build(); } + @Override + public Class getConversionClass() { + return CQExternal.class; + } + + @Override + public ConversionContext convert(CQExternal external, ConversionContext context) { + SqlFunctionProvider functionProvider = context.getDialectBundle().getFunctionProvider(); + + QueryStep externalIdsCte = createExternalIdsCte(external, functionProvider, context); + ConversionContext withExternalIdCte = context.withQueryStep(externalIdsCte); + + if (!external.isWithExtras()) { + return withExternalIdCte; + } + + QueryStep externalExtrasCte = createExternalExtrasCte(external, functionProvider, context); + return withExternalIdCte.withExternalExtras(externalExtrasCte); + } + + private QueryStep createExternalExtrasCte(CQExternal external, SqlFunctionProvider functionProvider, ConversionContext context) { + List unions = new ArrayList<>(); + for (Map.Entry entry : external.getValuesResolved().entrySet()) { + List>> extrasForId = external.getExtrasForId(entry.getKey()); + QueryStep rowSelects = createRowSelects(entry, extrasForId, functionProvider); + unions.add(rowSelects); + } + Preconditions.checkArgument(!unions.isEmpty(), "Expecting at least 1 converted resolved row when converting a CQExternal"); + return QueryStep.createUnionAllStep(unions, CQ_EXTERNAL_EXTRAS_CTE_NAME, Collections.emptyList(), context.isNegation()); + } + + /** + * For each entry, we need to create a SELECT statement of static values for each pid -> extras. + */ + private QueryStep createRowSelects( + Map.Entry entry, + List>> extra, + SqlFunctionProvider functionProvider + ) { + SqlIdColumns ids = createIdSelect(entry, functionProvider); + List extraSelects = extra.stream().map((Map.Entry> extraEntry) -> createExtraColumnValue(extraEntry, functionProvider)).collect(Collectors.toList()); + return createExtraRowSelect(ids, extraSelects, functionProvider); + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java index d9aed3a432..2bb90a6d62 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java @@ -1,36 +1,21 @@ package com.bakdata.conquery.sql.conversion.cqelement; import com.bakdata.conquery.apiv1.query.concept.specific.CQNegation; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.conversion.model.QueryStep; public class CQNegationConverter implements NodeConverter { - @Override - public Class getConversionClass() { - return CQNegation.class; - } + @Override + public Class getConversionClass() { + return CQNegation.class; + } - @Override - public ConversionContext convert(CQNegation negationNode, ConversionContext context) { - - ConversionContext converted = context.getNodeConversions() - .convert(negationNode.getChild(), context.withNegation(true)) - .withNegation(false); - - QueryStep queryStep = converted.getLastConvertedStep(); - if (negationNode.getDateAction() != DateAggregationAction.NEGATE) { - QueryStep withBlockedValidityDate = queryStep.toBuilder() - .selects(queryStep.getSelects().blockValidityDate()) - .build(); - return context.toBuilder().queryStep(withBlockedValidityDate).build(); - } - QueryStep withInvertedValidityDate = converted.getDialectBundle() - .getDateAggregator() - .invertAggregatedIntervals(queryStep, context); - return context.toBuilder().queryStep(withInvertedValidityDate).build(); - } + @Override + public ConversionContext convert(CQNegation negationNode, ConversionContext context) { + return context.getNodeConversions() + .convert(negationNode.getChild(), context.withNegation(true)) + .withNegation(false); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java index eaa8dfaa86..f45cba7d1b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java @@ -1,14 +1,18 @@ package com.bakdata.conquery.sql.conversion.cqelement; +import static org.jooq.impl.DSL.*; + +import java.util.Optional; + import com.bakdata.conquery.apiv1.query.CQYes; import com.bakdata.conquery.models.config.ColumnConfig; import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import org.jooq.Field; import org.jooq.Record; -import org.jooq.impl.DSL; public class CQYesConverter implements NodeConverter { @@ -23,11 +27,13 @@ public Class getConversionClass() { public ConversionContext convert(CQYes cqYes, ConversionContext context) { ColumnConfig primaryColumnConfig = context.getIdColumns().findPrimaryIdColumn(); - Field primaryColumn = DSL.field(DSL.name(primaryColumnConfig.getField())); + Field primaryColumn = field(name(primaryColumnConfig.getField()), String.class); SqlIdColumns ids = new SqlIdColumns(primaryColumn); - Selects selects = Selects.builder().ids(ids).build(); - org.jooq.Table fromTable = DSL.table(DSL.name(context.getIdColumns().getTable())); + Selects selects = Selects.builder().ids(ids) + .validityDate(Optional.of(context.getFunctionProvider().emptyColumnDateRange().asValidityDateRange(ALL_IDS_CTE))) + .build(); + org.jooq.Table fromTable = table(name(context.getIdColumns().getTable())); QueryStep cqYesTep = QueryStep.builder() .cteName(ALL_IDS_CTE) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java index 685e4c5f28..a2973ec90d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java @@ -25,6 +25,7 @@ import lombok.Value; import lombok.With; +//TODO this class is less context and more state. It's also incredibly dangerous because it hides a lot of moving parts and creates indirections. @Value @With @Builder(toBuilder = true) @@ -100,7 +101,7 @@ public ConversionContext getConversionContext() { * Get the last query {@link QueryStep} that has been added to this context query steps. */ public QueryStep getLastConvertedStep() { - return this.querySteps.get(this.querySteps.size() - 1); + return this.querySteps.getLast(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java index 6a0561a531..8656cd9bf4 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java @@ -1,5 +1,9 @@ package com.bakdata.conquery.sql.conversion.cqelement.aggregation; +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.inline; + +import java.sql.Date; import java.util.List; import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; @@ -8,18 +12,21 @@ import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep; import com.bakdata.conquery.sql.conversion.dialect.IntervalPacker; import com.bakdata.conquery.sql.conversion.dialect.SqlDateAggregator; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import lombok.Data; +import org.jooq.Field; +@Data public class AnsiSqlDateAggregator implements SqlDateAggregator { private final IntervalPacker intervalPacker; + private final SqlFunctionProvider functionProvider; - public AnsiSqlDateAggregator(IntervalPacker intervalPacker) { - this.intervalPacker = intervalPacker; - } @Override public QueryStep apply( @@ -66,6 +73,18 @@ public QueryStep apply( return this.intervalPacker.aggregateAsValidityDate(intervalPackingContext); } + @Override + public ColumnDateRange getAggregatedValidityDate(DateAggregationDates dateAggregationDates, DateAggregationAction dateAggregationAction) { + //TODO(FK): i think this is only ever relevant with dateMode=Logical which i want to remove + Field rangeStart = functionProvider.least(dateAggregationDates.allStarts()); + Field rangeEnd = functionProvider.greatest(dateAggregationDates.allEnds()); + + return ColumnDateRange.of( + rangeStart.as(DateAggregationCte.RANGE_START), + rangeEnd.as(DateAggregationCte.RANGE_END) + ); + } + @Override public QueryStep invertAggregatedIntervals(QueryStep baseStep, ConversionContext conversionContext) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java index 478d8d90c0..721d99c84f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java @@ -19,22 +19,23 @@ public QueryStep convert(DateAggregationContext context, QueryStep previous) { SqlTables dateAggregationTables = context.getDateAggregationTables(); // this way all selects are already qualified, and we don't need to care for that in the respective steps - context = context.qualify(dateAggregationTables.getPredecessor(cteStep)); + String predecessor = dateAggregationTables.getPredecessor(cteStep); + context = context.qualify(predecessor); - QueryStep.QueryStepBuilder builder = this.convertStep(context); + QueryStep.QueryStepBuilder builder = this.convertStep(context, predecessor); if (cteStep != DateAggregationCteStep.NODE_NO_OVERLAP) { builder = builder.cteName(dateAggregationTables.cteName(cteStep)) .predecessors(List.of(previous)); } if (cteStep != DateAggregationCteStep.INVERT && cteStep != DateAggregationCteStep.NODE_NO_OVERLAP) { - builder = builder.fromTable(QueryStep.toTableLike(dateAggregationTables.getPredecessor(cteStep))); + builder = builder.fromTable(QueryStep.toTableLike(predecessor)); } return builder.build(); } - protected abstract QueryStep.QueryStepBuilder convertStep(DateAggregationContext context); + protected abstract QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, String predecessor); public abstract DateAggregationCteStep getCteStep(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java index 64e78fb98f..c3c7e708e6 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java @@ -3,6 +3,7 @@ import java.sql.Date; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -22,10 +23,16 @@ @AllArgsConstructor(access = AccessLevel.PRIVATE) public class DateAggregationDates { - private static final String RANGE_START = "RANGE_START"; - private static final String RANGE_END = "RANGE_END"; private final List validityDates; + public static DateAggregationDates forValidityDates(final List> validityDates) { + final List filtered = validityDates.stream() + .filter(Optional::isPresent) + .map(Optional::get) + .toList(); + return new DateAggregationDates(filtered); + } + public static DateAggregationDates forSingleStep(QueryStep queryStep) { List validityDates = queryStep.getSelects() .getValidityDate() @@ -35,11 +42,9 @@ public static DateAggregationDates forSingleStep(QueryStep queryStep) { } public static DateAggregationDates forSteps(List querySteps) { - AtomicInteger validityDateCounter = new AtomicInteger(0); - List validityDates = querySteps.stream() - .filter(queryStep -> queryStep.getSelects().getValidityDate().isPresent()) - .map(queryStep -> numerateValidityDate(queryStep, validityDateCounter)) - .toList(); + final List validityDates = querySteps.stream() + .flatMap(queryStep -> queryStep.getQualifiedSelects().getValidityDate().stream()) + .toList(); return new DateAggregationDates(validityDates); } @@ -70,17 +75,4 @@ public DateAggregationDates qualify(String qualifier) { return new DateAggregationDates(qualified); } - private static ColumnDateRange numerateValidityDate(QueryStep queryStep, AtomicInteger validityDateCounter) { - ColumnDateRange validityDate = queryStep.getQualifiedSelects().getValidityDate().get(); - - if (validityDate.isSingleColumnRange()) { - return validityDate; - } - - Field rangeStart = validityDate.getStart().as("%s_%s".formatted(RANGE_START, validityDateCounter.get())); - Field rangeEnd = validityDate.getEnd().as("%s_%s".formatted(RANGE_END, validityDateCounter.getAndIncrement())); - - return ColumnDateRange.of(rangeStart, rangeEnd); - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java index f32b391f4e..d7ccea9966 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java @@ -22,7 +22,7 @@ public IntermediateTableCte(DateAggregationCteStep cteStep) { } @Override - protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context) { + protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, String predecessor) { List intermediateTableSelects = context.getSqlAggregationAction().getIntermediateTableSelects( diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java index 5d6207c03b..abd998364e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java @@ -1,5 +1,8 @@ package com.bakdata.conquery.sql.conversion.cqelement.aggregation; +import static org.jooq.impl.DSL.*; +import static org.jooq.impl.DSL.field; + import java.sql.Date; import java.util.List; import java.util.Optional; @@ -35,7 +38,7 @@ public InvertCte(DateAggregationCteStep cteStep) { } @Override - protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context) { + protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, String predecessor) { QueryStep rowNumberStep = context.getStep(DateAggregationCteStep.ROW_NUMBER); @@ -57,14 +60,14 @@ private Selects getInvertSelects(QueryStep rowNumberStep, SqlIdColumns coalesced SqlFunctionProvider functionProvider = context.getFunctionProvider(); ColumnDateRange validityDate = rowNumberStep.getSelects().getValidityDate().get(); - Field rangeStart = DSL.coalesce( + Field rangeStart = coalesce( QualifyingUtil.qualify(validityDate.getEnd(), ROWS_LEFT_TABLE_NAME), - functionProvider.toDateField(functionProvider.getMinDateExpression()) + functionProvider.getMinDateExpression() ).as(DateAggregationCte.RANGE_START); - Field rangeEnd = DSL.coalesce( + Field rangeEnd = coalesce( QualifyingUtil.qualify(validityDate.getStart(), ROWS_RIGHT_TABLE_NAME), - functionProvider.toDateField(functionProvider.getMaxDateExpression()) + functionProvider.getMaxDateExpression() ).as(DateAggregationCte.RANGE_END); return Selects.builder() @@ -76,9 +79,9 @@ private Selects getInvertSelects(QueryStep rowNumberStep, SqlIdColumns coalesced private TableOnConditionStep selfJoinWithShiftedRows(SqlIdColumns leftIds, SqlIdColumns rightIds, QueryStep rowNumberStep) { - Field leftRowNumber = DSL.field(DSL.name(ROWS_LEFT_TABLE_NAME, RowNumberCte.ROW_NUMBER_FIELD_NAME), Integer.class) - .plus(1); - Field rightRowNumber = DSL.field(DSL.name(ROWS_RIGHT_TABLE_NAME, RowNumberCte.ROW_NUMBER_FIELD_NAME), Integer.class); + Field leftRowNumber = field(name(ROWS_LEFT_TABLE_NAME, RowNumberCte.ROW_NUMBER_FIELD_NAME), Integer.class) + .plus(1); + Field rightRowNumber = field(name(ROWS_RIGHT_TABLE_NAME, RowNumberCte.ROW_NUMBER_FIELD_NAME), Integer.class); Condition[] joinConditions = Stream.concat( Stream.of(leftRowNumber.eq(rightRowNumber)), diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeCte.java index a0afe130b8..4fd1bc36cf 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeCte.java @@ -16,7 +16,7 @@ public MergeCte(DateAggregationCteStep cteStep) { } @Override - protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context) { + protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, String predecessor) { SqlAggregationAction aggregationAction = context.getSqlAggregationAction(); List noOverlapSteps = aggregationAction.getNoOverlapSelects(context); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java index 9af31b6d89..0da7709148 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java @@ -24,7 +24,7 @@ public NodeNoOverlapCte(DateAggregationCteStep cteStep) { } @Override - protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context) { + protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, String predecessor) { // we create a no-overlap node for each query step we need to aggregate DateAggregationDates dateAggregationDates = context.getDateAggregationDates(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java index 45c8e728b2..1d6790ea23 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java @@ -22,7 +22,7 @@ public OverlapCte(DateAggregationCteStep cteStep) { } @Override - protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context) { + protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, String predecessor) { SqlFunctionProvider functionProvider = context.getFunctionProvider(); @@ -33,7 +33,7 @@ protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context) ColumnDateRange overlapValidityDate = context.getSqlAggregationAction().getOverlapValidityDate(context.getDateAggregationDates(), functionProvider); Selects overlapSelects = Selects.builder() .ids(context.getIds()) - .validityDate(Optional.of(overlapValidityDate)) + .validityDate(Optional.of(overlapValidityDate.asValidityDateRange(predecessor))) .sqlSelects(context.getCarryThroughSelects()) .build(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/PostgreSqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/PostgreSqlDateAggregator.java index 8386331d4f..43bb14e36a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/PostgreSqlDateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/PostgreSqlDateAggregator.java @@ -1,5 +1,9 @@ package com.bakdata.conquery.sql.conversion.cqelement.aggregation; +import static org.jooq.impl.DSL.*; +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.keyword; + import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @@ -14,9 +18,13 @@ import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; import lombok.Getter; import lombok.RequiredArgsConstructor; import org.jooq.Field; +import org.jooq.Keyword; import org.jooq.impl.DSL; public class PostgreSqlDateAggregator implements SqlDateAggregator { @@ -46,8 +54,10 @@ public QueryStep apply( ConversionContext conversionContext ) { String joinedStepCteName = joinedStep.getCteName(); + DateAggregationDates qualified = dateAggregationDates.qualify(joinedStepCteName); - ColumnDateRange aggregatedValidityDate = getAggregatedValidityDate(dateAggregationDates, dateAggregationAction, joinedStepCteName); + ColumnDateRange aggregatedValidityDate = getAggregatedValidityDate(qualified, dateAggregationAction) + .asValidityDateRange(joinedStepCteName); Selects dateAggregationSelects = Selects.builder() .ids(joinedStep.getQualifiedSelects().getIds()) @@ -72,21 +82,21 @@ public QueryStep invertAggregatedIntervals(QueryStep baseStep, ConversionContext return baseStep; } - Field maxDateRange = DSL.function( + Field maxDateRange = function( "daterange", Object.class, - this.functionProvider.toDateField(this.functionProvider.getMinDateExpression()), - this.functionProvider.toDateField(this.functionProvider.getMaxDateExpression()), - DSL.val("[]") + this.functionProvider.getMinDateExpression(), + this.functionProvider.getMaxDateExpression(), + inline("[]") ); // see https://www.postgresql.org/docs/current/functions-range.html // {[-infinity,infinity]} - {multirange} computes the inverse of a {multirange} - Field invertedValidityDate = DSL.field( + Field invertedValidityDate = field( "{0}::{1} - {2}", Object.class, maxDateRange, - DSL.keyword("datemultirange"), + keyword("datemultirange"), validityDate.get().getRange() ).as(PostgresDateAggregationCteStep.DATES_INVERTED.getSuffix()); @@ -98,28 +108,26 @@ public QueryStep invertAggregatedIntervals(QueryStep baseStep, ConversionContext .build(); } - private ColumnDateRange getAggregatedValidityDate(DateAggregationDates dateAggregationDates, DateAggregationAction dateAggregationAction, String joinedStepCteName) { + public ColumnDateRange getAggregatedValidityDate(DateAggregationDates dateAggregationDates, DateAggregationAction dateAggregationAction) { // see https://www.postgresql.org/docs/current/functions-range.html String aggregatingOperator = switch (dateAggregationAction) { case MERGE -> " + "; case INTERSECT -> " * "; - default -> throw new IllegalStateException("Unexpected aggregation mode: " + dateAggregationAction); + case BLOCK, NEGATE -> throw new IllegalStateException("Unexpected aggregation mode: " + dateAggregationAction); }; - String aggregatedExpression = dateAggregationDates.qualify(joinedStepCteName) - .getValidityDates().stream() + String aggregatedExpression = dateAggregationDates.getValidityDates().stream() .flatMap(validityDate -> validityDate.toFields().stream()) .map(PostgreSqlDateAggregator::createEmptyRangeForNullValues) .collect(Collectors.joining(aggregatingOperator)); - return ColumnDateRange.of(DSL.field(aggregatedExpression)) - .asValidityDateRange(joinedStepCteName); + return ColumnDateRange.of(field(aggregatedExpression)); } private static String createEmptyRangeForNullValues(Field field) { - return DSL.when(field.isNull(), DSL.field("'{}'::{0}", DSL.keyword("datemultirange"))) - .otherwise(field) + Keyword datemultirange = keyword("datemultirange"); + return coalesce(field("{0}::{1}", field, datemultirange), field("'{}'::{0}", datemultirange)) .toString(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java index 53492f0982..01c1dd5299 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java @@ -27,7 +27,7 @@ public RowNumberCte(DateAggregationCteStep cteStep) { } @Override - protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context) { + protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, String predecessor) { SqlIdColumns ids = context.getIds(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index 7d26ca0017..efad103752 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -1,15 +1,14 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; +import static org.jooq.impl.DSL.*; + import java.util.ArrayList; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.function.Predicate; -import java.util.stream.Collectors; import java.util.stream.Stream; import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; -import com.bakdata.conquery.apiv1.query.concept.filter.FilterValue; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Table; @@ -20,7 +19,6 @@ import com.bakdata.conquery.models.datasets.concepts.select.concept.ConceptColumnSelect; import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeChild; import com.bakdata.conquery.models.identifiable.ids.specific.ConceptElementId; -import com.bakdata.conquery.models.identifiable.ids.specific.ConnectorSelectId; import com.bakdata.conquery.models.identifiable.ids.specific.SelectId; import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; import com.bakdata.conquery.sql.conversion.NodeConverter; @@ -34,7 +32,6 @@ import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.filter.ConditionUtil; -import com.bakdata.conquery.sql.conversion.model.filter.ConditionWrappingWhereCondition; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.bakdata.conquery.sql.conversion.model.filter.WhereClauses; import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; @@ -49,7 +46,6 @@ import org.jooq.Field; import org.jooq.Record; import org.jooq.TableLike; -import org.jooq.impl.DSL; public class CQConceptConverter implements NodeConverter { @@ -75,11 +71,11 @@ private static QueryStep finishConceptConversion(QueryStep predecessor, CQConcep SelectContext selectContext = SelectContext.create(ids, validityDate, universalTables, context); List converted = cqConcept.getSelects().stream() - .map(selectId -> { + .map(selectId -> { Select select = selectId.resolve(); - return select.createConverter().conceptSelect(select, selectContext); + return context.getDialectBundle().getSelectConverter(select).conceptSelect(select, selectContext); }) - .toList(); + .toList(); List queriesToJoin = new ArrayList<>(); queriesToJoin.add(predecessor); @@ -94,16 +90,16 @@ private static QueryStep finishConceptConversion(QueryStep predecessor, CQConcep List allConceptSelects = Stream.concat( converted.stream().flatMap(sqlSelects -> sqlSelects.getFinalSelects().stream()), // aggregate special selects (e.g. Exists) - predecessor.getQualifiedSelects().getSqlSelects().stream().map(SqlSelect::connectorAggregate) + predecessor.getQualifiedSelects().getSqlSelects().stream().map(SqlSelect::connectorAggregate) ) - .toList(); + .toList(); Selects finalSelects = Selects.builder() - .ids(ids) - .stratificationDate(predecessorSelects.getStratificationDate()) - .validityDate(validityDate) - .sqlSelects(allConceptSelects) - .build(); + .ids(ids) + .stratificationDate(predecessorSelects.getStratificationDate()) + .validityDate(validityDate) + .sqlSelects(allConceptSelects) + .build(); TableLike joinedTable = QueryStepJoiner.constructJoinedTable(queriesToJoin, ConqueryJoinType.INNER_JOIN, context); @@ -112,27 +108,28 @@ private static QueryStep finishConceptConversion(QueryStep predecessor, CQConcep Stream.concat( finalSelects.nonExplicitSelects().stream(), finalSelects.getSqlSelects().stream() - .filter(Predicate.not(SqlSelect::isUniversal)) - .flatMap(sqlSelect -> sqlSelect.toFields().stream()) + .filter(Predicate.not(SqlSelect::isUniversal)) + .flatMap(sqlSelect -> sqlSelect.toFields().stream()) ).toList(); return QueryStep.builder() - .cteName(universalTables.cteName(ConceptCteStep.UNIVERSAL_SELECTS)) - .selects(finalSelects) - .fromTable(joinedTable) - .groupBy(groupByFields) - .predecessors(queriesToJoin) - .build(); + .cteName(universalTables.cteName(ConceptCteStep.UNIVERSAL_SELECTS)) + .selects(finalSelects) + .fromTable(joinedTable) + .groupBy(groupByFields) + .predecessors(queriesToJoin) + .negate(context.isNegation()) + .build(); } public static SqlIdColumns convertIds(CQConcept cqConcept, CQTable cqTable, ConversionContext conversionContext) { Table table = cqTable.getConnector().resolve().getResolvedTable(); - Field primaryColumn = TablePrimaryColumnUtil.findPrimaryColumn(table, conversionContext.getDefaultPrimaryColumn()); + Field primaryColumn = TablePrimaryColumnUtil.findPrimaryColumn(table, conversionContext.getDefaultPrimaryColumn()); if (cqConcept.isExcludeFromSecondaryId() - || conversionContext.getSecondaryIdDescription() == null - || !cqTable.hasSelectedSecondaryId(conversionContext.getSecondaryIdDescription().getId()) + || conversionContext.getSecondaryIdDescription() == null + || !cqTable.hasSelectedSecondaryId(conversionContext.getSecondaryIdDescription().getId()) ) { return new SqlIdColumns(primaryColumn).withAlias(); } @@ -147,131 +144,125 @@ public static SqlIdColumns convertIds(CQConcept cqConcept, CQTable cqTable, Conv ) ); - Field secondaryId = DSL.field(DSL.name(table.getName(), secondaryIdColumn.getName())); + Field secondaryId = field(name(table.getName(), secondaryIdColumn.getName()), String.class); return new SqlIdColumns(primaryColumn, secondaryId).withAlias(); } - private static Optional convertValidityDate(ValidityDate selected, String connectorLabel, ConversionContext context) { - if (Objects.isNull(selected)) { - return Optional.empty(); - } + private static ColumnDateRange convertValidityDate(String connectorLabel, ConversionContext context, ValidityDate validityDate) { SqlFunctionProvider functionProvider = context.getFunctionProvider(); + ColumnDateRange sqlValidityDate; - if (context.getDateRestrictionRange() != null) { - return Optional.of(functionProvider.forValidityDate(selected, context.getDateRestrictionRange()).asValidityDateRange(connectorLabel)); - } + boolean hasValidityDate = validityDate != null; + boolean hasDateRestriction = context.getDateRestrictionRange() != null; - return Optional.of(functionProvider.forValidityDate(selected).asValidityDateRange(connectorLabel)); - } - - private static boolean dateRestrictionApplicable(boolean dateRestrictionRequired, Optional validityDateSelect) { - return dateRestrictionRequired && validityDateSelect.isPresent(); - } + if (hasValidityDate) { + if (hasDateRestriction) { + sqlValidityDate = functionProvider.forValidityDate(validityDate, context.getDateRestrictionRange()); + } + else { + sqlValidityDate = functionProvider.forValidityDate(validityDate); + } + } + else { + if (hasDateRestriction) { + sqlValidityDate = functionProvider.forCDateRange(context.getDateRestrictionRange()); + } + else { + sqlValidityDate = functionProvider.allRange(); + } + } - private static SqlFilters collectConceptConditions( - List> conceptElements, CQTable cqTable, SqlFunctionProvider functionProvider, - SqlIdColumns ids) { - WhereCondition collected = collectConditions(conceptElements, cqTable, functionProvider, ids); - return new SqlFilters( - ConnectorSqlSelects.none(), - WhereClauses.builder().preprocessingCondition(collected).build() - ); + return sqlValidityDate.asValidityDateRange(connectorLabel); } - private static WhereCondition collectConditions( - List> conceptElements, CQTable cqTable, SqlFunctionProvider functionProvider, - SqlIdColumns ids) { - WhereCondition connectorCondition = convertConnectorCondition(cqTable, functionProvider, ids); - + private static SqlFilters collectConditionFilters( + List> conceptElements, CQTable cqTable, SqlFunctionProvider functionProvider) { List conditions = new ArrayList<>(); + conditions.addAll(collectConditions(conceptElements, cqTable, functionProvider)); - for (ConceptElement conceptElement : conceptElements) { - conditions.add(convertConceptElementCondition(conceptElement, cqTable, functionProvider)); + ValidityDate validityDate = cqTable.findValidityDate(); + Condition validityDateFilter = noCondition(); + if (validityDate != null) { + validityDateFilter = functionProvider.isNotEmptyValidityDate(validityDate); } - Optional maybeElementConditions = conditions.stream().reduce(WhereCondition::or); + return new SqlFilters(ConnectorSqlSelects.none(), + WhereClauses.builder() + .preprocessingConditions(conditions) + .preprocessingCondition(ConditionUtil.wrap(validityDateFilter)) + .build() + ); + } - if (maybeElementConditions.isEmpty()) { - return connectorCondition; - } + private static List collectConditions(List> conceptElements, CQTable cqTable, SqlFunctionProvider functionProvider) { - return connectorCondition.and(maybeElementConditions.get()); + List conditions = new ArrayList<>(); - } + convertConnectorCondition(cqTable, functionProvider).ifPresent(conditions::add); - private static WhereCondition connectorPrerequisites(SqlIdColumns ids, ValidityDate validityDate, SqlFunctionProvider functionProvider) { - List conditions = new ArrayList<>(); - if (validityDate != null) { - conditions.add(functionProvider.validityDateFilter(validityDate)); - } - for (Field field : ids.getPredecessor().get().toFields()) { - conditions.add(field.isNotNull()); + for (ConceptElement conceptElement : conceptElements) { + collectConditions(cqTable, conceptElement, functionProvider) + .reduce(WhereCondition::and) + .ifPresent(conditions::add); } - return new ConditionWrappingWhereCondition(DSL.and(conditions)); + return conditions; } /** - * Collects all conditions of a given {@link ConceptTreeChild} by resolving the condition of the given node and all of its parent nodes. + * Collects all conditions of a given {@link ConceptElement} by resolving the condition of the given node and all of its parent nodes. */ - private static WhereCondition convertConceptElementCondition(ConceptElement conceptElement, CQTable cqTable, SqlFunctionProvider functionProvider) { - if (!(conceptElement instanceof ConceptTreeChild)) { - return new ConditionWrappingWhereCondition(DSL.noCondition()); + private static Stream collectConditions(CQTable cqTable, ConceptElement conceptElement, SqlFunctionProvider functionProvider) { + if (!(conceptElement instanceof ConceptTreeChild child)) { + return Stream.empty(); } - - ConceptTreeChild child = (ConceptTreeChild) conceptElement; - WhereCondition childCondition = child.getCondition().convertToSqlCondition(CTConditionContext.create(cqTable.getConnector().resolve(), functionProvider)); - WhereCondition parentCondition = convertConceptElementCondition(child.getParent(), cqTable, functionProvider); - - return parentCondition.and(childCondition); + return Stream.concat( + collectConditions(cqTable, child.getParent(), functionProvider), + Stream.of(childCondition) + ); } - private static WhereCondition convertConnectorCondition(CQTable cqTable, SqlFunctionProvider functionProvider, SqlIdColumns ids) { - + private static Optional convertConnectorCondition(CQTable cqTable, SqlFunctionProvider functionProvider) { final Connector connector = cqTable.getConnector().resolve(); - WhereCondition prerequisites = connectorPrerequisites(ids, cqTable.findValidityDate(), functionProvider); - - if (connector.getCondition() == null) { - return prerequisites; - } - WhereCondition converted = connector.getCondition().convertToSqlCondition(CTConditionContext.create(connector, functionProvider)); - - return converted.and(prerequisites); + return Optional.ofNullable(connector.getCondition()) + .map(condition -> condition.convertToSqlCondition(CTConditionContext.create(connector, functionProvider))); } - private static Optional getDateRestriction(ConversionContext context, Optional validityDate) { - - if (!dateRestrictionApplicable(context.dateRestrictionActive(), validityDate)) { - return Optional.empty(); - } + private static SqlFilters dateRestrictionFilter(ConversionContext context, ColumnDateRange validityDate) { + List dateRestrictionSelects = new ArrayList<>(); + List conditions = new ArrayList<>(); SqlFunctionProvider functionProvider = context.getFunctionProvider(); - ColumnDateRange dateRestriction = functionProvider.forCDateRange(context.getDateRestrictionRange()).as(SharedAliases.DATE_RESTRICTION.getAlias()); - List dateRestrictionSelects = dateRestriction.toFields().stream() - .map(FieldWrapper::new) - .collect(Collectors.toList()); + if (context.getDateRestrictionRange() != null) { + ColumnDateRange dateRestriction = functionProvider.forCDateRange(context.getDateRestrictionRange()).as(SharedAliases.DATE_RESTRICTION.getAlias()); + conditions.add(ConditionUtil.wrap(functionProvider.dateRestriction(dateRestriction, validityDate))); - Condition dateRestrictionCondition = functionProvider.dateRestriction(dateRestriction, validityDate.get()); + dateRestrictionSelects.addAll(dateRestriction.toFields().stream() + .map(FieldWrapper::new) + .toList()); + } - return Optional.of(new SqlFilters( + return new SqlFilters( ConnectorSqlSelects.builder().preprocessingSelects(dateRestrictionSelects).build(), - WhereClauses.builder().eventFilter(ConditionUtil.wrap(dateRestrictionCondition)).build() - )); + WhereClauses.builder() + .eventFilters(conditions) + .build() + ); } private static ConnectorSqlSelects createConceptColumnConnectorSqlSelects(CQConcept cqConcept, SelectContext selectContext) { - for (SelectId selectId : cqConcept.getSelects()) { - Select resolve = selectId.resolve(); - if (resolve instanceof ConceptColumnSelect select) { - return select.createConverter().connectorSelect(select, selectContext); - } - } - return ConnectorSqlSelects.none(); + + return cqConcept.getSelects().stream() + .map(SelectId::resolve) + .filter(select -> select instanceof ConceptColumnSelect) + .findFirst() + .map(select -> selectContext.getDialectBundle().getSelectConverter(select).connectorSelect(select, selectContext)) + .orElse(ConnectorSqlSelects.none()); } @Override @@ -284,8 +275,8 @@ public ConversionContext convert(CQConcept cqConcept, ConversionContext context) TablePath tablePath = new TablePath(cqConcept, context); List convertedCQTables = cqConcept.getTables().stream() - .flatMap(cqTable -> convertCqTable(tablePath, cqConcept, cqTable, context).stream()) - .toList(); + .flatMap(cqTable -> convertCqTable(tablePath, cqConcept, cqTable, context).stream()) + .toList(); QueryStep joinedStep = QueryStepJoiner.joinSteps(convertedCQTables, ConqueryJoinType.OUTER_JOIN, DateAggregationAction.MERGE, context); QueryStep lastConceptStep = finishConceptConversion(joinedStep, cqConcept, tablePath, context); @@ -308,47 +299,42 @@ private Optional convertCqTable(TablePath tablePath, CQConcept cqConc private CQTableContext createTableContext(TablePath tablePath, CQConcept cqConcept, CQTable cqTable, ConversionContext conversionContext) { - ConnectorSqlTables connectorTables = tablePath.getConnectorTables(cqTable); - - // Convert Ids SqlIdColumns ids = convertIds(cqConcept, cqTable, conversionContext); - - // Convert ValidityDate - Optional tablesValidityDate = convertValidityDate(cqTable.findValidityDate(), connectorTables.getLabel(), conversionContext); + ConnectorSqlTables connectorTables = tablePath.getConnectorTables(cqTable); + ColumnDateRange tablesValidityDate = convertValidityDate(connectorTables.getLabel(), conversionContext, cqTable.findValidityDate()); // convert filters SqlFunctionProvider functionProvider = conversionContext.getFunctionProvider(); List allSqlFiltersForTable = new ArrayList<>(); - for (FilterValue filterValue : cqTable.getFilters()) { - allSqlFiltersForTable.add(filterValue.convertToSqlFilter(ids, conversionContext, connectorTables)); - } + cqTable.getFilters().stream() + .map(filterValue -> filterValue.convertToSqlFilter(ids, conversionContext, connectorTables)) + .forEach(allSqlFiltersForTable::add); - List> resolvedConceptElements = cqConcept.getElements().stream().>map(ConceptElementId::resolve).toList(); - allSqlFiltersForTable.add(collectConceptConditions(resolvedConceptElements, cqTable, functionProvider, ids)); + List> conceptElements = cqConcept.getElements().stream().>map(ConceptElementId::resolve).toList(); + allSqlFiltersForTable.add(collectConditionFilters(conceptElements, cqTable, functionProvider)); - getDateRestriction(conversionContext, tablesValidityDate).ifPresent(allSqlFiltersForTable::add); + allSqlFiltersForTable.add(dateRestrictionFilter(conversionContext, tablesValidityDate)); // convert selects - SelectContext selectContext = SelectContext.create(ids, tablesValidityDate, connectorTables, conversionContext); + SelectContext selectContext = SelectContext.create(ids, Optional.of(tablesValidityDate), connectorTables, conversionContext); List allSelectsForTable = new ArrayList<>(); ConnectorSqlSelects conceptColumnSelect = createConceptColumnConnectorSqlSelects(cqConcept, selectContext); allSelectsForTable.add(conceptColumnSelect); - - for (ConnectorSelectId connectorSelectId : cqTable.getSelects()) { - Select select = connectorSelectId.resolve(); - ConnectorSqlSelects connectorSqlSelects = select.createConverter().connectorSelect(select, selectContext); - allSelectsForTable.add(connectorSqlSelects); - } + cqTable.getSelects() + .stream() + .map(SelectId::resolve) + .map(select -> selectContext.getDialectBundle().getSelectConverter(select).connectorSelect(select, selectContext)) + .forEach(allSelectsForTable::add); return CQTableContext.builder() - .ids(ids) - .validityDate(tablesValidityDate) - .sqlSelects(allSelectsForTable) - .sqlFilters(allSqlFiltersForTable) - .connectorTables(connectorTables) - .conversionContext(conversionContext) - .build(); + .ids(ids) + .validityDate(tablesValidityDate) + .sqlSelects(allSelectsForTable) + .sqlFilters(allSqlFiltersForTable) + .connectorTables(connectorTables) + .conversionContext(conversionContext) + .build(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java index 7c83306b23..de565ccb5f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java @@ -24,7 +24,7 @@ class CQTableContext implements Context { SqlIdColumns ids; - Optional validityDate; + ColumnDateRange validityDate; List sqlSelects; List sqlFilters; ConnectorSqlTables connectorTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java index fe9443edc1..e88c60c3cf 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java @@ -21,6 +21,11 @@ public class ConnectorSqlTables extends SqlTables { */ private final boolean withIntervalPacking; + /** + * True if these tables should not propagate a present validity date. + */ + private final boolean excludedFromTimeAggregation; + /** * Corresponding {@link Connector} of these {@link SqlTables}. */ @@ -32,12 +37,14 @@ public ConnectorSqlTables( String rootTable, Map cteNameMap, Map predecessorMap, - boolean containsIntervalPacking + boolean containsIntervalPacking, + boolean excludedFromTimeAggregation ) { super(rootTable, cteNameMap, predecessorMap); this.connector = connector; this.label = conceptConnectorLabel; this.withIntervalPacking = containsIntervalPacking; + this.excludedFromTimeAggregation = excludedFromTimeAggregation; } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventFilterCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventFilterCte.java index 10e13d61bd..18551f789d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventFilterCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventFilterCte.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.stream.Stream; @@ -22,10 +23,17 @@ class EventFilterCte extends ConnectorCte { @Override public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { + List conditions = new ArrayList<>(); + + if (tableContext.getIds().getSecondaryId().isPresent()) { + conditions.add(tableContext.getIds().getSecondaryId().get().isNotNull()); + } + + conditions.addAll(collectEventFilterConditions(tableContext)); return QueryStep.builder() .selects(collectSelects(tableContext)) - .conditions(collectEventFilterConditions(tableContext)); + .conditions(conditions); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java index 49cdb8e9d4..b2f6718dfd 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java @@ -72,6 +72,11 @@ protected QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { if (intervalPackingSelectsStep != lastIntervalPackingStep) { queriesToJoin.add(intervalPackingSelectsStep); } + + // interval packing was required for event date selects, but we won't propagate it + if (tableContext.getConnectorTables().isExcludedFromTimeAggregation()) { + validityDate = Optional.empty(); + } } // additional preceding tables @@ -92,7 +97,7 @@ private static IntervalPackingContext createIntervalPackingContext(CQTableContex Selects predcessorSelects = tableContext.getPrevious().getQualifiedSelects(); return IntervalPackingContext.builder() .ids(predcessorSelects.getIds()) - .daterange(tableContext.getValidityDate().get()) + .daterange(tableContext.getValidityDate()) .tables(tableContext.getConnectorTables()) .build(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java index 305d89eead..3ced541d97 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java @@ -2,6 +2,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Optional; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.QueryStep; @@ -32,10 +33,9 @@ public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { Selects preprocessingSelects = Selects.builder() .ids(tableContext.getIds()) - .validityDate(tableContext.getValidityDate()) + .validityDate(Optional.of(tableContext.getValidityDate())) .sqlSelects(forPreprocessing) .build(); - // all where clauses that don't require any preprocessing (connector/child conditions) List conditions = new ArrayList<>(); @@ -50,12 +50,13 @@ public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { .selects(preprocessingSelects) .conditions(conditions); - if (!tableContext.getConversionContext().isWithStratification()) { - TableLike rootTable = QueryStep.toTableLike(tableContext.getConnectorTables().getPredecessor(ConceptCteStep.PREPROCESSING)); - return builder.fromTable(rootTable); + if (tableContext.getConversionContext().isWithStratification()) { + return joinWithStratificationTable(forPreprocessing, conditions, tableContext); } - return joinWithStratificationTable(forPreprocessing, conditions, tableContext); + TableLike rootTable = QueryStep.toTableLike(tableContext.getConnectorTables().getPredecessor(ConceptCteStep.PREPROCESSING)); + return builder.fromTable(rootTable); + } @@ -81,7 +82,7 @@ private static QueryStep.QueryStepBuilder joinWithStratificationTable( Selects selects = Selects.builder() .ids(stratificationSelects.getIds()) - .validityDate(tableContext.getValidityDate()) + .validityDate(Optional.of(tableContext.getValidityDate())) .stratificationDate(stratificationSelects.getStratificationDate()) .sqlSelects(preprocessingSelects) .build(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java index 3488de7d65..dba9c97022 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java @@ -57,7 +57,8 @@ private static ConnectorSqlTables createConnectorTables(CQConcept cqConcept, CQT tableInfo.getRootTable(), cteNameMap, tableInfo.getMappings(), - tableInfo.isContainsIntervalPacking() + tableInfo.isContainsIntervalPacking(), + tableInfo.isExcludedFromTimeAggregation() ); } @@ -83,15 +84,20 @@ private static TablePathInfo collectConnectorTables(CQConcept cqConcept, CQTable tableInfo.addWithDefaultMapping(MANDATORY_STEPS); boolean eventDateSelectsPresent = cqTable.getSelects().stream().map(SelectId::resolve).anyMatch(Select::isEventDateSelect); - // no validity date aggregation possible nor necessary - if (cqTable.findValidityDate() == null || (!cqConcept.isAggregateEventDates() && !eventDateSelectsPresent)) { + // no validity date aggregation necessary + if (!cqConcept.isAggregateEventDates() && !eventDateSelectsPresent) { return tableInfo; } - // interval packing required + // interval packing requiredw tableInfo.setContainsIntervalPacking(true); tableInfo.addMappings(IntervalPackingCteStep.getMappings(EVENT_FILTER, context.getDialectBundle())); + // validity date propagation not necessary + if (!cqConcept.isAggregateEventDates()) { + tableInfo.setExcludedFromTimeAggregation(true); + } + if (!eventDateSelectsPresent) { return tableInfo; } @@ -160,6 +166,11 @@ private static class TablePathInfo { */ private boolean containsIntervalPacking; + /** + * True if these tables should not propagate a present validity date. + */ + private boolean excludedFromTimeAggregation; + public TablePathInfo() { this.mappings = new HashMap<>(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java index aa18433443..178136ea1a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java @@ -74,7 +74,7 @@ private QueryStep createRangeIndexStep(QueryStep previousEndStep, IntervalPackin Field rangeIndex = DSL.sum( - DSL.when(daterange.getStart().greaterThan(previousEnd), DSL.val(1)) + DSL.when(daterange.getStart().greaterThan(previousEnd), DSL.inline(1)) .otherwise(DSL.inline(null, Integer.class))) .over(DSL.partitionBy(ids.toFields()) .orderBy(daterange.getStart(), daterange.getEnd()) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java index 0b229b03db..d0596962c3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java @@ -1,11 +1,14 @@ package com.bakdata.conquery.sql.conversion.dialect; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.Dialect; +import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.sql.conversion.Converter; @@ -19,6 +22,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.CQConceptConverter; import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; import com.bakdata.conquery.sql.conversion.model.QueryStepTransformer; +import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.conversion.query.AbsoluteFormQueryConverter; import com.bakdata.conquery.sql.conversion.query.CQReusedQueryConverter; import com.bakdata.conquery.sql.conversion.query.ConceptQueryConverter; @@ -27,16 +31,14 @@ import com.bakdata.conquery.sql.conversion.query.RelativFormQueryConverter; import com.bakdata.conquery.sql.conversion.query.SecondaryIdQueryConverter; import com.bakdata.conquery.sql.conversion.query.TableExportQueryConverter; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.bakdata.conquery.sql.execution.SqlCDateSetParser; import org.jooq.DSLContext; import org.jooq.Field; import org.jooq.SQLDialect; -//TODO unify with com.bakdata.conquery.models.config.Dialect public interface DialectBundle { - Dialect getDialect(); - private static > List customize(List defaults, List substitutes) { Map, C> substituteMap = getSubstituteMap(substitutes); return defaults.stream() @@ -52,6 +54,10 @@ public interface DialectBundle { )); } + ResultSetProcessor getResultSetProcessor(ConqueryConfig config); + + Dialect getDialect(); + int getNameMaxLength(); String getConnectionTestString(); @@ -71,8 +77,6 @@ public interface DialectBundle { List> getNodeConverters(DSLContext context); - SqlCDateSetParser getCDateSetParser(); - default boolean supportsSingleColumnRanges() { return false; } @@ -100,4 +104,17 @@ default List> getDefaultNodeConverters(DSLCon ); } + default Map, ? extends SelectConverter> getSelectConverterOverrides(){ + return Collections.emptyMap(); + } + + default SelectConverter maybeOverride = (SelectConverter