diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FeatureDecoderSql.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FeatureDecoderSql.java index f246e7a7d..517bbaccd 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FeatureDecoderSql.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FeatureDecoderSql.java @@ -145,10 +145,14 @@ public void onPush(SqlRow sqlRow) { private void handleMetaRow(SqlRowMeta sqlRow) { - context - .metadata() - .numberReturned( - context.metadata().getNumberReturned().orElse(0) + sqlRow.getNumberReturned()); + // a negative numberReturned marks it as not computed (single-shot/unpaged); leave it unset so + // the encoder omits numberReturned instead of reporting 0 + if (sqlRow.getNumberReturned() >= 0) { + context + .metadata() + .numberReturned( + context.metadata().getNumberReturned().orElse(0) + sqlRow.getNumberReturned()); + } if (sqlRow.getNumberMatched().isPresent()) { context .metadata() diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FeatureQueryEncoderSql.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FeatureQueryEncoderSql.java index 086cac6a3..9fd955923 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FeatureQueryEncoderSql.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FeatureQueryEncoderSql.java @@ -55,6 +55,7 @@ public class FeatureQueryEncoderSql implements FeatureQueryEncoder> allQueryTemplates, @@ -65,6 +66,7 @@ public FeatureQueryEncoderSql( this.allQueryTemplatesMutations = allQueryTemplatesMutations; this.chunkSize = queryGeneratorSettings.getChunkSize(); this.geometryAsWkb = queryGeneratorSettings.getGeometryAsWkb(); + this.computeNumberMatched = queryGeneratorSettings.getComputeNumberMatched(); this.sqlDialect = sqlDialect; } @@ -114,6 +116,7 @@ private SqlQueryBatch encode(FeatureQuery query, Map additionalQ query, additionalQueryParameters, query.returnsSingleFeature(), + false, 0))) .flatMap(s -> s) .collect(Collectors.toList()); @@ -131,7 +134,13 @@ private SqlQueryBatch encode(FeatureQuery query, Map additionalQ private SqlQueryBatch encode( MultiFeatureQuery query, Map additionalQueryParameters) { - int chunks = (query.getLimit() / chunkSize) + (query.getLimit() % chunkSize > 0 ? 1 : 0); + // A multi-query that does not support paging is executed single-shot: every matching row is + // read in one pass per (sub-query, table), with no meta query, chunking or key-range window. + // This drops numberReturned/numberMatched in exchange for far fewer statements and round-trips + // at large sizes (an optional per-sub-query maximum may still cap each sub-query). + boolean unpaged = !query.getSupportPaging() && !query.hitsOnly(); + int chunks = + unpaged ? 1 : (query.getLimit() / chunkSize) + (query.getLimit() % chunkSize > 0 ? 1 : 0); List querySets = IntStream.range(0, query.getQueries().size()) @@ -156,7 +165,8 @@ private SqlQueryBatch encode( typeQuery, query, additionalQueryParameters, - false, + unpaged, + unpaged, queryIndex))) .flatMap(s -> s); }) @@ -168,6 +178,9 @@ private SqlQueryBatch encode( .limit(query.getLimit()) .offset(query.getOffset()) .chunkSize(chunkSize) + .isUnpaged(unpaged) + // when paging, count every sub-query so numberMatched is the full invariant total + .isComputeNumberMatched(query.getSupportPaging() && computeNumberMatched) .build() .withQuerySets(querySets); } @@ -181,14 +194,14 @@ private SqlQuerySet createQuerySet( Query query, Map additionalQueryParameters, boolean skipMetaQuery, + boolean unpaged, int queryIndex) { List sortKeys = transformSortKeys(typeQuery.getSortKeys(), queryTemplates.getMapping()); boolean useMinMaxKeys = queryTemplates.getMapping().getMainTable().isSortKeyUnique(); - // a multi-query may opt out of computing numberMatched to avoid a count query per sub-query - boolean computeNumberMatched = - !(query instanceof MultiFeatureQuery) - || ((MultiFeatureQuery) query).getComputeNumberMatched(); + // a paged multi-query computes numberMatched; a single-shot one (no paging) does not + boolean supportPaging = + !(query instanceof MultiFeatureQuery) || ((MultiFeatureQuery) query).getSupportPaging(); BiFunction> metaQuery = (maxLimit, skipped) -> @@ -209,27 +222,42 @@ private SqlQuerySet createQuerySet( query.hitsOnly(), // numberMatched is invariant across chunks, so compute it only on the // first chunk of each collection; later chunks reuse that value - chunk == 0 && computeNumberMatched)); + chunk == 0 && supportPaging)); TriFunction> valueQueries = (metaResult, maxLimit, skipped) -> queryTemplates.getValueQueryTemplates().stream() .map( valueQueryTemplate -> - valueQueryTemplate.generateValueQuery( - Math.min(limit, maxLimit), - Math.max(0L, offset - skipped), - sortKeys, - typeQuery.getFilter(), - typeQuery.forceSimpleFeatureGeometry(), - (useMinMaxKeys - && ((Objects.nonNull(metaResult.getMinKey()) - && Objects.nonNull(metaResult.getMaxKey())) - || metaResult.getNumberReturned() == 0)) - ? Optional.of( - Tuple.of(metaResult.getMinKey(), metaResult.getMaxKey())) - : Optional.empty(), - additionalQueryParameters)); + // single-shot reads matching rows in one pass: no offset and no key-range + // window (which would otherwise constrain the result set to the meta + // query's minKey/maxKey); an optional per-sub-query maximum caps each + // sub-query (0 = no limit) + unpaged + ? valueQueryTemplate.generateValueQuery( + query instanceof MultiFeatureQuery + ? ((MultiFeatureQuery) query).getMaxFeaturesPerSubQuery() + : 0L, + 0L, + sortKeys, + typeQuery.getFilter(), + typeQuery.forceSimpleFeatureGeometry(), + Optional.empty(), + additionalQueryParameters) + : valueQueryTemplate.generateValueQuery( + Math.min(limit, maxLimit), + Math.max(0L, offset - skipped), + sortKeys, + typeQuery.getFilter(), + typeQuery.forceSimpleFeatureGeometry(), + (useMinMaxKeys + && ((Objects.nonNull(metaResult.getMinKey()) + && Objects.nonNull(metaResult.getMaxKey())) + || metaResult.getNumberReturned() == 0)) + ? Optional.of( + Tuple.of(metaResult.getMinKey(), metaResult.getMaxKey())) + : Optional.empty(), + additionalQueryParameters)); // reuse SchemaSql instances instead of copying them; this is expensive and unnecessary, since // they are immutable diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlConnector.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlConnector.java index 778c4d04d..3b0d8f5ff 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlConnector.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlConnector.java @@ -57,11 +57,13 @@ class Paging { private long lastNumberReturned; private long lastNumberSkipped; private boolean noOffset; + private final boolean computeNumberMatched; - public Paging(long limit, long offset, long chunkSize) { + public Paging(long limit, long offset, long chunkSize, boolean computeNumberMatched) { this.limit = limit; this.offset = offset; this.chunkSize = chunkSize; + this.computeNumberMatched = computeNumberMatched; this.featureCountdown = limit; this.numberSkipped = 0L; @@ -75,9 +77,14 @@ Optional> get(String currentTable) { long found = lastNumberReturned + lastNumberSkipped; // Once the limit is reached or the current collection is exhausted (its last chunk returned - // fewer rows than the chunk size), no further meta query is needed: there are no more rows to - // read and numberMatched was already computed on the collection's first chunk. + // fewer rows than the chunk size), no further rows need to be read. When numberMatched is + // computed, a not-yet-counted sub-query is still given a count-only meta query (maxLimit 0) + // so the reported total covers every sub-query, not only those contributing to the page; its + // value query is skipped downstream because the meta reports numberReturned 0. if (featureCountdown <= 0 || (Objects.equals(lastTable, currentTable) && found < chunkSize)) { + if (computeNumberMatched && !Objects.equals(lastTable, currentTable)) { + return Optional.of(Tuple.of(0L, 0L)); + } return Optional.empty(); } @@ -109,7 +116,11 @@ void register(String currentTable, SqlRowMeta metaResult) { default Reactive.Source getSourceStream( SqlQueryBatch queryBatch, SqlQueryOptions options) { Paging paging = - new Paging(queryBatch.getLimit(), queryBatch.getOffset(), queryBatch.getChunkSize()); + new Paging( + queryBatch.getLimit(), + queryBatch.getOffset(), + queryBatch.getChunkSize(), + queryBatch.isComputeNumberMatched()); Source sqlRowSource1 = Source.iterable(queryBatch.getQuerySets()) @@ -196,10 +207,17 @@ default Reactive.Source getSourceStream( .via( Transformer.flatMap( plan -> { - if (queryBatch.isSingleFeature()) { + if (queryBatch.isSingleFeature() || queryBatch.isUnpaged()) { + boolean unpaged = queryBatch.isUnpaged(); List querySets = queryBatch.getQuerySets(); + // a single-shot query computes neither count; -1 marks both as absent (the + // decoder leaves numberReturned unset and numberMatched stays empty) + // instead of reporting 0 ImmutableSqlRowMeta sqlRowMeta = - getMetaQueryResult(0L, 0L, 0L, 0L, -1L).build(); + getMetaQueryResult(0L, 0L, unpaged ? -1L : 0L, unpaged ? -1L : 0L, -1L) + .build(); + // a server-side cursor keeps memory bounded while streaming all rows + int fetchSize = unpaged ? (int) queryBatch.getChunkSize() : 0; return Source.iterable( IntStream.range(0, querySets.size()) .boxed() @@ -235,6 +253,7 @@ default Reactive.Source getSourceStream( querySets .get(index) .getQueryIndex()) + .fetchSize(fetchSize) .build())) .toArray( (IntFunction[]>) Source[]::new); @@ -247,11 +266,14 @@ default Reactive.Source getSourceStream( List querySets = plan.first(); SqlRowMeta aggregatedMetaResult = plan.second().get(0); List metaResults = plan.second().subList(1, plan.second().size()); + // the value phase only reads window sub-queries; count-only ones are skipped + // via the numberReturned<=0 guard below, so it needs no count-only handling Paging paging2 = new Paging( queryBatch.getLimit(), queryBatch.getOffset(), - queryBatch.getChunkSize()); + queryBatch.getChunkSize(), + false); int[] i = {0}; if (options.isHitsOnly()) { diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlQueryBatch.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlQueryBatch.java index 9763e666e..15acb2d6d 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlQueryBatch.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlQueryBatch.java @@ -24,5 +24,26 @@ default boolean isSingleFeature() { return false; } + /** + * Single-shot (unpaged) mode: every matching row is read in one pass per (sub-query, table) + * without a meta query, without chunking, and without a key-range window. {@code limit} does not + * page and {@code numberReturned}/{@code numberMatched} are not computed. + */ + @Value.Default + default boolean isUnpaged() { + return false; + } + + /** + * Whether {@code numberMatched} is reported. When enabled for a multi-query, the count is + * computed for every sub-query (not only those contributing to the current page) so that the + * reported {@code numberMatched} is the invariant total across all sub-queries; the value queries + * are still executed only for the sub-queries needed for the page. + */ + @Value.Default + default boolean isComputeNumberMatched() { + return false; + } + List getQuerySets(); } diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlQueryOptions.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlQueryOptions.java index 30e949302..d2ba5ebb1 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlQueryOptions.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlQueryOptions.java @@ -79,6 +79,16 @@ default int getChunkSize() { return 1000; } + /** + * JDBC fetch size for the result set. When greater than 0, the query is executed inside a + * transaction so the database driver can use a server-side cursor and stream rows instead of + * buffering the whole result set in memory. Used for single-shot (unpaged) queries. + */ + @Value.Default + default int getFetchSize() { + return 0; + } + @Value.Default default boolean isGeometryWkb() { return false; diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/infra/db/SqlClientRx.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/infra/db/SqlClientRx.java index 39693d796..dd671d55f 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/infra/db/SqlClientRx.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/infra/db/SqlClientRx.java @@ -98,21 +98,31 @@ public Reactive.Source getSourceStream(String query, SqlQueryOptions opt } List logBuffer = new ArrayList<>(5); - // TODO encapsulating the query in a transaction is a workaround for what appears to be a bug in - // rxjava3-jdbc, see https://github.com/interactive-instruments/ldproxy/issues/1293 - Flowable flowable = - session - .select(query) - .get( - resultSet -> { - SqlRow row = new SqlRowVals(collator).read(resultSet, options); + org.davidmoten.rxjava3.jdbc.ResultSetMapper mapper = + resultSet -> { + SqlRow row = new SqlRowVals(collator).read(resultSet, options); - if (LOGGER.isDebugEnabled(MARKER.SQL_RESULT) && logBuffer.size() < 10) { - logBuffer.add(row); - } + if (LOGGER.isDebugEnabled(MARKER.SQL_RESULT) && logBuffer.size() < 10) { + logBuffer.add(row); + } - return row; - }); + return row; + }; + + // A positive fetch size requires a transaction so the database driver uses a server-side cursor + // and streams rows instead of buffering the whole result set in memory (PostgreSQL ignores the + // fetch size with autoCommit=true). + // TODO encapsulating the query in a transaction is also a workaround for what appears to be a + // bug in rxjava3-jdbc, see https://github.com/interactive-instruments/ldproxy/issues/1293 + Flowable flowable = + options.getFetchSize() > 0 + ? session + .select(query) + .transacted() + .fetchSize(options.getFetchSize()) + .valuesOnly() + .get(mapper) + : session.select(query).get(mapper); // TODO: prettify, see // https://github.com/slick/slick/blob/main/slick/src/main/scala/slick/jdbc/StatementInvoker.scala diff --git a/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/ResultSetPagingSpec.groovy b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/ResultSetPagingSpec.groovy new file mode 100644 index 000000000..10057dc87 --- /dev/null +++ b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/ResultSetPagingSpec.groovy @@ -0,0 +1,111 @@ +/* + * Copyright 2026 interactive instruments GmbH + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package de.ii.xtraplatform.features.sql.app + +import com.google.common.collect.ImmutableMap +import de.ii.xtraplatform.cql.app.CqlImpl +import de.ii.xtraplatform.cql.domain.Eq +import de.ii.xtraplatform.cql.domain.ImmutableInResultSet +import de.ii.xtraplatform.cql.domain.InResultSet +import de.ii.xtraplatform.cql.domain.Property +import de.ii.xtraplatform.cql.domain.ScalarLiteral +import de.ii.xtraplatform.crs.domain.OgcCrs +import de.ii.xtraplatform.features.domain.FeatureSchemaFixtures +import de.ii.xtraplatform.features.domain.MappingOperationResolver +import de.ii.xtraplatform.features.domain.MappingRuleFixtures +import de.ii.xtraplatform.features.domain.SortKey +import de.ii.xtraplatform.features.domain.Tuple +import de.ii.xtraplatform.features.json.app.DecoderFactoryJson +import de.ii.xtraplatform.features.sql.domain.ImmutableQueryGeneratorSettings +import de.ii.xtraplatform.features.sql.domain.ImmutableSqlPathDefaults +import de.ii.xtraplatform.features.sql.domain.SqlDialectPgis +import de.ii.xtraplatform.features.sql.domain.SqlPathParser +import de.ii.xtraplatform.features.sql.domain.SqlQueryMapping +import spock.lang.Shared +import spock.lang.Specification + +import java.util.function.Function + +/** + * A paged query expression (supportPaging=true) executes the chunked/keyset path while its + * sub-queries may carry a result-set ('inResultSet') filter. This verifies that the value query + * composes both: the result-set membership sub-query is kept and the keyset paging window is + * applied alongside it. A single-shot query (supportPaging=false) keeps the membership but adds no + * paging. + */ +class ResultSetPagingSpec extends Specification { + + @Shared + Map mappings = [:] + @Shared + SqlQueryTemplatesDeriver deriver + @Shared + SqlMappingDeriver mappingDeriver + @Shared + MappingOperationResolver mappingOperationResolver + + def setupSpec() { + def defaults = new ImmutableSqlPathDefaults.Builder().build() + def cql = new CqlImpl() + def pathParser = new SqlPathParser(defaults, cql, Map.of("JSON", new DecoderFactoryJson(), "EXPRESSION", new DecoderFactorySqlExpression())) + mappingDeriver = new SqlMappingDeriver(pathParser, new ImmutableQueryGeneratorSettings.Builder().build()) + mappingOperationResolver = new MappingOperationResolver() + + ["simple", "value_array"].each { name -> + def schema = FeatureSchemaFixtures.fromYaml(name) + def resolvedSchema = schema.accept(mappingOperationResolver, List.of()) + def rules = MappingRuleFixtures.fromYaml(name) + mappings[name] = mappingDeriver.derive(rules, resolvedSchema).get(0) + } + + // a filter encoder that can resolve result-set producers, like the one used at runtime + def filterEncoder = new FilterEncoderSql(OgcCrs.CRS84, new SqlDialectPgis(), null, null, cql, List.of(), null, + { type -> Optional.ofNullable(mappings[type]) } as Function) + deriver = new SqlQueryTemplatesDeriver(filterEncoder, new SqlDialectPgis(), true, false, Optional.empty()) + } + + static InResultSet inResultSet(String producerType, de.ii.xtraplatform.cql.domain.Cql2Expression producerFilter) { + return new ImmutableInResultSet.Builder() + .from(InResultSet.of("id", "s1")) + .producerType(producerType) + .producerFilter(producerFilter) + .build() + } + + List mainValueQuery(int limit, int offset, Optional> minMaxKeys) { + def schema = FeatureSchemaFixtures.fromYaml("value_array") + def resolvedSchema = schema.accept(mappingOperationResolver, List.of()) + def rules = MappingRuleFixtures.fromYaml("value_array") + def templates = mappingDeriver.derive(rules, resolvedSchema).stream().map(deriver.&derive).toList() + def filter = inResultSet("simple", Eq.of(Property.of("id"), ScalarLiteral.of("foo"))) + return templates.get(0).getValueQueryTemplates() + .collect { it.generateValueQuery(limit, offset, [] as List, Optional.of(filter), false, minMaxKeys, ImmutableMap.of()) } + } + + def 'a paged value query keeps the result-set membership and applies the keyset window'() { + + when: 'the main value query is generated with the filter and a keyset window (limit 10, offset 10)' + String mainQuery = mainValueQuery(10, 10, Optional.of(Tuple.of(10, 19))).get(0) + + then: 'the result-set membership sub-query and the keyset window both appear' + mainQuery.contains("_rs_0_s1 AS MATERIALIZED") + mainQuery.contains(">= 10") + mainQuery.contains("<= 19") + } + + def 'a single-shot value query keeps the result-set membership but has no paging window'() { + + when: 'the main value query is generated single-shot (no limit, no keyset)' + String mainQuery = mainValueQuery(0, 0, Optional.> empty()).get(0) + + then: 'the result-set membership is present and there is no LIMIT/OFFSET paging clause' + mainQuery.contains("_rs_0_s1 AS MATERIALIZED") + !mainQuery.contains("LIMIT") + !mainQuery.contains("OFFSET") + } +} diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/MultiFeatureQuery.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/MultiFeatureQuery.java index 7700c0de7..30c002383 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/MultiFeatureQuery.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/MultiFeatureQuery.java @@ -30,11 +30,22 @@ default boolean getDeduplicate() { } /** - * If disabled, {@code numberMatched} is not computed. For a multi-query this avoids a count query - * per sub-query, each of which carries the full (possibly deeply nested) filter. + * If enabled (the default), the query is executed with paging support: {@code numberReturned} and + * {@code numberMatched} are computed and {@code limit}/{@code offset} select a page of the result + * set. If disabled, the query is executed single-shot: all matching features are returned in one + * pass, without meta queries and without {@code numberReturned}/{@code numberMatched}. */ @Value.Default - default boolean getComputeNumberMatched() { + default boolean getSupportPaging() { return true; } + + /** + * In single-shot mode (paging disabled), an optional maximum number of features read per + * sub-query. {@code 0} means no limit. + */ + @Value.Default + default int getMaxFeaturesPerSubQuery() { + return 0; + } }