From cb8c5089ee4af4b2f535ca5f42a21623a731f9a6 Mon Sep 17 00:00:00 2001 From: Dan Reynolds Date: Wed, 10 Jun 2026 11:29:34 -0400 Subject: [PATCH 01/10] exp 160: tier-1 incremental stream maintenance from row deltas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The writer preupdate hook now captures bounded per-row old/new values (256 rows / 32 cols / 256 KB per cycle; overflow, OOM, or a savepoint rollback poisons the cycle and the write falls back to plain re-query invalidation). Deltas ride the writer reply as raw bytes and are decoded lazily on the main isolate. At stream registration the engine classifies the query against a fail-closed tier-1 grammar (single table, AND-ed integer comparisons, INTEGER PRIMARY KEY projected, ORDER BY pk or pk-equality pin) using PRAGMA table_info. Admitted streams maintain their materialized result from deltas: proven misses skip the reader pool entirely, in-window changes patch clone-on-write and emit, and anything unprovable bails to the existing re-query path. A hash sentinel after patched emissions guarantees the next fallback re-query can never be suppressed against a pre-patch baseline. Results: Tracelite stream gate in both collection orders — many-streams -18.5% / +22.2% main-slower (CIs -122..-96 ms, +100..+113 ms), keyed-PK -14.1% / +9.4% (CIs -60..-27 ms, +13..+37 ms), high-cardinality neutral after order-flipped adjudication. A11c-overlap engagement: 24,500 of 25,000 invalidation decisions proven misses, 500 local patches, 0 bails, zero reader re-queries; audit wall -29% overlap / -46% keyed-PK with emissions 44 -> 500 (per-write patches no longer coalesce behind re-query latency). 26 new equivalence tests; full suite green. Co-Authored-By: Claude Fable 5 --- benchmark/profile/ivm_engage_check.dart | 43 + .../exp-160-stream-delta-ivm-aggregate.md | 70 + ...6-10T11-25-55-exp160-stream-delta-ivm.json | 4094 +++++++++++++++++ ...-06-10T11-25-55-exp160-stream-delta-ivm.md | 922 ++++ experiments/160-stream-delta-ivm.md | 179 + experiments/README.md | 1 + experiments/signals.json | 15 + lib/src/database.dart | 2 + lib/src/native/resqlite_bindings.dart | 61 + lib/src/profile_counters.dart | 15 + lib/src/row_deltas.dart | 141 + lib/src/stream_engine.dart | 125 + lib/src/stream_ivm.dart | 651 +++ lib/src/writer/write_worker.dart | 44 +- lib/src/writer/writer.dart | 1 + native/resqlite.c | 172 +- native/resqlite.h | 43 + test/profile_counters_test.dart | 6 + test/stream_ivm_test.dart | 545 +++ 19 files changed, 7123 insertions(+), 7 deletions(-) create mode 100644 benchmark/profile/ivm_engage_check.dart create mode 100644 benchmark/profile/results/exp-160-stream-delta-ivm-aggregate.md create mode 100644 benchmark/results/2026-06-10T11-25-55-exp160-stream-delta-ivm.json create mode 100644 benchmark/results/2026-06-10T11-25-55-exp160-stream-delta-ivm.md create mode 100644 experiments/160-stream-delta-ivm.md create mode 100644 lib/src/row_deltas.dart create mode 100644 lib/src/stream_ivm.dart create mode 100644 test/stream_ivm_test.dart diff --git a/benchmark/profile/ivm_engage_check.dart b/benchmark/profile/ivm_engage_check.dart new file mode 100644 index 00000000..1227e044 --- /dev/null +++ b/benchmark/profile/ivm_engage_check.dart @@ -0,0 +1,43 @@ +import 'dart:io'; +import 'package:resqlite/resqlite.dart'; +import 'package:resqlite/src/profile_counters.dart'; + +Future main() async { + final dir = await Directory.systemTemp.createTemp('ivm_engage_'); + final db = await Database.open('${dir.path}/t.db'); + final cols = [for (var i = 0; i < 20; i++) String.fromCharCode(97 + i)]; + await db.execute( + 'CREATE TABLE wide(id INTEGER PRIMARY KEY, ${cols.map((c) => '$c TEXT NOT NULL').join(', ')})'); + await db.executeBatch( + 'INSERT INTO wide(id, ${cols.join(', ')}) VALUES (?, ${List.filled(20, '?').join(', ')})', + [for (var i = 0; i < 100; i++) [i, for (final _ in cols) 'v$i']]); + + // A11c-overlap shape: 50 streams projecting id, a, b over id ranges; + // writes update column a (intersects every stream's projection). + final subs = [ + for (var p = 0; p < 50; p++) + db.stream( + 'SELECT id, a, b FROM wide WHERE id >= ? AND id < ? ORDER BY id', + [p * 2, p * 2 + 2], + ).listen((_) {}), + ]; + await Future.delayed(const Duration(milliseconds: 200)); + ProfileCounters.reset(); + for (var w = 0; w < 500; w++) { + await db.execute('UPDATE wide SET a = ? WHERE id = ?', ['w$w', w % 100]); + await Future.delayed(Duration.zero); + await Future.delayed(Duration.zero); + } + await Future.delayed(const Duration(milliseconds: 300)); + final snap = ProfileCounters.snapshot(); + print('ivm_skipped=${snap['ivm_skipped_total']} ' + 'ivm_applied=${snap['ivm_applied_total']} ' + 'ivm_bail=${snap['ivm_bail_total']} ' + 'invalidate_count=${snap['invalidate_count']}'); + for (final s in subs) { + await s.cancel(); + } + await db.close(); + await dir.delete(recursive: true); + exit(0); +} diff --git a/benchmark/profile/results/exp-160-stream-delta-ivm-aggregate.md b/benchmark/profile/results/exp-160-stream-delta-ivm-aggregate.md new file mode 100644 index 00000000..2b26bb27 --- /dev/null +++ b/benchmark/profile/results/exp-160-stream-delta-ivm-aggregate.md @@ -0,0 +1,70 @@ +# Experiment 160 — Tier-1 incremental stream maintenance: profile aggregate + +Date: 2026-06-10 + +Harnesses: + +- `benchmark/profile/ivm_engage_check.dart` (new): A11c-overlap-shaped + engagement check printing the exp 160 profile counters. +- `benchmark/profile/writer_sqlite_wall_audit.dart` (exp 147 harness), + single pass per side, baseline (main) → candidate (exp-160), machine + otherwise idle. +- `benchmark/run_tracelite_experiment.dart --direction=stream-rerun-dispatch`, + two passes (standard order + order-flipped). + +## Engagement (profile counters, candidate) + +50 streams x 500 writes, A11c-overlap shape (every write touches a +projected column): + +``` +ivm_skipped=24,500 ivm_applied=500 ivm_bail=0 invalidate_count=500 +``` + +24,500 of 25,000 per-stream invalidation decisions were proven misses +(no reader dispatch, no SQLite); the remaining 500 were local patches. +Zero re-queries reached the reader pool during the burst. + +## Writer wall split audit (single pass per side) + +Baseline (main @ b32177a): + +| workload | wall_ms | writer_sqlite_us | invalidate_us | residual_us | emissions | +|---|---:|---:|---:|---:|---:| +| A11c baseline (0 streams x 500 writes) | 96.50 | 28,080 | 0 | 68,417 | 0 | +| A11c disjoint (50 x 500) | 100.57 | 22,995 | 21,909 | 55,669 | 0 | +| A11c overlap (50 x 500) | 186.46 | 21,418 | 29,974 | 135,071 | 44 | +| keyed PK subscriptions (50 x 200) | 46.87 | 11,208 | 7,317 | 28,340 | 3 | + +Candidate (exp-160): + +| workload | wall_ms | writer_sqlite_us | invalidate_us | residual_us | emissions | +|---|---:|---:|---:|---:|---:| +| A11c baseline (0 streams x 500 writes) | 75.38 | 22,380 | 0 | 52,998 | 0 | +| A11c disjoint (50 x 500) | 104.16 | 17,684 | 24,120 | 62,360 | 0 | +| A11c overlap (50 x 500) | 132.02 | 16,717 | 67,340 | 47,966 | 500 | +| keyed PK subscriptions (50 x 200) | 25.50 | 6,321 | 8,896 | 10,287 | 3 | + +A11c overlap −29% wall while delivering 500 emissions vs 44 (the +baseline's re-query latency coalesces/suppresses most per-write +changes); keyed-PK −46%. `invalidate_us` grows on overlap because +patch+emit now run inline in the invalidation pass; the residual bucket +(round-trip + completion scheduling) collapses 135k → 48k µs. + +## Tracelite decision summary + +Pass 1 (baseline phase first, candidate second; CVs tight on both +sides): + +| scenario | delta | 95% CI | p | +|---|---:|---|---| +| many-streams-writer-throughput | −18.5% | −122..−96.3 ms | 3.4e-6 | +| keyed-pk-subscriptions | −14.1% | −60.0..−27.4 ms | 1.6e-10 | +| high-cardinality-fanout | +2.11% | +2.59..+12.8 ms | 0.009 | + +Keyed-PK within-run CV drops 0.13–0.15 → 0.02–0.05 under the candidate. + +Pass 2 (order flipped): see experiment writeup. + +Raw per-run JSONs and tracelite artifacts are local-only under +`build/tracelite-experiments/`. diff --git a/benchmark/results/2026-06-10T11-25-55-exp160-stream-delta-ivm.json b/benchmark/results/2026-06-10T11-25-55-exp160-stream-delta-ivm.json new file mode 100644 index 00000000..872a8f78 --- /dev/null +++ b/benchmark/results/2026-06-10T11-25-55-exp160-stream-delta-ivm.json @@ -0,0 +1,4094 @@ +{ + "schemaVersion": 3, + "kind": "release-benchmark-run", + "generatedAt": "2026-06-10T11:25:55.763350", + "label": "exp160-stream-delta-ivm", + "repeatCount": 5, + "environment": { + "cwd": "/Users/dan/Coding/resqlite/.claude/worktrees/exp-160-stream-delta-ivm", + "hostname": "macbookpro.lan", + "os": "macos", + "osVersion": "Version 26.2 (Build 25C56)", + "dartVersion": "3.12.1", + "runtime": "dart-runtime", + "executable": "dart", + "resolvedExecutable": "/usr/local/Cellar/dart/3.12.1/libexec/bin/dart", + "script": "file:///Users/dan/Coding/resqlite/.claude/worktrees/exp-160-stream-delta-ivm/benchmark/run_release.dart", + "processors": 10, + "ci": false, + "githubActions": false, + "gitSha": "b32177adfadd7926fc3f3a61ae0c8e938e4c8c31", + "gitBranch": "exp-160-stream-delta-ivm", + "gitDirty": true, + "benchmarkMode": "release", + "includeSlow": false + }, + "comparisonBaselineCompatibility": { + "selectedBaselineFile": "2026-06-09T11-20-00-exp150-nullable-batch-packing.md", + "mode": "automatic", + "compatible": false, + "reasons": [ + "baseline sidecar is missing environment metadata" + ], + "comparisonExecuted": false + }, + "metrics": { + "Select → Maps / 10 rows / resqlite select()": 0.021, + "Select → Maps / 10 rows / resqlite select() [main]": 0.001, + "Select → Maps / 100 rows / resqlite select()": 0.066, + "Select → Maps / 100 rows / resqlite select() [main]": 0.007, + "Select → Maps / 1000 rows / resqlite select()": 0.512, + "Select → Maps / 1000 rows / resqlite select() [main]": 0.064, + "Select → Maps / 10000 rows / resqlite select()": 8.708, + "Select → Maps / 10000 rows / resqlite select() [main]": 0.717, + "Select → JSON Bytes / 10 rows / resqlite + jsonEncode": 0.039, + "Select → JSON Bytes / 10 rows / resqlite + jsonEncode [main]": 0.02, + "Select → JSON Bytes / 10 rows / resqlite selectBytes()": 0.022, + "Select → JSON Bytes / 10 rows / resqlite selectBytes() [main]": 0.0, + "Select → JSON Bytes / 100 rows / resqlite + jsonEncode": 0.258, + "Select → JSON Bytes / 100 rows / resqlite + jsonEncode [main]": 0.191, + "Select → JSON Bytes / 100 rows / resqlite selectBytes()": 0.086, + "Select → JSON Bytes / 100 rows / resqlite selectBytes() [main]": 0.0, + "Select → JSON Bytes / 1000 rows / resqlite + jsonEncode": 2.372, + "Select → JSON Bytes / 1000 rows / resqlite + jsonEncode [main]": 1.888, + "Select → JSON Bytes / 1000 rows / resqlite selectBytes()": 0.7, + "Select → JSON Bytes / 1000 rows / resqlite selectBytes() [main]": 0.0, + "Select → JSON Bytes / 10000 rows / resqlite + jsonEncode": 32.654, + "Select → JSON Bytes / 10000 rows / resqlite + jsonEncode [main]": 20.903, + "Select → JSON Bytes / 10000 rows / resqlite selectBytes()": 7.114, + "Select → JSON Bytes / 10000 rows / resqlite selectBytes() [main]": 0.006, + "Schema Shapes (1000 rows) / Narrow (2 cols: id + int) / resqlite": 0.123, + "Schema Shapes (1000 rows) / Narrow (2 cols: id + int) / resqlite [main]": 0.028, + "Schema Shapes (1000 rows) / Wide (20 cols: mixed types) / resqlite": 1.237, + "Schema Shapes (1000 rows) / Wide (20 cols: mixed types) / resqlite [main]": 0.269, + "Schema Shapes (1000 rows) / Text-heavy (4 long TEXT cols) / resqlite": 0.947, + "Schema Shapes (1000 rows) / Text-heavy (4 long TEXT cols) / resqlite [main]": 0.072, + "Schema Shapes (1000 rows) / Numeric-heavy (5 numeric cols) / resqlite": 0.353, + "Schema Shapes (1000 rows) / Numeric-heavy (5 numeric cols) / resqlite [main]": 0.071, + "Schema Shapes (1000 rows) / Nullable (50% NULLs) / resqlite": 0.347, + "Schema Shapes (1000 rows) / Nullable (50% NULLs) / resqlite [main]": 0.069, + "Scaling (10 → 20,000 rows) / 10 rows / resqlite": 0.021, + "Scaling (10 → 20,000 rows) / 10 rows / resqlite [main]": 0.001, + "Scaling (10 → 20,000 rows) / 50 rows / resqlite": 0.042, + "Scaling (10 → 20,000 rows) / 50 rows / resqlite [main]": 0.004, + "Scaling (10 → 20,000 rows) / 100 rows / resqlite": 0.065, + "Scaling (10 → 20,000 rows) / 100 rows / resqlite [main]": 0.007, + "Scaling (10 → 20,000 rows) / 500 rows / resqlite": 0.276, + "Scaling (10 → 20,000 rows) / 500 rows / resqlite [main]": 0.036, + "Scaling (10 → 20,000 rows) / 1000 rows / resqlite": 0.529, + "Scaling (10 → 20,000 rows) / 1000 rows / resqlite [main]": 0.071, + "Scaling (10 → 20,000 rows) / 2000 rows / resqlite": 1.099, + "Scaling (10 → 20,000 rows) / 2000 rows / resqlite [main]": 0.14, + "Scaling (10 → 20,000 rows) / 5000 rows / resqlite": 3.361, + "Scaling (10 → 20,000 rows) / 5000 rows / resqlite [main]": 0.36, + "Scaling (10 → 20,000 rows) / 10000 rows / resqlite": 8.588, + "Scaling (10 → 20,000 rows) / 10000 rows / resqlite [main]": 0.737, + "Scaling (10 → 20,000 rows) / 20000 rows / resqlite": 19.172, + "Scaling (10 → 20,000 rows) / 20000 rows / resqlite [main]": 1.625, + "Scaling (10 → 20,000 rows) / 10 rows / resqlite + jsonEncode": 0.04, + "Scaling (10 → 20,000 rows) / 10 rows / resqlite + jsonEncode [main]": 0.04, + "Scaling (10 → 20,000 rows) / 10 rows / resqlite selectBytes()": 0.021, + "Scaling (10 → 20,000 rows) / 10 rows / resqlite selectBytes() [main]": 0.021, + "Scaling (10 → 20,000 rows) / 50 rows / resqlite + jsonEncode": 0.131, + "Scaling (10 → 20,000 rows) / 50 rows / resqlite + jsonEncode [main]": 0.131, + "Scaling (10 → 20,000 rows) / 50 rows / resqlite selectBytes()": 0.048, + "Scaling (10 → 20,000 rows) / 50 rows / resqlite selectBytes() [main]": 0.048, + "Scaling (10 → 20,000 rows) / 100 rows / resqlite + jsonEncode": 0.24, + "Scaling (10 → 20,000 rows) / 100 rows / resqlite + jsonEncode [main]": 0.24, + "Scaling (10 → 20,000 rows) / 100 rows / resqlite selectBytes()": 0.082, + "Scaling (10 → 20,000 rows) / 100 rows / resqlite selectBytes() [main]": 0.082, + "Scaling (10 → 20,000 rows) / 500 rows / resqlite + jsonEncode": 1.125, + "Scaling (10 → 20,000 rows) / 500 rows / resqlite + jsonEncode [main]": 1.125, + "Scaling (10 → 20,000 rows) / 500 rows / resqlite selectBytes()": 0.342, + "Scaling (10 → 20,000 rows) / 500 rows / resqlite selectBytes() [main]": 0.342, + "Scaling (10 → 20,000 rows) / 1000 rows / resqlite + jsonEncode": 2.203, + "Scaling (10 → 20,000 rows) / 1000 rows / resqlite + jsonEncode [main]": 2.203, + "Scaling (10 → 20,000 rows) / 1000 rows / resqlite selectBytes()": 0.657, + "Scaling (10 → 20,000 rows) / 1000 rows / resqlite selectBytes() [main]": 0.657, + "Scaling (10 → 20,000 rows) / 2000 rows / resqlite + jsonEncode": 5.783, + "Scaling (10 → 20,000 rows) / 2000 rows / resqlite + jsonEncode [main]": 5.783, + "Scaling (10 → 20,000 rows) / 2000 rows / resqlite selectBytes()": 1.704, + "Scaling (10 → 20,000 rows) / 2000 rows / resqlite selectBytes() [main]": 1.704, + "Scaling (10 → 20,000 rows) / 5000 rows / resqlite + jsonEncode": 19.581, + "Scaling (10 → 20,000 rows) / 5000 rows / resqlite + jsonEncode [main]": 19.581, + "Scaling (10 → 20,000 rows) / 5000 rows / resqlite selectBytes()": 4.22, + "Scaling (10 → 20,000 rows) / 5000 rows / resqlite selectBytes() [main]": 4.22, + "Scaling (10 → 20,000 rows) / 10000 rows / resqlite + jsonEncode": 32.354, + "Scaling (10 → 20,000 rows) / 10000 rows / resqlite + jsonEncode [main]": 32.354, + "Scaling (10 → 20,000 rows) / 10000 rows / resqlite selectBytes()": 8.079, + "Scaling (10 → 20,000 rows) / 10000 rows / resqlite selectBytes() [main]": 8.079, + "Scaling (10 → 20,000 rows) / 20000 rows / resqlite + jsonEncode": 65.593, + "Scaling (10 → 20,000 rows) / 20000 rows / resqlite + jsonEncode [main]": 65.593, + "Scaling (10 → 20,000 rows) / 20000 rows / resqlite selectBytes()": 16.827, + "Scaling (10 → 20,000 rows) / 20000 rows / resqlite selectBytes() [main]": 16.827, + "Concurrent Reads (1000 rows per query) / 1× concurrency / resqlite": 0.43, + "Concurrent Reads (1000 rows per query) / 1× concurrency / resqlite [main]": 0.43, + "Concurrent Reads (1000 rows per query) / 2× concurrency / resqlite": 0.48, + "Concurrent Reads (1000 rows per query) / 2× concurrency / resqlite [main]": 0.24, + "Concurrent Reads (1000 rows per query) / 4× concurrency / resqlite": 0.52, + "Concurrent Reads (1000 rows per query) / 4× concurrency / resqlite [main]": 0.13, + "Concurrent Reads (1000 rows per query) / 8× concurrency / resqlite": 0.98, + "Concurrent Reads (1000 rows per query) / 8× concurrency / resqlite [main]": 0.12, + "Point Query Throughput / resqlite qps": 70059.0, + "Parameterized Queries / 100 queries × ~500 rows each / resqlite": 21.398, + "Parameterized Queries / 100 queries × ~500 rows each / resqlite [main]": 21.398, + "Write Performance / Single Inserts (100 sequential) / resqlite execute()": 2.635, + "Write Performance / Single Inserts (100 sequential) / resqlite execute() [main]": 2.635, + "Write Performance / Batch Insert (100 rows) / resqlite executeBatch()": 0.098, + "Write Performance / Batch Insert (100 rows) / resqlite executeBatch() [main]": 0.098, + "Write Performance / Batch Insert (1000 rows) / resqlite executeBatch()": 0.61, + "Write Performance / Batch Insert (1000 rows) / resqlite executeBatch() [main]": 0.61, + "Write Performance / Batch Insert (10000 rows) / resqlite executeBatch()": 5.746, + "Write Performance / Batch Insert (10000 rows) / resqlite executeBatch() [main]": 5.746, + "Write Performance / Wide Batch Insert (10000 rows x 20 params) / resqlite executeBatch()": 19.178, + "Write Performance / Wide Batch Insert (10000 rows x 20 params) / resqlite executeBatch() [main]": 19.178, + "Write Performance / Interactive Transaction (insert + select + conditional delete) / resqlite transaction()": 0.086, + "Write Performance / Interactive Transaction (insert + select + conditional delete) / resqlite transaction() [main]": 0.086, + "Write Performance / Batched Write Inside Transaction (100 rows) / resqlite tx.executeBatch()": 0.15, + "Write Performance / Batched Write Inside Transaction (100 rows) / resqlite tx.executeBatch() [main]": 0.15, + "Write Performance / Batched Write Inside Transaction (100 rows) / resqlite tx.execute() loop": 1.401, + "Write Performance / Batched Write Inside Transaction (100 rows) / resqlite tx.execute() loop [main]": 1.401, + "Write Performance / Batched Write Inside Transaction (1000 rows) / resqlite tx.executeBatch()": 0.621, + "Write Performance / Batched Write Inside Transaction (1000 rows) / resqlite tx.executeBatch() [main]": 0.621, + "Write Performance / Batched Write Inside Transaction (1000 rows) / resqlite tx.execute() loop": 13.556, + "Write Performance / Batched Write Inside Transaction (1000 rows) / resqlite tx.execute() loop [main]": 13.556, + "Write Performance / Transaction Read (500 rows) / resqlite tx.select()": 0.159, + "Write Performance / Transaction Read (500 rows) / resqlite tx.select() [main]": 0.159, + "Write Performance / Transaction Read (1000 rows) / resqlite tx.select()": 0.273, + "Write Performance / Transaction Read (1000 rows) / resqlite tx.select() [main]": 0.273, + "Write Performance / Nested Transactions (savepoints) / resqlite nested transaction() x50": 2.941, + "Write Performance / Nested Transactions (savepoints) / resqlite nested transaction() x50 [main]": 2.941, + "Write Performance / Nested Transactions (savepoints) / resqlite nested transaction() depth=5": 0.188, + "Write Performance / Nested Transactions (savepoints) / resqlite nested transaction() depth=5 [main]": 0.188, + "Streaming / Initial Emission / resqlite stream()": 0.054, + "Streaming / Initial Emission / resqlite stream() [main]": 0.054, + "Streaming / Invalidation Latency / resqlite": 0.127, + "Streaming / Invalidation Latency / resqlite [main]": 0.127, + "Streaming / Unchanged Fanout Throughput (1 canary + 10 unchanged streams) / resqlite": 0.325, + "Streaming / Unchanged Fanout Throughput (1 canary + 10 unchanged streams) / resqlite [main]": 0.325, + "Streaming / Long-Text Unchanged Fanout (8 unchanged streams, 256 rows x 4KB TEXT) / resqlite": 0.135, + "Streaming / Long-Text Unchanged Fanout (8 unchanged streams, 256 rows x 4KB TEXT) / resqlite [main]": 0.135, + "Streaming / Fan-out (10 streams) / resqlite": 0.31, + "Streaming / Fan-out (10 streams) / resqlite [main]": 0.31, + "Streaming / Stream Churn (100 cycles) / resqlite": 3.433, + "Streaming / Stream Churn (100 cycles) / resqlite [main]": 3.433, + "Streaming / No-Streams Write Throughput (200 inserts, no active streams) / resqlite": 5.976, + "Streaming / No-Streams Write Throughput (200 inserts, no active streams) / resqlite [main]": 5.976, + "Streaming / Growing-Stream Invalidation (batch-insert 100 into watched stream) / resqlite": 0.173, + "Streaming / Growing-Stream Invalidation (batch-insert 100 into watched stream) / resqlite [main]": 0.173, + "Streaming / Stream Subscription Rate (500 subscribe+cancel cycles) / resqlite": 14.421, + "Streaming / Stream Subscription Rate (500 subscribe+cancel cycles) / resqlite [main]": 14.421, + "Keyed PK Subscriptions (v1) / 50 streams × 200 random-PK writes / resqlite stream()": 212.08, + "Keyed PK Subscriptions (v1) / 50 streams × 200 random-PK writes / resqlite stream() [main]": 0.0, + "Chat Sim (v1) / Insert message / resqlite": 0.049, + "Chat Sim (v1) / Insert message / resqlite [main]": 0.0, + "Chat Sim (v1) / Update conversation / resqlite": 0.039, + "Chat Sim (v1) / Update conversation / resqlite [main]": 0.0, + "Chat Sim (v1) / Fetch last-20 messages (JOIN users) / resqlite": 0.05, + "Chat Sim (v1) / Fetch last-20 messages (JOIN users) / resqlite [main]": 0.0, + "Chat Sim (v1) / Fetch user by PK / resqlite": 0.019, + "Chat Sim (v1) / Fetch user by PK / resqlite [main]": 0.0, + "Feed Paging (v1) / Keyset pagination (20 pages × 50 rows) / resqlite": 0.071, + "Feed Paging (v1) / Keyset pagination (20 pages × 50 rows) / resqlite [main]": 0.001, + "Feed Paging (v1) / Reactive feed with 100 concurrent writes / resqlite": 106.908, + "Feed Paging (v1) / Reactive feed with 100 concurrent writes / resqlite [main]": 0.0, + "High-Cardinality Stream Fan-out (v1) / 100 streams × 200 writes / resqlite": 230.33, + "High-Cardinality Stream Fan-out (v1) / 100 streams × 200 writes / resqlite [main]": 0.0 + }, + "memoryMetrics": { + "Memory / Select 10k rows → Maps / resqlite select()": { + "rssDeltaMedMB": 6.14, + "rssDeltaP90MB": 24.32, + "ciLowMB": 3.03, + "ciHighMB": 9.67, + "mdeMB": 3.32 + }, + "Memory / Select 10k rows → Maps / sqlite3 select()": { + "rssDeltaMedMB": 1.98, + "rssDeltaP90MB": 9.56, + "ciLowMB": 0.0, + "ciHighMB": 4.53, + "mdeMB": 2.27 + }, + "Memory / Select 10k rows → Maps / sqlite_async select()": { + "rssDeltaMedMB": 1.0, + "rssDeltaP90MB": 1.66, + "ciLowMB": 0.96, + "ciHighMB": 1.0, + "mdeMB": 0.02 + }, + "Memory / Select 10k rows → Maps / drift select()": { + "rssDeltaMedMB": 5.25, + "rssDeltaP90MB": 19.36, + "ciLowMB": 0.59, + "ciHighMB": 12.28, + "mdeMB": 5.85 + }, + "Memory / Select 10k rows → JSON Bytes / resqlite selectBytes()": { + "rssDeltaMedMB": 0.24, + "rssDeltaP90MB": 8.97, + "ciLowMB": 0.0, + "ciHighMB": 1.16, + "mdeMB": 0.58 + }, + "Memory / Select 10k rows → JSON Bytes / resqlite + jsonEncode": { + "rssDeltaMedMB": 7.83, + "rssDeltaP90MB": 67.64, + "ciLowMB": 0.0, + "ciHighMB": 47.54, + "mdeMB": 23.77 + }, + "Memory / Select 10k rows → JSON Bytes / sqlite3 + jsonEncode": { + "rssDeltaMedMB": 7.68, + "rssDeltaP90MB": 58.14, + "ciLowMB": 0.0, + "ciHighMB": 14.66, + "mdeMB": 7.33 + }, + "Memory / Select 10k rows → JSON Bytes / sqlite_async + jsonEncode": { + "rssDeltaMedMB": 2.38, + "rssDeltaP90MB": 26.09, + "ciLowMB": 0.0, + "ciHighMB": 10.2, + "mdeMB": 5.1 + }, + "Memory / Select 10k rows → JSON Bytes / drift + jsonEncode": { + "rssDeltaMedMB": 0.0, + "rssDeltaP90MB": 31.68, + "ciLowMB": 0.0, + "ciHighMB": 1.94, + "mdeMB": 0.97 + }, + "Memory / Batch insert 10k rows / resqlite executeBatch()": { + "rssDeltaMedMB": 6.73, + "rssDeltaP90MB": 12.64, + "ciLowMB": 0.0, + "ciHighMB": 9.56, + "mdeMB": 4.78 + }, + "Memory / Batch insert 10k rows / sqlite3 executeBatch()": { + "rssDeltaMedMB": 0.0, + "rssDeltaP90MB": 0.62, + "ciLowMB": 0.0, + "ciHighMB": 0.0, + "mdeMB": 0.0 + }, + "Memory / Batch insert 10k rows / sqlite_async executeBatch()": { + "rssDeltaMedMB": 0.29, + "rssDeltaP90MB": 4.98, + "ciLowMB": 0.0, + "ciHighMB": 0.54, + "mdeMB": 0.27 + }, + "Memory / Batch insert 10k rows / drift batch()": { + "rssDeltaMedMB": 0.01, + "rssDeltaP90MB": 2.0, + "ciLowMB": 0.0, + "ciHighMB": 0.11, + "mdeMB": 0.06 + }, + "Memory / Streaming fan-out (10 streams × 100 writes) / resqlite stream()": { + "rssDeltaMedMB": 0.0, + "rssDeltaP90MB": 0.13, + "ciLowMB": 0.0, + "ciHighMB": 0.12, + "mdeMB": 0.06 + }, + "Memory / Streaming fan-out (10 streams × 100 writes) / sqlite_async watch()": { + "rssDeltaMedMB": 0.0, + "rssDeltaP90MB": 0.91, + "ciLowMB": 0.0, + "ciHighMB": 0.03, + "mdeMB": 0.02 + } + }, + "sqliteDiagnosticsMetrics": { + "SQLite Diagnostics / Warm read working set (20000 rows + 2000 point lookups) / resqlite": { + "sqliteTotalKiB": 3210.4, + "pageCacheKiB": 3189.5, + "schemaKiB": 5.3, + "stmtKiB": 15.6, + "walKiB": 2048.0, + "readersBusy": 0 + }, + "SQLite Diagnostics / Statement cache footprint (48 distinct SELECT texts) / resqlite": { + "sqliteTotalKiB": 3299.6, + "pageCacheKiB": 3189.5, + "schemaKiB": 5.3, + "stmtKiB": 104.8, + "walKiB": 2048.0, + "readersBusy": 0 + }, + "SQLite Diagnostics / WAL after write burst (1000 inserted rows) / resqlite": { + "sqliteTotalKiB": 260.9, + "pageCacheKiB": 240.0, + "schemaKiB": 5.3, + "stmtKiB": 15.6, + "walKiB": 161.0, + "readersBusy": 0 + } + }, + "streamingColumnMetrics": { + "Streaming (Column Granularity) / Disjoint column writes (SET c = ?) / resqlite": { + "reemits": 0, + "drainMs": 79.5, + "ratio": 0.0 + }, + "Streaming (Column Granularity) / Disjoint column writes (SET c = ?) / sqlite_async": { + "reemits": 3825, + "drainMs": 1560.8, + "ratio": 0.951 + }, + "Streaming (Column Granularity) / Disjoint column writes (SET c = ?) / drift": { + "reemits": 5000, + "drainMs": 1594.0, + "ratio": 1.0 + }, + "Streaming (Column Granularity) / Overlapping column writes (SET a = ?) / resqlite": { + "reemits": 5000, + "drainMs": 88.4, + "ratio": 0.0 + }, + "Streaming (Column Granularity) / Overlapping column writes (SET a = ?) / sqlite_async": { + "reemits": 4021, + "drainMs": 1556.4, + "ratio": 0.951 + }, + "Streaming (Column Granularity) / Overlapping column writes (SET a = ?) / drift": { + "reemits": 5000, + "drainMs": 1566.8, + "ratio": 1.0 + } + }, + "repeatAggregates": { + "Select → Maps / 10 rows / resqlite select()": { + "median": 0.025, + "madPct": 8.000000000000007, + "stability": "noisy", + "comparisonThresholdPct": 24.00000000000002 + }, + "Select → Maps / 10 rows / resqlite select() [main]": { + "median": 0.001, + "madPct": 0.0, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Select → Maps / 100 rows / resqlite select()": { + "median": 0.067, + "madPct": 1.492537313432837, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Select → Maps / 100 rows / resqlite select() [main]": { + "median": 0.007, + "madPct": 14.285714285714285, + "stability": "noisy", + "comparisonThresholdPct": 42.857142857142854 + }, + "Select → Maps / 1000 rows / resqlite select()": { + "median": 0.52, + "madPct": 1.5384615384615397, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Select → Maps / 1000 rows / resqlite select() [main]": { + "median": 0.064, + "madPct": 1.5625000000000013, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Select → Maps / 10000 rows / resqlite select()": { + "median": 8.313, + "madPct": 4.7515938890893725, + "stability": "moderate", + "comparisonThresholdPct": 14.254781667268118 + }, + "Select → Maps / 10000 rows / resqlite select() [main]": { + "median": 0.683, + "madPct": 4.978038067349915, + "stability": "moderate", + "comparisonThresholdPct": 14.934114202049745 + }, + "Select → JSON Bytes / 10 rows / resqlite + jsonEncode": { + "median": 0.042, + "madPct": 7.1428571428571495, + "stability": "moderate", + "comparisonThresholdPct": 21.42857142857145 + }, + "Select → JSON Bytes / 10 rows / resqlite + jsonEncode [main]": { + "median": 0.02, + "madPct": 5.000000000000004, + "stability": "moderate", + "comparisonThresholdPct": 15.000000000000014 + }, + "Select → JSON Bytes / 10 rows / resqlite selectBytes()": { + "median": 0.025, + "madPct": 4.0000000000000036, + "stability": "moderate", + "comparisonThresholdPct": 12.00000000000001 + }, + "Select → JSON Bytes / 10 rows / resqlite selectBytes() [main]": { + "median": 0.0, + "madPct": 0.0, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Select → JSON Bytes / 100 rows / resqlite + jsonEncode": { + "median": 0.256, + "madPct": 0.7812500000000007, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Select → JSON Bytes / 100 rows / resqlite + jsonEncode [main]": { + "median": 0.19, + "madPct": 1.0526315789473693, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Select → JSON Bytes / 100 rows / resqlite selectBytes()": { + "median": 0.086, + "madPct": 1.1627906976744036, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Select → JSON Bytes / 100 rows / resqlite selectBytes() [main]": { + "median": 0.0, + "madPct": 0.0, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Select → JSON Bytes / 1000 rows / resqlite + jsonEncode": { + "median": 2.56, + "madPct": 1.3671875000000056, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Select → JSON Bytes / 1000 rows / resqlite + jsonEncode [main]": { + "median": 1.949, + "madPct": 2.2575679835813256, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Select → JSON Bytes / 1000 rows / resqlite selectBytes()": { + "median": 0.7, + "madPct": 1.9999999999999862, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Select → JSON Bytes / 1000 rows / resqlite selectBytes() [main]": { + "median": 0.0, + "madPct": 0.0, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Select → JSON Bytes / 10000 rows / resqlite + jsonEncode": { + "median": 29.674, + "madPct": 0.4279840938195016, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Select → JSON Bytes / 10000 rows / resqlite + jsonEncode [main]": { + "median": 20.781, + "madPct": 0.5870747317260955, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Select → JSON Bytes / 10000 rows / resqlite selectBytes()": { + "median": 7.253, + "madPct": 2.7436922652695412, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Select → JSON Bytes / 10000 rows / resqlite selectBytes() [main]": { + "median": 0.008, + "madPct": 12.49999999999999, + "stability": "noisy", + "comparisonThresholdPct": 37.49999999999997 + }, + "Schema Shapes (1000 rows) / Narrow (2 cols: id + int) / resqlite": { + "median": 0.127, + "madPct": 5.511811023622052, + "stability": "moderate", + "comparisonThresholdPct": 16.535433070866155 + }, + "Schema Shapes (1000 rows) / Narrow (2 cols: id + int) / resqlite [main]": { + "median": 0.028, + "madPct": 0.0, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Schema Shapes (1000 rows) / Wide (20 cols: mixed types) / resqlite": { + "median": 1.236, + "madPct": 2.5080906148867244, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Schema Shapes (1000 rows) / Wide (20 cols: mixed types) / resqlite [main]": { + "median": 0.267, + "madPct": 0.7490636704119856, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Schema Shapes (1000 rows) / Text-heavy (4 long TEXT cols) / resqlite": { + "median": 0.979, + "madPct": 2.655771195097029, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Schema Shapes (1000 rows) / Text-heavy (4 long TEXT cols) / resqlite [main]": { + "median": 0.074, + "madPct": 2.7027027027027053, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Schema Shapes (1000 rows) / Numeric-heavy (5 numeric cols) / resqlite": { + "median": 0.356, + "madPct": 2.247191011235957, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Schema Shapes (1000 rows) / Numeric-heavy (5 numeric cols) / resqlite [main]": { + "median": 0.071, + "madPct": 0.0, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Schema Shapes (1000 rows) / Nullable (50% NULLs) / resqlite": { + "median": 0.359, + "madPct": 1.9498607242339852, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Schema Shapes (1000 rows) / Nullable (50% NULLs) / resqlite [main]": { + "median": 0.069, + "madPct": 1.449275362318842, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 10 rows / resqlite": { + "median": 0.024, + "madPct": 12.499999999999996, + "stability": "noisy", + "comparisonThresholdPct": 37.499999999999986 + }, + "Scaling (10 → 20,000 rows) / 10 rows / resqlite [main]": { + "median": 0.001, + "madPct": 0.0, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 50 rows / resqlite": { + "median": 0.045, + "madPct": 6.666666666666657, + "stability": "moderate", + "comparisonThresholdPct": 19.99999999999997 + }, + "Scaling (10 → 20,000 rows) / 50 rows / resqlite [main]": { + "median": 0.004, + "madPct": 0.0, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 100 rows / resqlite": { + "median": 0.074, + "madPct": 4.054054054054058, + "stability": "moderate", + "comparisonThresholdPct": 12.162162162162174 + }, + "Scaling (10 → 20,000 rows) / 100 rows / resqlite [main]": { + "median": 0.007, + "madPct": 0.0, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 500 rows / resqlite": { + "median": 0.276, + "madPct": 1.811594202898552, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 500 rows / resqlite [main]": { + "median": 0.036, + "madPct": 0.0, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 1000 rows / resqlite": { + "median": 0.529, + "madPct": 3.2136105860113453, + "stability": "moderate", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 1000 rows / resqlite [main]": { + "median": 0.071, + "madPct": 0.0, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 2000 rows / resqlite": { + "median": 1.175, + "madPct": 2.2127659574468104, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 2000 rows / resqlite [main]": { + "median": 0.143, + "madPct": 0.6993006993007, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 5000 rows / resqlite": { + "median": 3.412, + "madPct": 1.4947245017584911, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 5000 rows / resqlite [main]": { + "median": 0.367, + "madPct": 1.9073569482288846, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 10000 rows / resqlite": { + "median": 9.205, + "madPct": 11.102661596958177, + "stability": "noisy", + "comparisonThresholdPct": 33.30798479087453 + }, + "Scaling (10 → 20,000 rows) / 10000 rows / resqlite [main]": { + "median": 0.807, + "madPct": 8.674101610904591, + "stability": "noisy", + "comparisonThresholdPct": 26.022304832713772 + }, + "Scaling (10 → 20,000 rows) / 20000 rows / resqlite": { + "median": 20.897, + "madPct": 4.038857252237169, + "stability": "moderate", + "comparisonThresholdPct": 12.116571756711508 + }, + "Scaling (10 → 20,000 rows) / 20000 rows / resqlite [main]": { + "median": 1.773, + "madPct": 4.512126339537499, + "stability": "moderate", + "comparisonThresholdPct": 13.536379018612497 + }, + "Scaling (10 → 20,000 rows) / 10 rows / resqlite + jsonEncode": { + "median": 0.044, + "madPct": 2.2727272727272747, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 10 rows / resqlite + jsonEncode [main]": { + "median": 0.044, + "madPct": 2.2727272727272747, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 10 rows / resqlite selectBytes()": { + "median": 0.028, + "madPct": 7.1428571428571495, + "stability": "moderate", + "comparisonThresholdPct": 21.42857142857145 + }, + "Scaling (10 → 20,000 rows) / 10 rows / resqlite selectBytes() [main]": { + "median": 0.028, + "madPct": 7.1428571428571495, + "stability": "moderate", + "comparisonThresholdPct": 21.42857142857145 + }, + "Scaling (10 → 20,000 rows) / 50 rows / resqlite + jsonEncode": { + "median": 0.142, + "madPct": 0.7042253521126768, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 50 rows / resqlite + jsonEncode [main]": { + "median": 0.142, + "madPct": 0.7042253521126768, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 50 rows / resqlite selectBytes()": { + "median": 0.054, + "madPct": 11.111111111111107, + "stability": "noisy", + "comparisonThresholdPct": 33.33333333333332 + }, + "Scaling (10 → 20,000 rows) / 50 rows / resqlite selectBytes() [main]": { + "median": 0.054, + "madPct": 11.111111111111107, + "stability": "noisy", + "comparisonThresholdPct": 33.33333333333332 + }, + "Scaling (10 → 20,000 rows) / 100 rows / resqlite + jsonEncode": { + "median": 0.255, + "madPct": 1.176470588235295, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 100 rows / resqlite + jsonEncode [main]": { + "median": 0.255, + "madPct": 1.176470588235295, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 100 rows / resqlite selectBytes()": { + "median": 0.085, + "madPct": 3.529411764705885, + "stability": "moderate", + "comparisonThresholdPct": 10.588235294117656 + }, + "Scaling (10 → 20,000 rows) / 100 rows / resqlite selectBytes() [main]": { + "median": 0.085, + "madPct": 3.529411764705885, + "stability": "moderate", + "comparisonThresholdPct": 10.588235294117656 + }, + "Scaling (10 → 20,000 rows) / 500 rows / resqlite + jsonEncode": { + "median": 1.192, + "madPct": 1.67785234899329, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 500 rows / resqlite + jsonEncode [main]": { + "median": 1.192, + "madPct": 1.67785234899329, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 500 rows / resqlite selectBytes()": { + "median": 0.351, + "madPct": 2.564102564102551, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 500 rows / resqlite selectBytes() [main]": { + "median": 0.351, + "madPct": 2.564102564102551, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 1000 rows / resqlite + jsonEncode": { + "median": 2.52, + "madPct": 0.9920634920634885, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 1000 rows / resqlite + jsonEncode [main]": { + "median": 2.52, + "madPct": 0.9920634920634885, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 1000 rows / resqlite selectBytes()": { + "median": 0.696, + "madPct": 2.44252873563217, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 1000 rows / resqlite selectBytes() [main]": { + "median": 0.696, + "madPct": 2.44252873563217, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 2000 rows / resqlite + jsonEncode": { + "median": 6.499, + "madPct": 4.939221418679793, + "stability": "moderate", + "comparisonThresholdPct": 14.817664256039379 + }, + "Scaling (10 → 20,000 rows) / 2000 rows / resqlite + jsonEncode [main]": { + "median": 6.499, + "madPct": 4.939221418679793, + "stability": "moderate", + "comparisonThresholdPct": 14.817664256039379 + }, + "Scaling (10 → 20,000 rows) / 2000 rows / resqlite selectBytes()": { + "median": 1.679, + "madPct": 2.7397260273972623, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 2000 rows / resqlite selectBytes() [main]": { + "median": 1.679, + "madPct": 2.7397260273972623, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 5000 rows / resqlite + jsonEncode": { + "median": 20.341, + "madPct": 3.7362961506317363, + "stability": "moderate", + "comparisonThresholdPct": 11.20888845189521 + }, + "Scaling (10 → 20,000 rows) / 5000 rows / resqlite + jsonEncode [main]": { + "median": 20.341, + "madPct": 3.7362961506317363, + "stability": "moderate", + "comparisonThresholdPct": 11.20888845189521 + }, + "Scaling (10 → 20,000 rows) / 5000 rows / resqlite selectBytes()": { + "median": 4.22, + "madPct": 9.360189573459705, + "stability": "noisy", + "comparisonThresholdPct": 28.080568720379112 + }, + "Scaling (10 → 20,000 rows) / 5000 rows / resqlite selectBytes() [main]": { + "median": 4.22, + "madPct": 9.360189573459705, + "stability": "noisy", + "comparisonThresholdPct": 28.080568720379112 + }, + "Scaling (10 → 20,000 rows) / 10000 rows / resqlite + jsonEncode": { + "median": 32.354, + "madPct": 8.345181430425912, + "stability": "noisy", + "comparisonThresholdPct": 25.035544291277738 + }, + "Scaling (10 → 20,000 rows) / 10000 rows / resqlite + jsonEncode [main]": { + "median": 32.354, + "madPct": 8.345181430425912, + "stability": "noisy", + "comparisonThresholdPct": 25.035544291277738 + }, + "Scaling (10 → 20,000 rows) / 10000 rows / resqlite selectBytes()": { + "median": 7.359, + "madPct": 5.58499796167957, + "stability": "moderate", + "comparisonThresholdPct": 16.75499388503871 + }, + "Scaling (10 → 20,000 rows) / 10000 rows / resqlite selectBytes() [main]": { + "median": 7.359, + "madPct": 5.58499796167957, + "stability": "moderate", + "comparisonThresholdPct": 16.75499388503871 + }, + "Scaling (10 → 20,000 rows) / 20000 rows / resqlite + jsonEncode": { + "median": 68.093, + "madPct": 1.2365441381639797, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 20000 rows / resqlite + jsonEncode [main]": { + "median": 68.093, + "madPct": 1.2365441381639797, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Scaling (10 → 20,000 rows) / 20000 rows / resqlite selectBytes()": { + "median": 16.827, + "madPct": 8.42693290544957, + "stability": "noisy", + "comparisonThresholdPct": 25.280798716348713 + }, + "Scaling (10 → 20,000 rows) / 20000 rows / resqlite selectBytes() [main]": { + "median": 16.827, + "madPct": 8.42693290544957, + "stability": "noisy", + "comparisonThresholdPct": 25.280798716348713 + }, + "Concurrent Reads (1000 rows per query) / 1× concurrency / resqlite": { + "median": 0.47, + "madPct": 6.382978723404249, + "stability": "moderate", + "comparisonThresholdPct": 19.148936170212746 + }, + "Concurrent Reads (1000 rows per query) / 1× concurrency / resqlite [main]": { + "median": 0.47, + "madPct": 6.382978723404249, + "stability": "moderate", + "comparisonThresholdPct": 19.148936170212746 + }, + "Concurrent Reads (1000 rows per query) / 2× concurrency / resqlite": { + "median": 0.5, + "madPct": 2.0000000000000018, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Concurrent Reads (1000 rows per query) / 2× concurrency / resqlite [main]": { + "median": 0.25, + "madPct": 4.0000000000000036, + "stability": "moderate", + "comparisonThresholdPct": 12.00000000000001 + }, + "Concurrent Reads (1000 rows per query) / 4× concurrency / resqlite": { + "median": 0.54, + "madPct": 3.703703703703707, + "stability": "moderate", + "comparisonThresholdPct": 11.111111111111121 + }, + "Concurrent Reads (1000 rows per query) / 4× concurrency / resqlite [main]": { + "median": 0.13, + "madPct": 0.0, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Concurrent Reads (1000 rows per query) / 8× concurrency / resqlite": { + "median": 1.11, + "madPct": 7.207207207207213, + "stability": "moderate", + "comparisonThresholdPct": 21.62162162162164 + }, + "Concurrent Reads (1000 rows per query) / 8× concurrency / resqlite [main]": { + "median": 0.14, + "madPct": 7.142857142857148, + "stability": "moderate", + "comparisonThresholdPct": 21.428571428571445 + }, + "Point Query Throughput / resqlite qps": { + "median": 64561.0, + "madPct": 2.2660739455708554, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Parameterized Queries / 100 queries × ~500 rows each / resqlite": { + "median": 20.984, + "madPct": 1.3820053373999366, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Parameterized Queries / 100 queries × ~500 rows each / resqlite [main]": { + "median": 20.984, + "madPct": 1.3820053373999366, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Write Performance / Single Inserts (100 sequential) / resqlite execute()": { + "median": 3.116, + "madPct": 8.664955070603337, + "stability": "noisy", + "comparisonThresholdPct": 25.994865211810012 + }, + "Write Performance / Single Inserts (100 sequential) / resqlite execute() [main]": { + "median": 3.116, + "madPct": 8.664955070603337, + "stability": "noisy", + "comparisonThresholdPct": 25.994865211810012 + }, + "Write Performance / Batch Insert (100 rows) / resqlite executeBatch()": { + "median": 0.099, + "madPct": 1.0101010101010108, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Write Performance / Batch Insert (100 rows) / resqlite executeBatch() [main]": { + "median": 0.099, + "madPct": 1.0101010101010108, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Write Performance / Batch Insert (1000 rows) / resqlite executeBatch()": { + "median": 0.603, + "madPct": 1.1608623548922068, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Write Performance / Batch Insert (1000 rows) / resqlite executeBatch() [main]": { + "median": 0.603, + "madPct": 1.1608623548922068, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Write Performance / Batch Insert (10000 rows) / resqlite executeBatch()": { + "median": 5.787, + "madPct": 5.78883704855711, + "stability": "moderate", + "comparisonThresholdPct": 17.366511145671332 + }, + "Write Performance / Batch Insert (10000 rows) / resqlite executeBatch() [main]": { + "median": 5.787, + "madPct": 5.78883704855711, + "stability": "moderate", + "comparisonThresholdPct": 17.366511145671332 + }, + "Write Performance / Wide Batch Insert (10000 rows x 20 params) / resqlite executeBatch()": { + "median": 19.522, + "madPct": 1.4086671447597692, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Write Performance / Wide Batch Insert (10000 rows x 20 params) / resqlite executeBatch() [main]": { + "median": 19.522, + "madPct": 1.4086671447597692, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Write Performance / Interactive Transaction (insert + select + conditional delete) / resqlite transaction()": { + "median": 0.101, + "madPct": 8.910891089108905, + "stability": "noisy", + "comparisonThresholdPct": 26.732673267326714 + }, + "Write Performance / Interactive Transaction (insert + select + conditional delete) / resqlite transaction() [main]": { + "median": 0.101, + "madPct": 8.910891089108905, + "stability": "noisy", + "comparisonThresholdPct": 26.732673267326714 + }, + "Write Performance / Batched Write Inside Transaction (100 rows) / resqlite tx.executeBatch()": { + "median": 0.139, + "madPct": 7.913669064748189, + "stability": "moderate", + "comparisonThresholdPct": 23.741007194244567 + }, + "Write Performance / Batched Write Inside Transaction (100 rows) / resqlite tx.executeBatch() [main]": { + "median": 0.139, + "madPct": 7.913669064748189, + "stability": "moderate", + "comparisonThresholdPct": 23.741007194244567 + }, + "Write Performance / Batched Write Inside Transaction (100 rows) / resqlite tx.execute() loop": { + "median": 1.434, + "madPct": 13.38912133891213, + "stability": "noisy", + "comparisonThresholdPct": 40.167364016736386 + }, + "Write Performance / Batched Write Inside Transaction (100 rows) / resqlite tx.execute() loop [main]": { + "median": 1.434, + "madPct": 13.38912133891213, + "stability": "noisy", + "comparisonThresholdPct": 40.167364016736386 + }, + "Write Performance / Batched Write Inside Transaction (1000 rows) / resqlite tx.executeBatch()": { + "median": 0.645, + "madPct": 4.496124031007756, + "stability": "moderate", + "comparisonThresholdPct": 13.48837209302327 + }, + "Write Performance / Batched Write Inside Transaction (1000 rows) / resqlite tx.executeBatch() [main]": { + "median": 0.645, + "madPct": 4.496124031007756, + "stability": "moderate", + "comparisonThresholdPct": 13.48837209302327 + }, + "Write Performance / Batched Write Inside Transaction (1000 rows) / resqlite tx.execute() loop": { + "median": 13.556, + "madPct": 8.099734434936552, + "stability": "noisy", + "comparisonThresholdPct": 24.29920330480966 + }, + "Write Performance / Batched Write Inside Transaction (1000 rows) / resqlite tx.execute() loop [main]": { + "median": 13.556, + "madPct": 8.099734434936552, + "stability": "noisy", + "comparisonThresholdPct": 24.29920330480966 + }, + "Write Performance / Transaction Read (500 rows) / resqlite tx.select()": { + "median": 0.164, + "madPct": 3.0487804878048808, + "stability": "moderate", + "comparisonThresholdPct": 10.0 + }, + "Write Performance / Transaction Read (500 rows) / resqlite tx.select() [main]": { + "median": 0.164, + "madPct": 3.0487804878048808, + "stability": "moderate", + "comparisonThresholdPct": 10.0 + }, + "Write Performance / Transaction Read (1000 rows) / resqlite tx.select()": { + "median": 0.278, + "madPct": 2.5179856115107935, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Write Performance / Transaction Read (1000 rows) / resqlite tx.select() [main]": { + "median": 0.278, + "madPct": 2.5179856115107935, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Write Performance / Nested Transactions (savepoints) / resqlite nested transaction() x50": { + "median": 2.773, + "madPct": 6.058420483231147, + "stability": "moderate", + "comparisonThresholdPct": 18.17526144969344 + }, + "Write Performance / Nested Transactions (savepoints) / resqlite nested transaction() x50 [main]": { + "median": 2.773, + "madPct": 6.058420483231147, + "stability": "moderate", + "comparisonThresholdPct": 18.17526144969344 + }, + "Write Performance / Nested Transactions (savepoints) / resqlite nested transaction() depth=5": { + "median": 0.233, + "madPct": 13.733905579399142, + "stability": "noisy", + "comparisonThresholdPct": 41.201716738197426 + }, + "Write Performance / Nested Transactions (savepoints) / resqlite nested transaction() depth=5 [main]": { + "median": 0.233, + "madPct": 13.733905579399142, + "stability": "noisy", + "comparisonThresholdPct": 41.201716738197426 + }, + "Streaming / Initial Emission / resqlite stream()": { + "median": 0.057, + "madPct": 14.035087719298245, + "stability": "noisy", + "comparisonThresholdPct": 42.10526315789473 + }, + "Streaming / Initial Emission / resqlite stream() [main]": { + "median": 0.057, + "madPct": 14.035087719298245, + "stability": "noisy", + "comparisonThresholdPct": 42.10526315789473 + }, + "Streaming / Invalidation Latency / resqlite": { + "median": 0.107, + "madPct": 18.691588785046733, + "stability": "noisy", + "comparisonThresholdPct": 56.0747663551402 + }, + "Streaming / Invalidation Latency / resqlite [main]": { + "median": 0.107, + "madPct": 18.691588785046733, + "stability": "noisy", + "comparisonThresholdPct": 56.0747663551402 + }, + "Streaming / Unchanged Fanout Throughput (1 canary + 10 unchanged streams) / resqlite": { + "median": 0.325, + "madPct": 4.307692307692312, + "stability": "moderate", + "comparisonThresholdPct": 12.923076923076936 + }, + "Streaming / Unchanged Fanout Throughput (1 canary + 10 unchanged streams) / resqlite [main]": { + "median": 0.325, + "madPct": 4.307692307692312, + "stability": "moderate", + "comparisonThresholdPct": 12.923076923076936 + }, + "Streaming / Long-Text Unchanged Fanout (8 unchanged streams, 256 rows x 4KB TEXT) / resqlite": { + "median": 0.157, + "madPct": 3.821656050955417, + "stability": "moderate", + "comparisonThresholdPct": 11.464968152866252 + }, + "Streaming / Long-Text Unchanged Fanout (8 unchanged streams, 256 rows x 4KB TEXT) / resqlite [main]": { + "median": 0.157, + "madPct": 3.821656050955417, + "stability": "moderate", + "comparisonThresholdPct": 11.464968152866252 + }, + "Streaming / Fan-out (10 streams) / resqlite": { + "median": 0.34, + "madPct": 8.823529411764714, + "stability": "noisy", + "comparisonThresholdPct": 26.47058823529414 + }, + "Streaming / Fan-out (10 streams) / resqlite [main]": { + "median": 0.34, + "madPct": 8.823529411764714, + "stability": "noisy", + "comparisonThresholdPct": 26.47058823529414 + }, + "Streaming / Stream Churn (100 cycles) / resqlite": { + "median": 4.011, + "madPct": 14.55996010969832, + "stability": "noisy", + "comparisonThresholdPct": 43.679880329094956 + }, + "Streaming / Stream Churn (100 cycles) / resqlite [main]": { + "median": 4.011, + "madPct": 14.55996010969832, + "stability": "noisy", + "comparisonThresholdPct": 43.679880329094956 + }, + "Streaming / No-Streams Write Throughput (200 inserts, no active streams) / resqlite": { + "median": 6.32, + "madPct": 3.3069620253164635, + "stability": "moderate", + "comparisonThresholdPct": 10.0 + }, + "Streaming / No-Streams Write Throughput (200 inserts, no active streams) / resqlite [main]": { + "median": 6.32, + "madPct": 3.3069620253164635, + "stability": "moderate", + "comparisonThresholdPct": 10.0 + }, + "Streaming / Growing-Stream Invalidation (batch-insert 100 into watched stream) / resqlite": { + "median": 0.191, + "madPct": 2.6178010471204214, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Streaming / Growing-Stream Invalidation (batch-insert 100 into watched stream) / resqlite [main]": { + "median": 0.191, + "madPct": 2.6178010471204214, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Streaming / Stream Subscription Rate (500 subscribe+cancel cycles) / resqlite": { + "median": 15.475, + "madPct": 6.810985460420034, + "stability": "moderate", + "comparisonThresholdPct": 20.4329563812601 + }, + "Streaming / Stream Subscription Rate (500 subscribe+cancel cycles) / resqlite [main]": { + "median": 15.475, + "madPct": 6.810985460420034, + "stability": "moderate", + "comparisonThresholdPct": 20.4329563812601 + }, + "Keyed PK Subscriptions (v1) / 50 streams × 200 random-PK writes / resqlite stream()": { + "median": 211.92, + "madPct": 0.62759531898829, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Keyed PK Subscriptions (v1) / 50 streams × 200 random-PK writes / resqlite stream() [main]": { + "median": 0.0, + "madPct": 0.0, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Chat Sim (v1) / Insert message / resqlite": { + "median": 0.044, + "madPct": 9.090909090909085, + "stability": "noisy", + "comparisonThresholdPct": 27.272727272727252 + }, + "Chat Sim (v1) / Insert message / resqlite [main]": { + "median": 0.0, + "madPct": 0.0, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Chat Sim (v1) / Update conversation / resqlite": { + "median": 0.036, + "madPct": 5.555555555555541, + "stability": "moderate", + "comparisonThresholdPct": 16.66666666666662 + }, + "Chat Sim (v1) / Update conversation / resqlite [main]": { + "median": 0.0, + "madPct": 0.0, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Chat Sim (v1) / Fetch last-20 messages (JOIN users) / resqlite": { + "median": 0.046, + "madPct": 2.173913043478263, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Chat Sim (v1) / Fetch last-20 messages (JOIN users) / resqlite [main]": { + "median": 0.0, + "madPct": 0.0, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Chat Sim (v1) / Fetch user by PK / resqlite": { + "median": 0.018, + "madPct": 5.555555555555541, + "stability": "moderate", + "comparisonThresholdPct": 16.66666666666662 + }, + "Chat Sim (v1) / Fetch user by PK / resqlite [main]": { + "median": 0.0, + "madPct": 0.0, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Feed Paging (v1) / Keyset pagination (20 pages × 50 rows) / resqlite": { + "median": 0.072, + "madPct": 1.3888888888888902, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Feed Paging (v1) / Keyset pagination (20 pages × 50 rows) / resqlite [main]": { + "median": 0.002, + "madPct": 0.0, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Feed Paging (v1) / Reactive feed with 100 concurrent writes / resqlite": { + "median": 106.632, + "madPct": 0.2588341210893505, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "Feed Paging (v1) / Reactive feed with 100 concurrent writes / resqlite [main]": { + "median": 0.0, + "madPct": 0.0, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "High-Cardinality Stream Fan-out (v1) / 100 streams × 200 writes / resqlite": { + "median": 229.17, + "madPct": 0.5061744556442924, + "stability": "stable", + "comparisonThresholdPct": 10.0 + }, + "High-Cardinality Stream Fan-out (v1) / 100 streams × 200 writes / resqlite [main]": { + "median": 0.0, + "madPct": 0.0, + "stability": "stable", + "comparisonThresholdPct": 10.0 + } + }, + "benchmarkSummary": { + "sections": [ + { + "title": "Select → Maps", + "subtitle": "10 rows", + "entries": [ + [ + "resqlite select()", + [ + 0.021, + 0.049, + 0.001, + 0.002 + ] + ], + [ + "sqlite3 select()", + [ + 0.03, + 0.033, + 0.03, + 0.033 + ] + ], + [ + "sqlite_async select()", + [ + 0.055, + 0.074, + 0.003, + 0.005 + ] + ], + [ + "drift select()", + [ + 0.063, + 0.066, + 0.002, + 0.003 + ] + ] + ] + }, + { + "title": "Select → Maps", + "subtitle": "100 rows", + "entries": [ + [ + "resqlite select()", + [ + 0.066, + 0.069, + 0.007, + 0.007 + ] + ], + [ + "sqlite3 select()", + [ + 0.204, + 0.211, + 0.204, + 0.211 + ] + ], + [ + "sqlite_async select()", + [ + 0.199, + 0.205, + 0.018, + 0.02 + ] + ], + [ + "drift select()", + [ + 0.299, + 0.323, + 0.017, + 0.02 + ] + ] + ] + }, + { + "title": "Select → Maps", + "subtitle": "1000 rows", + "entries": [ + [ + "resqlite select()", + [ + 0.512, + 0.553, + 0.064, + 0.066 + ] + ], + [ + "sqlite3 select()", + [ + 1.93, + 1.992, + 1.93, + 1.992 + ] + ], + [ + "sqlite_async select()", + [ + 1.659, + 1.921, + 0.172, + 0.18 + ] + ], + [ + "drift select()", + [ + 2.65, + 3.132, + 0.166, + 0.17 + ] + ] + ] + }, + { + "title": "Select → Maps", + "subtitle": "10000 rows", + "entries": [ + [ + "resqlite select()", + [ + 8.708, + 20.188, + 0.717, + 1.299 + ] + ], + [ + "sqlite3 select()", + [ + 24.015, + 28.868, + 24.015, + 28.868 + ] + ], + [ + "sqlite_async select()", + [ + 19.142, + 24.053, + 1.681, + 2.684 + ] + ], + [ + "drift select()", + [ + 41.652, + 51.899, + 2.494, + 4.098 + ] + ] + ] + }, + { + "title": "Select → JSON Bytes", + "subtitle": "10 rows", + "entries": [ + [ + "resqlite + jsonEncode", + [ + 0.039, + 0.047, + 0.02, + 0.021 + ] + ], + [ + "sqlite3 + jsonEncode", + [ + 0.047, + 0.051, + 0.047, + 0.051 + ] + ], + [ + "sqlite_async + jsonEncode", + [ + 0.076, + 0.088, + 0.021, + 0.025 + ] + ], + [ + "drift + jsonEncode", + [ + 0.089, + 0.103, + 0.022, + 0.026 + ] + ], + [ + "resqlite selectBytes()", + [ + 0.022, + 0.025, + 0.0, + 0.0 + ] + ] + ] + }, + { + "title": "Select → JSON Bytes", + "subtitle": "100 rows", + "entries": [ + [ + "resqlite + jsonEncode", + [ + 0.258, + 0.262, + 0.191, + 0.194 + ] + ], + [ + "sqlite3 + jsonEncode", + [ + 0.379, + 0.403, + 0.379, + 0.403 + ] + ], + [ + "sqlite_async + jsonEncode", + [ + 0.382, + 0.396, + 0.189, + 0.194 + ] + ], + [ + "drift + jsonEncode", + [ + 0.477, + 0.495, + 0.187, + 0.197 + ] + ], + [ + "resqlite selectBytes()", + [ + 0.086, + 0.096, + 0.0, + 0.0 + ] + ] + ] + }, + { + "title": "Select → JSON Bytes", + "subtitle": "1000 rows", + "entries": [ + [ + "resqlite + jsonEncode", + [ + 2.372, + 5.598, + 1.888, + 3.517 + ] + ], + [ + "sqlite3 + jsonEncode", + [ + 3.693, + 9.09, + 3.693, + 9.09 + ] + ], + [ + "sqlite_async + jsonEncode", + [ + 3.373, + 11.882, + 1.888, + 4.689 + ] + ], + [ + "drift + jsonEncode", + [ + 4.468, + 13.947, + 1.886, + 3.518 + ] + ], + [ + "resqlite selectBytes()", + [ + 0.7, + 0.755, + 0.0, + 0.004 + ] + ] + ] + }, + { + "title": "Select → JSON Bytes", + "subtitle": "10000 rows", + "entries": [ + [ + "resqlite + jsonEncode", + [ + 32.654, + 42.762, + 20.903, + 24.667 + ] + ], + [ + "sqlite3 + jsonEncode", + [ + 43.098, + 57.609, + 43.098, + 57.609 + ] + ], + [ + "sqlite_async + jsonEncode", + [ + 55.225, + 64.016, + 21.279, + 24.435 + ] + ], + [ + "drift + jsonEncode", + [ + 69.361, + 83.358, + 21.346, + 28.551 + ] + ], + [ + "resqlite selectBytes()", + [ + 7.114, + 12.536, + 0.006, + 0.364 + ] + ] + ] + }, + { + "title": "Schema Shapes (1000 rows)", + "subtitle": "Narrow (2 cols: id + int)", + "entries": [ + [ + "resqlite", + [ + 0.123, + 0.138, + 0.028, + 0.03 + ] + ], + [ + "sqlite3", + [ + 0.612, + 0.654, + 0.612, + 0.654 + ] + ], + [ + "sqlite_async", + [ + 0.593, + 0.714, + 0.066, + 0.074 + ] + ], + [ + "drift", + [ + 0.985, + 1.009, + 0.065, + 0.068 + ] + ] + ] + }, + { + "title": "Schema Shapes (1000 rows)", + "subtitle": "Wide (20 cols: mixed types)", + "entries": [ + [ + "resqlite", + [ + 1.237, + 1.345, + 0.269, + 0.281 + ] + ], + [ + "sqlite3", + [ + 5.731, + 6.488, + 5.731, + 6.488 + ] + ], + [ + "sqlite_async", + [ + 4.682, + 5.302, + 0.464, + 0.493 + ] + ], + [ + "drift", + [ + 8.084, + 13.013, + 0.482, + 0.534 + ] + ] + ] + }, + { + "title": "Schema Shapes (1000 rows)", + "subtitle": "Text-heavy (4 long TEXT cols)", + "entries": [ + [ + "resqlite", + [ + 0.947, + 1.017, + 0.072, + 0.076 + ] + ], + [ + "sqlite3", + [ + 2.349, + 3.939, + 2.349, + 3.939 + ] + ], + [ + "sqlite_async", + [ + 1.931, + 2.052, + 0.169, + 0.176 + ] + ], + [ + "drift", + [ + 3.038, + 3.872, + 0.167, + 0.172 + ] + ] + ] + }, + { + "title": "Schema Shapes (1000 rows)", + "subtitle": "Numeric-heavy (5 numeric cols)", + "entries": [ + [ + "resqlite", + [ + 0.353, + 0.367, + 0.071, + 0.073 + ] + ], + [ + "sqlite3", + [ + 1.74, + 1.888, + 1.74, + 1.888 + ] + ], + [ + "sqlite_async", + [ + 1.452, + 1.756, + 0.181, + 0.2 + ] + ], + [ + "drift", + [ + 2.489, + 2.689, + 0.178, + 0.184 + ] + ] + ] + }, + { + "title": "Schema Shapes (1000 rows)", + "subtitle": "Nullable (50% NULLs)", + "entries": [ + [ + "resqlite", + [ + 0.347, + 0.382, + 0.069, + 0.074 + ] + ], + [ + "sqlite3", + [ + 1.574, + 1.755, + 1.574, + 1.755 + ] + ], + [ + "sqlite_async", + [ + 1.388, + 1.491, + 0.166, + 0.18 + ] + ], + [ + "drift", + [ + 2.349, + 2.457, + 0.165, + 0.174 + ] + ] + ] + }, + { + "title": "Scaling (10 → 20,000 rows)", + "subtitle": "10 rows", + "entries": [ + [ + "resqlite", + [ + 0.021, + 0.027, + 0.001, + 0.001 + ] + ], + [ + "sqlite3", + [ + 0.028, + 0.031, + 0.028, + 0.031 + ] + ], + [ + "sqlite_async", + [ + 0.052, + 0.072, + 0.002, + 0.003 + ] + ], + [ + "drift", + [ + 0.062, + 0.067, + 0.002, + 0.002 + ] + ], + [ + "resqlite + jsonEncode", + [ + 0.04, + 0.084, + 0.04, + 0.084 + ] + ], + [ + "sqlite3 + jsonEncode", + [ + 0.044, + 0.056, + 0.044, + 0.056 + ] + ], + [ + "sqlite_async + jsonEncode", + [ + 0.068, + 0.077, + 0.068, + 0.077 + ] + ], + [ + "drift + jsonEncode", + [ + 0.08, + 0.082, + 0.08, + 0.082 + ] + ], + [ + "resqlite selectBytes()", + [ + 0.021, + 0.024, + 0.021, + 0.024 + ] + ] + ] + }, + { + "title": "Scaling (10 → 20,000 rows)", + "subtitle": "50 rows", + "entries": [ + [ + "resqlite", + [ + 0.042, + 0.046, + 0.004, + 0.004 + ] + ], + [ + "sqlite3", + [ + 0.106, + 0.112, + 0.106, + 0.112 + ] + ], + [ + "sqlite_async", + [ + 0.117, + 0.124, + 0.009, + 0.009 + ] + ], + [ + "drift", + [ + 0.169, + 0.178, + 0.009, + 0.009 + ] + ], + [ + "resqlite + jsonEncode", + [ + 0.131, + 0.141, + 0.131, + 0.141 + ] + ], + [ + "sqlite3 + jsonEncode", + [ + 0.182, + 0.188, + 0.182, + 0.188 + ] + ], + [ + "sqlite_async + jsonEncode", + [ + 0.193, + 0.197, + 0.193, + 0.197 + ] + ], + [ + "drift + jsonEncode", + [ + 0.255, + 0.272, + 0.255, + 0.272 + ] + ], + [ + "resqlite selectBytes()", + [ + 0.048, + 0.049, + 0.048, + 0.049 + ] + ] + ] + }, + { + "title": "Scaling (10 → 20,000 rows)", + "subtitle": "100 rows", + "entries": [ + [ + "resqlite", + [ + 0.065, + 0.067, + 0.007, + 0.007 + ] + ], + [ + "sqlite3", + [ + 0.197, + 0.202, + 0.197, + 0.202 + ] + ], + [ + "sqlite_async", + [ + 0.191, + 0.205, + 0.017, + 0.018 + ] + ], + [ + "drift", + [ + 0.304, + 0.319, + 0.016, + 0.018 + ] + ], + [ + "resqlite + jsonEncode", + [ + 0.24, + 0.256, + 0.24, + 0.256 + ] + ], + [ + "sqlite3 + jsonEncode", + [ + 0.359, + 0.395, + 0.359, + 0.395 + ] + ], + [ + "sqlite_async + jsonEncode", + [ + 0.35, + 0.43, + 0.35, + 0.43 + ] + ], + [ + "drift + jsonEncode", + [ + 0.444, + 0.454, + 0.444, + 0.454 + ] + ], + [ + "resqlite selectBytes()", + [ + 0.082, + 0.085, + 0.082, + 0.085 + ] + ] + ] + }, + { + "title": "Scaling (10 → 20,000 rows)", + "subtitle": "500 rows", + "entries": [ + [ + "resqlite", + [ + 0.276, + 0.285, + 0.036, + 0.037 + ] + ], + [ + "sqlite3", + [ + 0.951, + 1.082, + 0.951, + 1.082 + ] + ], + [ + "sqlite_async", + [ + 0.848, + 0.913, + 0.085, + 0.101 + ] + ], + [ + "drift", + [ + 1.341, + 1.462, + 0.082, + 0.089 + ] + ], + [ + "resqlite + jsonEncode", + [ + 1.125, + 1.139, + 1.125, + 1.139 + ] + ], + [ + "sqlite3 + jsonEncode", + [ + 1.754, + 1.909, + 1.754, + 1.909 + ] + ], + [ + "sqlite_async + jsonEncode", + [ + 1.592, + 1.681, + 1.592, + 1.681 + ] + ], + [ + "drift + jsonEncode", + [ + 2.078, + 2.113, + 2.078, + 2.113 + ] + ], + [ + "resqlite selectBytes()", + [ + 0.342, + 0.347, + 0.342, + 0.347 + ] + ] + ] + }, + { + "title": "Scaling (10 → 20,000 rows)", + "subtitle": "1000 rows", + "entries": [ + [ + "resqlite", + [ + 0.529, + 0.607, + 0.071, + 0.076 + ] + ], + [ + "sqlite3", + [ + 1.941, + 2.011, + 1.941, + 2.011 + ] + ], + [ + "sqlite_async", + [ + 1.609, + 1.703, + 0.165, + 0.182 + ] + ], + [ + "drift", + [ + 2.638, + 3.085, + 0.165, + 0.168 + ] + ], + [ + "resqlite + jsonEncode", + [ + 2.203, + 2.267, + 2.203, + 2.267 + ] + ], + [ + "sqlite3 + jsonEncode", + [ + 3.762, + 7.176, + 3.762, + 7.176 + ] + ], + [ + "sqlite_async + jsonEncode", + [ + 3.178, + 11.331, + 3.178, + 11.331 + ] + ], + [ + "drift + jsonEncode", + [ + 4.165, + 12.146, + 4.165, + 12.146 + ] + ], + [ + "resqlite selectBytes()", + [ + 0.657, + 0.666, + 0.657, + 0.666 + ] + ] + ] + }, + { + "title": "Scaling (10 → 20,000 rows)", + "subtitle": "2000 rows", + "entries": [ + [ + "resqlite", + [ + 1.099, + 1.29, + 0.14, + 0.144 + ] + ], + [ + "sqlite3", + [ + 3.858, + 4.871, + 3.858, + 4.871 + ] + ], + [ + "sqlite_async", + [ + 3.263, + 5.224, + 0.331, + 0.415 + ] + ], + [ + "drift", + [ + 5.324, + 5.912, + 0.323, + 0.921 + ] + ], + [ + "resqlite + jsonEncode", + [ + 5.783, + 12.954, + 5.783, + 12.954 + ] + ], + [ + "sqlite3 + jsonEncode", + [ + 8.297, + 16.767, + 8.297, + 16.767 + ] + ], + [ + "sqlite_async + jsonEncode", + [ + 7.246, + 14.48, + 7.246, + 14.48 + ] + ], + [ + "drift + jsonEncode", + [ + 9.698, + 20.192, + 9.698, + 20.192 + ] + ], + [ + "resqlite selectBytes()", + [ + 1.704, + 4.706, + 1.704, + 4.706 + ] + ] + ] + }, + { + "title": "Scaling (10 → 20,000 rows)", + "subtitle": "5000 rows", + "entries": [ + [ + "resqlite", + [ + 3.361, + 10.958, + 0.36, + 2.112 + ] + ], + [ + "sqlite3", + [ + 9.805, + 12.605, + 9.805, + 12.605 + ] + ], + [ + "sqlite_async", + [ + 8.48, + 9.558, + 0.813, + 0.915 + ] + ], + [ + "drift", + [ + 14.177, + 14.529, + 0.817, + 0.889 + ] + ], + [ + "resqlite + jsonEncode", + [ + 19.581, + 25.753, + 19.581, + 25.753 + ] + ], + [ + "sqlite3 + jsonEncode", + [ + 22.179, + 34.178, + 22.179, + 34.178 + ] + ], + [ + "sqlite_async + jsonEncode", + [ + 23.133, + 34.563, + 23.133, + 34.563 + ] + ], + [ + "drift + jsonEncode", + [ + 28.103, + 42.442, + 28.103, + 42.442 + ] + ], + [ + "resqlite selectBytes()", + [ + 4.22, + 9.309, + 4.22, + 9.309 + ] + ] + ] + }, + { + "title": "Scaling (10 → 20,000 rows)", + "subtitle": "10000 rows", + "entries": [ + [ + "resqlite", + [ + 8.588, + 16.947, + 0.737, + 1.592 + ] + ], + [ + "sqlite3", + [ + 23.166, + 30.116, + 23.166, + 30.116 + ] + ], + [ + "sqlite_async", + [ + 16.345, + 17.844, + 1.54, + 1.586 + ] + ], + [ + "drift", + [ + 31.663, + 45.264, + 1.847, + 3.859 + ] + ], + [ + "resqlite + jsonEncode", + [ + 32.354, + 43.597, + 32.354, + 43.597 + ] + ], + [ + "sqlite3 + jsonEncode", + [ + 49.143, + 66.132, + 49.143, + 66.132 + ] + ], + [ + "sqlite_async + jsonEncode", + [ + 60.308, + 71.855, + 60.308, + 71.855 + ] + ], + [ + "drift + jsonEncode", + [ + 70.884, + 86.577, + 70.884, + 86.577 + ] + ], + [ + "resqlite selectBytes()", + [ + 8.079, + 13.317, + 8.079, + 13.317 + ] + ] + ] + }, + { + "title": "Scaling (10 → 20,000 rows)", + "subtitle": "20000 rows", + "entries": [ + [ + "resqlite", + [ + 19.172, + 28.498, + 1.625, + 8.176 + ] + ], + [ + "sqlite3", + [ + 54.991, + 58.473, + 54.991, + 58.473 + ] + ], + [ + "sqlite_async", + [ + 63.204, + 85.638, + 4.681, + 5.123 + ] + ], + [ + "drift", + [ + 85.513, + 99.975, + 4.18, + 4.897 + ] + ], + [ + "resqlite + jsonEncode", + [ + 65.593, + 75.579, + 65.593, + 75.579 + ] + ], + [ + "sqlite3 + jsonEncode", + [ + 98.68, + 108.502, + 98.68, + 108.502 + ] + ], + [ + "sqlite_async + jsonEncode", + [ + 98.032, + 115.728, + 98.032, + 115.728 + ] + ], + [ + "drift + jsonEncode", + [ + 132.373, + 147.909, + 132.373, + 147.909 + ] + ], + [ + "resqlite selectBytes()", + [ + 16.827, + 18.788, + 16.827, + 18.788 + ] + ] + ] + }, + { + "title": "Concurrent Reads (1000 rows per query)", + "subtitle": "1× concurrency", + "entries": [ + [ + "resqlite", + [ + 0.43, + 0.48, + 0.43 + ] + ], + [ + "sqlite_async", + [ + 1.37, + 1.5, + 1.37 + ] + ], + [ + "drift", + [ + 2.51, + 2.96, + 2.51 + ] + ] + ] + }, + { + "title": "Concurrent Reads (1000 rows per query)", + "subtitle": "2× concurrency", + "entries": [ + [ + "resqlite", + [ + 0.48, + 0.53, + 0.24 + ] + ], + [ + "sqlite_async", + [ + 2.15, + 2.7, + 1.08 + ] + ], + [ + "drift", + [ + 4.58, + 5.1, + 2.29 + ] + ] + ] + }, + { + "title": "Concurrent Reads (1000 rows per query)", + "subtitle": "4× concurrency", + "entries": [ + [ + "resqlite", + [ + 0.52, + 1.09, + 0.13 + ] + ], + [ + "sqlite_async", + [ + 3.52, + 4.47, + 0.88 + ] + ], + [ + "drift", + [ + 8.65, + 9.24, + 2.16 + ] + ] + ] + }, + { + "title": "Concurrent Reads (1000 rows per query)", + "subtitle": "8× concurrency", + "entries": [ + [ + "resqlite", + [ + 0.98, + 1.78, + 0.12 + ] + ], + [ + "sqlite_async", + [ + 6.92, + 7.79, + 0.86 + ] + ], + [ + "drift", + [ + 17.98, + 18.42, + 2.25 + ] + ] + ] + }, + { + "title": "Point Query Throughput", + "entries": [ + [ + "Metric", + [ + null + ] + ], + [ + "resqlite qps", + [ + 70059.0 + ] + ], + [ + "resqlite per query", + [ + null + ] + ] + ] + }, + { + "title": "Point Query Throughput", + "subtitle": "QPS + MDE", + "entries": [ + [ + "resqlite", + [ + 70059.0, + null, + 3.0, + 9.1 + ] + ], + [ + "sqlite3", + [ + 96271.0, + null, + 4.0, + 8.2 + ] + ], + [ + "sqlite_async", + [ + 24757.0, + null, + 9.0, + 22.5 + ] + ], + [ + "drift", + [ + 17604.0, + null, + 12.7, + 39.6 + ] + ] + ] + }, + { + "title": "Parameterized Queries", + "subtitle": "100 queries × ~500 rows each", + "entries": [ + [ + "resqlite", + [ + 21.398, + 26.672, + 21.398, + 26.672 + ] + ], + [ + "sqlite_async", + [ + 57.318, + 65.288, + 57.318, + 65.288 + ] + ], + [ + "drift", + [ + 100.867, + 151.538, + 100.867, + 151.538 + ] + ], + [ + "sqlite3 (no cache)", + [ + 50.808, + 79.109, + 50.808, + 79.109 + ] + ], + [ + "sqlite3 (cached stmt)", + [ + 44.103, + 50.156, + 44.103, + 50.156 + ] + ] + ] + }, + { + "title": "Write Performance", + "subtitle": "Single Inserts (100 sequential)", + "entries": [ + [ + "resqlite execute()", + [ + 2.635, + 3.252, + 2.635, + 3.252 + ] + ], + [ + "sqlite3 execute()", + [ + 1.702, + 4.769, + 1.702, + 4.769 + ] + ], + [ + "sqlite_async execute()", + [ + 5.816, + 7.161, + 5.816, + 7.161 + ] + ], + [ + "drift execute()", + [ + 4.995, + 5.987, + 4.995, + 5.987 + ] + ] + ] + }, + { + "title": "Write Performance", + "subtitle": "Batch Insert (100 rows)", + "entries": [ + [ + "resqlite executeBatch()", + [ + 0.098, + 0.106, + 0.098, + 0.106 + ] + ], + [ + "sqlite3 executeBatch()", + [ + 0.088, + 0.097, + 0.088, + 0.097 + ] + ], + [ + "sqlite_async executeBatch()", + [ + 0.184, + 0.313, + 0.184, + 0.313 + ] + ], + [ + "drift executeBatch()", + [ + 0.25, + 0.336, + 0.25, + 0.336 + ] + ] + ] + }, + { + "title": "Write Performance", + "subtitle": "Batch Insert (1000 rows)", + "entries": [ + [ + "resqlite executeBatch()", + [ + 0.61, + 0.91, + 0.61, + 0.91 + ] + ], + [ + "sqlite3 executeBatch()", + [ + 0.802, + 0.979, + 0.802, + 0.979 + ] + ], + [ + "sqlite_async executeBatch()", + [ + 0.921, + 4.477, + 0.921, + 4.477 + ] + ], + [ + "drift executeBatch()", + [ + 1.264, + 1.893, + 1.264, + 1.893 + ] + ] + ] + }, + { + "title": "Write Performance", + "subtitle": "Batch Insert (10000 rows)", + "entries": [ + [ + "resqlite executeBatch()", + [ + 5.746, + 6.686, + 5.746, + 6.686 + ] + ], + [ + "sqlite3 executeBatch()", + [ + 7.705, + 8.272, + 7.705, + 8.272 + ] + ], + [ + "sqlite_async executeBatch()", + [ + 7.885, + 8.676, + 7.885, + 8.676 + ] + ], + [ + "drift executeBatch()", + [ + 10.689, + 13.943, + 10.689, + 13.943 + ] + ] + ] + }, + { + "title": "Write Performance", + "subtitle": "Wide Batch Insert (10000 rows x 20 params)", + "entries": [ + [ + "resqlite executeBatch()", + [ + 19.178, + 44.984, + 19.178, + 44.984 + ] + ], + [ + "sqlite3 executeBatch()", + [ + 39.718, + 42.194, + 39.718, + 42.194 + ] + ], + [ + "sqlite_async executeBatch()", + [ + 43.805, + 63.616, + 43.805, + 63.616 + ] + ], + [ + "drift executeBatch()", + [ + 48.958, + 62.041, + 48.958, + 62.041 + ] + ] + ] + }, + { + "title": "Write Performance", + "subtitle": "Interactive Transaction (insert + select + conditional delete)", + "entries": [ + [ + "resqlite transaction()", + [ + 0.086, + 0.178, + 0.086, + 0.178 + ] + ], + [ + "sqlite_async writeTransaction()", + [ + 0.143, + 0.195, + 0.143, + 0.195 + ] + ] + ] + }, + { + "title": "Write Performance", + "subtitle": "Batched Write Inside Transaction (100 rows)", + "entries": [ + [ + "resqlite tx.executeBatch()", + [ + 0.15, + 0.175, + 0.15, + 0.175 + ] + ], + [ + "resqlite tx.execute() loop", + [ + 1.401, + 1.901, + 1.401, + 1.901 + ] + ], + [ + "sqlite_async tx.execute() loop", + [ + 2.615, + 3.394, + 2.615, + 3.394 + ] + ] + ] + }, + { + "title": "Write Performance", + "subtitle": "Batched Write Inside Transaction (1000 rows)", + "entries": [ + [ + "resqlite tx.executeBatch()", + [ + 0.621, + 0.71, + 0.621, + 0.71 + ] + ], + [ + "resqlite tx.execute() loop", + [ + 13.556, + 17.375, + 13.556, + 17.375 + ] + ], + [ + "sqlite_async tx.execute() loop", + [ + 23.02, + 25.111, + 23.02, + 25.111 + ] + ] + ] + }, + { + "title": "Write Performance", + "subtitle": "Transaction Read (500 rows)", + "entries": [ + [ + "resqlite tx.select()", + [ + 0.159, + 0.167, + 0.159, + 0.167 + ] + ], + [ + "sqlite_async tx.getAll()", + [ + 0.352, + 0.376, + 0.352, + 0.376 + ] + ] + ] + }, + { + "title": "Write Performance", + "subtitle": "Transaction Read (1000 rows)", + "entries": [ + [ + "resqlite tx.select()", + [ + 0.273, + 0.338, + 0.273, + 0.338 + ] + ], + [ + "sqlite_async tx.getAll()", + [ + 0.569, + 0.699, + 0.569, + 0.699 + ] + ] + ] + }, + { + "title": "Write Performance", + "subtitle": "Nested Transactions (savepoints)", + "entries": [ + [ + "resqlite nested transaction() x50", + [ + 2.941, + 3.509, + 2.941, + 3.509 + ] + ], + [ + "resqlite nested transaction() depth=5", + [ + 0.188, + 0.218, + 0.188, + 0.218 + ] + ] + ] + }, + { + "title": "Streaming", + "subtitle": "Initial Emission", + "entries": [ + [ + "resqlite stream()", + [ + 0.054, + 0.091, + 0.054, + 0.091 + ] + ], + [ + "sqlite_async watch()", + [ + 0.2, + 0.326, + 0.2, + 0.326 + ] + ] + ] + }, + { + "title": "Streaming", + "subtitle": "Invalidation Latency", + "entries": [ + [ + "resqlite", + [ + 0.127, + 0.259, + 0.127, + 0.259 + ] + ], + [ + "sqlite_async", + [ + 0.142, + 0.713, + 0.142, + 0.713 + ] + ] + ] + }, + { + "title": "Streaming", + "subtitle": "Unchanged Fanout Throughput (1 canary + 10 unchanged streams)", + "entries": [ + [ + "resqlite", + [ + 0.325, + 0.389, + 0.325, + 0.389 + ] + ], + [ + "sqlite_async", + [ + 0.767, + 2.168, + 0.767, + 2.168 + ] + ] + ] + }, + { + "title": "Streaming", + "subtitle": "Long-Text Unchanged Fanout (8 unchanged streams, 256 rows x 4KB TEXT)", + "entries": [ + [ + "resqlite", + [ + 0.135, + 0.206, + 0.135, + 0.206 + ] + ] + ] + }, + { + "title": "Streaming", + "subtitle": "Fan-out (10 streams)", + "entries": [ + [ + "resqlite", + [ + 0.31, + 0.406, + 0.31, + 0.406 + ] + ], + [ + "sqlite_async", + [ + 0.36, + 0.478, + 0.36, + 0.478 + ] + ] + ] + }, + { + "title": "Streaming", + "subtitle": "Stream Churn (100 cycles)", + "entries": [ + [ + "resqlite", + [ + 3.433, + 3.433, + 3.433, + 3.433 + ] + ], + [ + "sqlite_async", + [ + 16.332, + 16.332, + 16.332, + 16.332 + ] + ] + ] + }, + { + "title": "Streaming", + "subtitle": "No-Streams Write Throughput (200 inserts, no active streams)", + "entries": [ + [ + "resqlite", + [ + 5.976, + 6.64, + 5.976, + 6.64 + ] + ], + [ + "sqlite_async", + [ + 10.428, + 11.678, + 10.428, + 11.678 + ] + ] + ] + }, + { + "title": "Streaming", + "subtitle": "Growing-Stream Invalidation (batch-insert 100 into watched stream)", + "entries": [ + [ + "resqlite", + [ + 0.173, + 0.193, + 0.173, + 0.193 + ] + ] + ] + }, + { + "title": "Streaming", + "subtitle": "Stream Subscription Rate (500 subscribe+cancel cycles)", + "entries": [ + [ + "resqlite", + [ + 14.421, + 16.022, + 14.421, + 16.022 + ] + ] + ] + }, + { + "title": "Streaming (Column Granularity)", + "subtitle": "Disjoint column writes (SET c = ?)", + "entries": [ + [ + "resqlite", + [ + 0.0, + 79.5, + 0.0 + ] + ], + [ + "sqlite_async", + [ + 3825.0, + 1560.8, + 0.951 + ] + ], + [ + "drift", + [ + 5000.0, + 1594.0, + 1.0 + ] + ] + ] + }, + { + "title": "Streaming (Column Granularity)", + "subtitle": "Overlapping column writes (SET a = ?)", + "entries": [ + [ + "resqlite", + [ + 5000.0, + 88.4, + 0.0 + ] + ], + [ + "sqlite_async", + [ + 4021.0, + 1556.4, + 0.951 + ] + ], + [ + "drift", + [ + 5000.0, + 1566.8, + 1.0 + ] + ] + ] + }, + { + "title": "Keyed PK Subscriptions (v1)", + "subtitle": "50 streams × 200 random-PK writes", + "entries": [ + [ + "resqlite stream()", + [ + 212.08, + 212.1, + 0.0, + 0.0, + 0.0, + 3.0 + ] + ], + [ + "sqlite_async stream()", + [ + 445.65, + 446.84, + 0.0, + 0.01, + 1195.0, + 3.0 + ] + ], + [ + "drift stream()", + [ + 659.43, + 664.34, + 0.06, + 0.08, + 10000.0, + 3.0 + ] + ] + ] + }, + { + "title": "Chat Sim (v1)", + "subtitle": "Insert message", + "entries": [ + [ + "resqlite", + [ + 0.049, + 0.093, + 0.0, + 0.0 + ] + ], + [ + "sqlite3", + [ + 0.026, + 0.048, + 0.026, + 0.048 + ] + ], + [ + "sqlite_async", + [ + 0.088, + 0.145, + 0.0, + 0.0 + ] + ], + [ + "drift", + [ + 0.079, + 0.131, + 0.0, + 0.0 + ] + ] + ] + }, + { + "title": "Chat Sim (v1)", + "subtitle": "Update conversation", + "entries": [ + [ + "resqlite", + [ + 0.039, + 0.08, + 0.0, + 0.0 + ] + ], + [ + "sqlite3", + [ + 0.018, + 0.027, + 0.018, + 0.027 + ] + ], + [ + "sqlite_async", + [ + 0.073, + 0.125, + 0.0, + 0.0 + ] + ], + [ + "drift", + [ + 0.067, + 0.117, + 0.0, + 0.0 + ] + ] + ] + }, + { + "title": "Chat Sim (v1)", + "subtitle": "Fetch last-20 messages (JOIN users)", + "entries": [ + [ + "resqlite", + [ + 0.05, + 0.085, + 0.0, + 0.0 + ] + ], + [ + "sqlite3", + [ + 0.051, + 0.057, + 0.051, + 0.057 + ] + ], + [ + "sqlite_async", + [ + 0.102, + 0.158, + 0.001, + 0.001 + ] + ], + [ + "drift", + [ + 0.097, + 0.127, + 0.001, + 0.001 + ] + ] + ] + }, + { + "title": "Chat Sim (v1)", + "subtitle": "Fetch user by PK", + "entries": [ + [ + "resqlite", + [ + 0.019, + 0.049, + 0.0, + 0.0 + ] + ], + [ + "sqlite3", + [ + 0.01, + 0.012, + 0.01, + 0.012 + ] + ], + [ + "sqlite_async", + [ + 0.043, + 0.075, + 0.0, + 0.0 + ] + ], + [ + "drift", + [ + 0.042, + 0.064, + 0.0, + 0.0 + ] + ] + ] + }, + { + "title": "Feed Paging (v1)", + "subtitle": "Keyset pagination (20 pages × 50 rows)", + "entries": [ + [ + "resqlite", + [ + 0.071, + 0.094, + 0.001, + 0.002 + ] + ], + [ + "sqlite3", + [ + 0.112, + 0.116, + 0.112, + 0.116 + ] + ], + [ + "sqlite_async", + [ + 0.132, + 0.148, + 0.002, + 0.002 + ] + ], + [ + "drift", + [ + 0.194, + 0.253, + 0.002, + 0.003 + ] + ] + ] + }, + { + "title": "Feed Paging (v1)", + "subtitle": "Reactive feed with 100 concurrent writes", + "entries": [ + [ + "resqlite", + [ + 106.908, + 107.198, + 0.0, + 0.0, + 0.0 + ] + ], + [ + "sqlite_async", + [ + 212.821, + 213.096, + 0.0, + 0.0, + 40.0 + ] + ], + [ + "drift", + [ + 225.667, + 226.407, + 0.0, + 0.0, + 100.0 + ] + ] + ] + }, + { + "title": "High-Cardinality Stream Fan-out (v1)", + "subtitle": "100 streams × 200 writes", + "entries": [ + [ + "resqlite", + [ + 230.33, + 230.33, + 0.0, + 0.0, + 12.38, + 218.0, + 2.0 + ] + ], + [ + "sqlite_async", + [ + 500.46, + 500.46, + 0.0, + 0.0, + 23.76, + 476.7, + 1183.0 + ] + ], + [ + "drift", + [ + 3077.56, + 3077.56, + 2.53, + 2.53, + 17.91, + 3059.56, + 20000.0 + ] + ] + ] + }, + { + "title": "Memory", + "subtitle": "Select 10k rows → Maps", + "entries": [ + [ + "resqlite select()", + [ + 6.14, + 24.32, + null, + null + ] + ], + [ + "sqlite3 select()", + [ + 1.98, + 9.56, + null, + null + ] + ], + [ + "sqlite_async select()", + [ + 1.0, + 1.66, + null, + null + ] + ], + [ + "drift select()", + [ + 5.25, + 19.36, + null, + null + ] + ] + ] + }, + { + "title": "Memory", + "subtitle": "Select 10k rows → JSON Bytes", + "entries": [ + [ + "resqlite selectBytes()", + [ + 0.24, + 8.97, + null, + null + ] + ], + [ + "resqlite + jsonEncode", + [ + 7.83, + 67.64, + null, + null + ] + ], + [ + "sqlite3 + jsonEncode", + [ + 7.68, + 58.14, + null, + null + ] + ], + [ + "sqlite_async + jsonEncode", + [ + 2.38, + 26.09, + null, + null + ] + ], + [ + "drift + jsonEncode", + [ + 0.0, + 31.68, + null, + null + ] + ] + ] + }, + { + "title": "Memory", + "subtitle": "Batch insert 10k rows", + "entries": [ + [ + "resqlite executeBatch()", + [ + 6.73, + 12.64, + null, + null + ] + ], + [ + "sqlite3 executeBatch()", + [ + 0.0, + 0.62, + null, + null + ] + ], + [ + "sqlite_async executeBatch()", + [ + 0.29, + 4.98, + null, + null + ] + ], + [ + "drift batch()", + [ + 0.01, + 2.0, + null, + null + ] + ] + ] + }, + { + "title": "Memory", + "subtitle": "Streaming fan-out (10 streams × 100 writes)", + "entries": [ + [ + "resqlite stream()", + [ + 0.0, + 0.13, + null, + null + ] + ], + [ + "sqlite_async watch()", + [ + 0.0, + 0.91, + null, + null + ] + ] + ] + }, + { + "title": "SQLite Diagnostics", + "subtitle": "Warm read working set (20000 rows + 2000 point lookups)", + "entries": [ + [ + "resqlite", + [ + 3210.4, + 3189.5, + 5.3, + 15.6, + 2048.0, + 0.0 + ] + ] + ] + }, + { + "title": "SQLite Diagnostics", + "subtitle": "Statement cache footprint (48 distinct SELECT texts)", + "entries": [ + [ + "resqlite", + [ + 3299.6, + 3189.5, + 5.3, + 104.8, + 2048.0, + 0.0 + ] + ] + ] + }, + { + "title": "SQLite Diagnostics", + "subtitle": "WAL after write burst (1000 inserted rows)", + "entries": [ + [ + "resqlite", + [ + 260.9, + 240.0, + 5.3, + 15.6, + 161.0, + 0.0 + ] + ] + ] + } + ] + } +} \ No newline at end of file diff --git a/benchmark/results/2026-06-10T11-25-55-exp160-stream-delta-ivm.md b/benchmark/results/2026-06-10T11-25-55-exp160-stream-delta-ivm.md new file mode 100644 index 00000000..e8566636 --- /dev/null +++ b/benchmark/results/2026-06-10T11-25-55-exp160-stream-delta-ivm.md @@ -0,0 +1,922 @@ +# resqlite Benchmark Results + +Generated: 2026-06-10T11:25:55.763350 + +Libraries compared: +- **resqlite** — raw FFI + C JSON/binary serialization + Isolate.exit zero-copy +- **sqlite3** — raw FFI, synchronous, per-cell column reads +- **sqlite_async** — PowerSync, async connection pool + +Run settings: +- Label: `exp160-stream-delta-ivm` +- Repeats: `5` +- Runtime: `dart-runtime / Dart 3.12.1` +- OS: `macos Version 26.2 (Build 25C56)` +- Git: `exp-160-stream-delta-ivm @ b32177adfadd (dirty)` +- Comparison baseline: `2026-06-09T11-20-00-exp150-nullable-batch-packing.md` +- Comparison mode: `automatic` +- Comparison baseline compatibility: `incompatible (automatic comparison skipped)` + +## Select → Maps + +Query returns `List>`, caller iterates every field. + +### 10 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite select() | 0.021 | 0.049 | 0.001 | 0.002 | +| sqlite3 select() | 0.030 | 0.033 | 0.030 | 0.033 | +| sqlite_async select() | 0.055 | 0.074 | 0.003 | 0.005 | +| drift select() | 0.063 | 0.066 | 0.002 | 0.003 | + +### 100 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite select() | 0.066 | 0.069 | 0.007 | 0.007 | +| sqlite3 select() | 0.204 | 0.211 | 0.204 | 0.211 | +| sqlite_async select() | 0.199 | 0.205 | 0.018 | 0.020 | +| drift select() | 0.299 | 0.323 | 0.017 | 0.020 | + +### 1000 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite select() | 0.512 | 0.553 | 0.064 | 0.066 | +| sqlite3 select() | 1.930 | 1.992 | 1.930 | 1.992 | +| sqlite_async select() | 1.659 | 1.921 | 0.172 | 0.180 | +| drift select() | 2.650 | 3.132 | 0.166 | 0.170 | + +### 10000 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite select() | 8.708 | 20.188 | 0.717 | 1.299 | +| sqlite3 select() | 24.015 | 28.868 | 24.015 | 28.868 | +| sqlite_async select() | 19.142 | 24.053 | 1.681 | 2.684 | +| drift select() | 41.652 | 51.899 | 2.494 | 4.098 | + +## Select → JSON Bytes + +Query result serialized to JSON-encoded `Uint8List` for HTTP response. resqlite's `selectBytes()` encodes natively on the worker isolate (zero-copy transfer to main); other peers and resqlite's own `select()` path go through `jsonEncode + utf8.encode` on the main isolate. Both numbers are reported per peer for the select+encode path; resqlite also reports its native selectBytes path as a separate row. + +### 10 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite + jsonEncode | 0.039 | 0.047 | 0.020 | 0.021 | +| sqlite3 + jsonEncode | 0.047 | 0.051 | 0.047 | 0.051 | +| sqlite_async + jsonEncode | 0.076 | 0.088 | 0.021 | 0.025 | +| drift + jsonEncode | 0.089 | 0.103 | 0.022 | 0.026 | +| resqlite selectBytes() | 0.022 | 0.025 | 0.000 | 0.000 | + +### 100 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite + jsonEncode | 0.258 | 0.262 | 0.191 | 0.194 | +| sqlite3 + jsonEncode | 0.379 | 0.403 | 0.379 | 0.403 | +| sqlite_async + jsonEncode | 0.382 | 0.396 | 0.189 | 0.194 | +| drift + jsonEncode | 0.477 | 0.495 | 0.187 | 0.197 | +| resqlite selectBytes() | 0.086 | 0.096 | 0.000 | 0.000 | + +### 1000 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite + jsonEncode | 2.372 | 5.598 | 1.888 | 3.517 | +| sqlite3 + jsonEncode | 3.693 | 9.090 | 3.693 | 9.090 | +| sqlite_async + jsonEncode | 3.373 | 11.882 | 1.888 | 4.689 | +| drift + jsonEncode | 4.468 | 13.947 | 1.886 | 3.518 | +| resqlite selectBytes() | 0.700 | 0.755 | 0.000 | 0.004 | + +### 10000 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite + jsonEncode | 32.654 | 42.762 | 20.903 | 24.667 | +| sqlite3 + jsonEncode | 43.098 | 57.609 | 43.098 | 57.609 | +| sqlite_async + jsonEncode | 55.225 | 64.016 | 21.279 | 24.435 | +| drift + jsonEncode | 69.361 | 83.358 | 21.346 | 28.551 | +| resqlite selectBytes() | 7.114 | 12.536 | 0.006 | 0.364 | + +## Schema Shapes (1000 rows) + +Tests performance across different column counts and data types. + +### Narrow (2 cols: id + int) + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 0.123 | 0.138 | 0.028 | 0.030 | +| sqlite3 | 0.612 | 0.654 | 0.612 | 0.654 | +| sqlite_async | 0.593 | 0.714 | 0.066 | 0.074 | +| drift | 0.985 | 1.009 | 0.065 | 0.068 | + +### Wide (20 cols: mixed types) + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 1.237 | 1.345 | 0.269 | 0.281 | +| sqlite3 | 5.731 | 6.488 | 5.731 | 6.488 | +| sqlite_async | 4.682 | 5.302 | 0.464 | 0.493 | +| drift | 8.084 | 13.013 | 0.482 | 0.534 | + +### Text-heavy (4 long TEXT cols) + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 0.947 | 1.017 | 0.072 | 0.076 | +| sqlite3 | 2.349 | 3.939 | 2.349 | 3.939 | +| sqlite_async | 1.931 | 2.052 | 0.169 | 0.176 | +| drift | 3.038 | 3.872 | 0.167 | 0.172 | + +### Numeric-heavy (5 numeric cols) + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 0.353 | 0.367 | 0.071 | 0.073 | +| sqlite3 | 1.740 | 1.888 | 1.740 | 1.888 | +| sqlite_async | 1.452 | 1.756 | 0.181 | 0.200 | +| drift | 2.489 | 2.689 | 0.178 | 0.184 | + +### Nullable (50% NULLs) + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 0.347 | 0.382 | 0.069 | 0.074 | +| sqlite3 | 1.574 | 1.755 | 1.574 | 1.755 | +| sqlite_async | 1.388 | 1.491 | 0.166 | 0.180 | +| drift | 2.349 | 2.457 | 0.165 | 0.174 | + +## Scaling (10 → 20,000 rows) + +Shows how each library scales with result size. Identifies the crossover point where resqlite's isolate overhead becomes negligible. + +### Maps (select → iterate all fields) + +### 10 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 0.021 | 0.027 | 0.001 | 0.001 | +| sqlite3 | 0.028 | 0.031 | 0.028 | 0.031 | +| sqlite_async | 0.052 | 0.072 | 0.002 | 0.003 | +| drift | 0.062 | 0.067 | 0.002 | 0.002 | + +### 50 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 0.042 | 0.046 | 0.004 | 0.004 | +| sqlite3 | 0.106 | 0.112 | 0.106 | 0.112 | +| sqlite_async | 0.117 | 0.124 | 0.009 | 0.009 | +| drift | 0.169 | 0.178 | 0.009 | 0.009 | + +### 100 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 0.065 | 0.067 | 0.007 | 0.007 | +| sqlite3 | 0.197 | 0.202 | 0.197 | 0.202 | +| sqlite_async | 0.191 | 0.205 | 0.017 | 0.018 | +| drift | 0.304 | 0.319 | 0.016 | 0.018 | + +### 500 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 0.276 | 0.285 | 0.036 | 0.037 | +| sqlite3 | 0.951 | 1.082 | 0.951 | 1.082 | +| sqlite_async | 0.848 | 0.913 | 0.085 | 0.101 | +| drift | 1.341 | 1.462 | 0.082 | 0.089 | + +### 1000 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 0.529 | 0.607 | 0.071 | 0.076 | +| sqlite3 | 1.941 | 2.011 | 1.941 | 2.011 | +| sqlite_async | 1.609 | 1.703 | 0.165 | 0.182 | +| drift | 2.638 | 3.085 | 0.165 | 0.168 | + +### 2000 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 1.099 | 1.290 | 0.140 | 0.144 | +| sqlite3 | 3.858 | 4.871 | 3.858 | 4.871 | +| sqlite_async | 3.263 | 5.224 | 0.331 | 0.415 | +| drift | 5.324 | 5.912 | 0.323 | 0.921 | + +### 5000 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 3.361 | 10.958 | 0.360 | 2.112 | +| sqlite3 | 9.805 | 12.605 | 9.805 | 12.605 | +| sqlite_async | 8.480 | 9.558 | 0.813 | 0.915 | +| drift | 14.177 | 14.529 | 0.817 | 0.889 | + +### 10000 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 8.588 | 16.947 | 0.737 | 1.592 | +| sqlite3 | 23.166 | 30.116 | 23.166 | 30.116 | +| sqlite_async | 16.345 | 17.844 | 1.540 | 1.586 | +| drift | 31.663 | 45.264 | 1.847 | 3.859 | + +### 20000 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 19.172 | 28.498 | 1.625 | 8.176 | +| sqlite3 | 54.991 | 58.473 | 54.991 | 58.473 | +| sqlite_async | 63.204 | 85.638 | 4.681 | 5.123 | +| drift | 85.513 | 99.975 | 4.180 | 4.897 | + + +### Bytes (selectBytes → JSON) + +### 10 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite + jsonEncode | 0.040 | 0.084 | 0.040 | 0.084 | +| sqlite3 + jsonEncode | 0.044 | 0.056 | 0.044 | 0.056 | +| sqlite_async + jsonEncode | 0.068 | 0.077 | 0.068 | 0.077 | +| drift + jsonEncode | 0.080 | 0.082 | 0.080 | 0.082 | +| resqlite selectBytes() | 0.021 | 0.024 | 0.021 | 0.024 | + +### 50 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite + jsonEncode | 0.131 | 0.141 | 0.131 | 0.141 | +| sqlite3 + jsonEncode | 0.182 | 0.188 | 0.182 | 0.188 | +| sqlite_async + jsonEncode | 0.193 | 0.197 | 0.193 | 0.197 | +| drift + jsonEncode | 0.255 | 0.272 | 0.255 | 0.272 | +| resqlite selectBytes() | 0.048 | 0.049 | 0.048 | 0.049 | + +### 100 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite + jsonEncode | 0.240 | 0.256 | 0.240 | 0.256 | +| sqlite3 + jsonEncode | 0.359 | 0.395 | 0.359 | 0.395 | +| sqlite_async + jsonEncode | 0.350 | 0.430 | 0.350 | 0.430 | +| drift + jsonEncode | 0.444 | 0.454 | 0.444 | 0.454 | +| resqlite selectBytes() | 0.082 | 0.085 | 0.082 | 0.085 | + +### 500 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite + jsonEncode | 1.125 | 1.139 | 1.125 | 1.139 | +| sqlite3 + jsonEncode | 1.754 | 1.909 | 1.754 | 1.909 | +| sqlite_async + jsonEncode | 1.592 | 1.681 | 1.592 | 1.681 | +| drift + jsonEncode | 2.078 | 2.113 | 2.078 | 2.113 | +| resqlite selectBytes() | 0.342 | 0.347 | 0.342 | 0.347 | + +### 1000 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite + jsonEncode | 2.203 | 2.267 | 2.203 | 2.267 | +| sqlite3 + jsonEncode | 3.762 | 7.176 | 3.762 | 7.176 | +| sqlite_async + jsonEncode | 3.178 | 11.331 | 3.178 | 11.331 | +| drift + jsonEncode | 4.165 | 12.146 | 4.165 | 12.146 | +| resqlite selectBytes() | 0.657 | 0.666 | 0.657 | 0.666 | + +### 2000 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite + jsonEncode | 5.783 | 12.954 | 5.783 | 12.954 | +| sqlite3 + jsonEncode | 8.297 | 16.767 | 8.297 | 16.767 | +| sqlite_async + jsonEncode | 7.246 | 14.480 | 7.246 | 14.480 | +| drift + jsonEncode | 9.698 | 20.192 | 9.698 | 20.192 | +| resqlite selectBytes() | 1.704 | 4.706 | 1.704 | 4.706 | + +### 5000 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite + jsonEncode | 19.581 | 25.753 | 19.581 | 25.753 | +| sqlite3 + jsonEncode | 22.179 | 34.178 | 22.179 | 34.178 | +| sqlite_async + jsonEncode | 23.133 | 34.563 | 23.133 | 34.563 | +| drift + jsonEncode | 28.103 | 42.442 | 28.103 | 42.442 | +| resqlite selectBytes() | 4.220 | 9.309 | 4.220 | 9.309 | + +### 10000 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite + jsonEncode | 32.354 | 43.597 | 32.354 | 43.597 | +| sqlite3 + jsonEncode | 49.143 | 66.132 | 49.143 | 66.132 | +| sqlite_async + jsonEncode | 60.308 | 71.855 | 60.308 | 71.855 | +| drift + jsonEncode | 70.884 | 86.577 | 70.884 | 86.577 | +| resqlite selectBytes() | 8.079 | 13.317 | 8.079 | 13.317 | + +### 20000 rows + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite + jsonEncode | 65.593 | 75.579 | 65.593 | 75.579 | +| sqlite3 + jsonEncode | 98.680 | 108.502 | 98.680 | 108.502 | +| sqlite_async + jsonEncode | 98.032 | 115.728 | 98.032 | 115.728 | +| drift + jsonEncode | 132.373 | 147.909 | 132.373 | 147.909 | +| resqlite selectBytes() | 16.827 | 18.788 | 16.827 | 18.788 | + + +## Concurrent Reads (1000 rows per query) + +Multiple parallel `select()` calls via `Future.wait`. sqlite3 is excluded (synchronous, no concurrency). Each concurrency level runs `N` parallel queries; we report both total wall time and effective per-query latency (total / N). + +### 1× concurrency + +| Library | Wall med (ms) | Wall p90 (ms) | Per-query (ms) | +|---|---|---|---| +| resqlite | 0.43 | 0.48 | 0.43 | +| sqlite_async | 1.37 | 1.50 | 1.37 | +| drift | 2.51 | 2.96 | 2.51 | + +### 2× concurrency + +| Library | Wall med (ms) | Wall p90 (ms) | Per-query (ms) | +|---|---|---|---| +| resqlite | 0.48 | 0.53 | 0.24 | +| sqlite_async | 2.15 | 2.70 | 1.08 | +| drift | 4.58 | 5.10 | 2.29 | + +### 4× concurrency + +| Library | Wall med (ms) | Wall p90 (ms) | Per-query (ms) | +|---|---|---|---| +| resqlite | 0.52 | 1.09 | 0.13 | +| sqlite_async | 3.52 | 4.47 | 0.88 | +| drift | 8.65 | 9.24 | 2.16 | + +### 8× concurrency + +| Library | Wall med (ms) | Wall p90 (ms) | Per-query (ms) | +|---|---|---|---| +| resqlite | 0.98 | 1.78 | 0.12 | +| sqlite_async | 6.92 | 7.79 | 0.86 | +| drift | 17.98 | 18.42 | 2.25 | + +## Point Query Throughput + +Single-row lookup by primary key in a hot loop. Measures the per-query dispatch overhead. Each sample runs the same adaptive number of 500-query batches, chosen after warmup so that 15 samples target about 1000 ms of total measurement per library after warmup. 95% CI and MDE values derive from per-sample QPS via percentile bootstrap (deterministic, seed=202440478). + +Adaptive schedule: `15 samples, target 1000 ms total` (batch count chosen per library after warmup). + +| Metric | Value | +|---|---:| +| resqlite qps | 70059 | +| resqlite per query | 0.014 ms | + +### QPS + MDE + +| Library | QPS median | 95% CI | MDE_ci % | MDE_mad % | +|---|---:|---:|---:|---:| +| resqlite | 70059 | 66213..70397 | 3.0 | 9.1 | +| sqlite3 | 96271 | 90377..98171 | 4.0 | 8.2 | +| sqlite_async | 24757 | 22177..26616 | 9.0 | 22.5 | +| drift | 17604 | 15271..19735 | 12.7 | 39.6 | + +## Parameterized Queries + +Same `SELECT WHERE category = ?` query run 100 times with different parameter values. Table has 5000 rows with an index on `category` (~500 rows per category). + +### 100 queries × ~500 rows each + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 21.398 | 26.672 | 21.398 | 26.672 | +| sqlite_async | 57.318 | 65.288 | 57.318 | 65.288 | +| drift | 100.867 | 151.538 | 100.867 | 151.538 | +| sqlite3 (no cache) | 50.808 | 79.109 | 50.808 | 79.109 | +| sqlite3 (cached stmt) | 44.103 | 50.156 | 44.103 | 50.156 | + +## Write Performance + +### Single Inserts (100 sequential) + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite execute() | 2.635 | 3.252 | 2.635 | 3.252 | +| sqlite3 execute() | 1.702 | 4.769 | 1.702 | 4.769 | +| sqlite_async execute() | 5.816 | 7.161 | 5.816 | 7.161 | +| drift execute() | 4.995 | 5.987 | 4.995 | 5.987 | + +### Batch Insert (100 rows) + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite executeBatch() | 0.098 | 0.106 | 0.098 | 0.106 | +| sqlite3 executeBatch() | 0.088 | 0.097 | 0.088 | 0.097 | +| sqlite_async executeBatch() | 0.184 | 0.313 | 0.184 | 0.313 | +| drift executeBatch() | 0.250 | 0.336 | 0.250 | 0.336 | + +### Batch Insert (1000 rows) + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite executeBatch() | 0.610 | 0.910 | 0.610 | 0.910 | +| sqlite3 executeBatch() | 0.802 | 0.979 | 0.802 | 0.979 | +| sqlite_async executeBatch() | 0.921 | 4.477 | 0.921 | 4.477 | +| drift executeBatch() | 1.264 | 1.893 | 1.264 | 1.893 | + +### Batch Insert (10000 rows) + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite executeBatch() | 5.746 | 6.686 | 5.746 | 6.686 | +| sqlite3 executeBatch() | 7.705 | 8.272 | 7.705 | 8.272 | +| sqlite_async executeBatch() | 7.885 | 8.676 | 7.885 | 8.676 | +| drift executeBatch() | 10.689 | 13.943 | 10.689 | 13.943 | + +### Wide Batch Insert (10000 rows x 20 params) + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite executeBatch() | 19.178 | 44.984 | 19.178 | 44.984 | +| sqlite3 executeBatch() | 39.718 | 42.194 | 39.718 | 42.194 | +| sqlite_async executeBatch() | 43.805 | 63.616 | 43.805 | 63.616 | +| drift executeBatch() | 48.958 | 62.041 | 48.958 | 62.041 | + +### Interactive Transaction (insert + select + conditional delete) + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite transaction() | 0.086 | 0.178 | 0.086 | 0.178 | +| sqlite_async writeTransaction() | 0.143 | 0.195 | 0.143 | 0.195 | + +### Batched Write Inside Transaction (100 rows) + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite tx.executeBatch() | 0.150 | 0.175 | 0.150 | 0.175 | +| resqlite tx.execute() loop | 1.401 | 1.901 | 1.401 | 1.901 | +| sqlite_async tx.execute() loop | 2.615 | 3.394 | 2.615 | 3.394 | + +### Batched Write Inside Transaction (1000 rows) + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite tx.executeBatch() | 0.621 | 0.710 | 0.621 | 0.710 | +| resqlite tx.execute() loop | 13.556 | 17.375 | 13.556 | 17.375 | +| sqlite_async tx.execute() loop | 23.020 | 25.111 | 23.020 | 25.111 | + +### Transaction Read (500 rows) + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite tx.select() | 0.159 | 0.167 | 0.159 | 0.167 | +| sqlite_async tx.getAll() | 0.352 | 0.376 | 0.352 | 0.376 | + +### Transaction Read (1000 rows) + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite tx.select() | 0.273 | 0.338 | 0.273 | 0.338 | +| sqlite_async tx.getAll() | 0.569 | 0.699 | 0.569 | 0.699 | + +### Nested Transactions (savepoints) + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite nested transaction() x50 | 2.941 | 3.509 | 2.941 | 3.509 | +| resqlite nested transaction() depth=5 | 0.188 | 0.218 | 0.188 | 0.218 | + +## Streaming + +Reactive query performance. resqlite uses per-subscriber buffered controllers with authorizer-based dependency tracking. sqlite_async uses a 30ms default throttle (disabled here via throttle: Duration.zero). + +### Initial Emission + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite stream() | 0.054 | 0.091 | 0.054 | 0.091 | +| sqlite_async watch() | 0.200 | 0.326 | 0.200 | 0.326 | + +### Invalidation Latency + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 0.127 | 0.259 | 0.127 | 0.259 | +| sqlite_async | 0.142 | 0.713 | 0.142 | 0.713 | + +### Unchanged Fanout Throughput (1 canary + 10 unchanged streams) + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 0.325 | 0.389 | 0.325 | 0.389 | +| sqlite_async | 0.767 | 2.168 | 0.767 | 2.168 | + +### Long-Text Unchanged Fanout (8 unchanged streams, 256 rows x 4KB TEXT) + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 0.135 | 0.206 | 0.135 | 0.206 | + +### Fan-out (10 streams) + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 0.310 | 0.406 | 0.310 | 0.406 | +| sqlite_async | 0.360 | 0.478 | 0.360 | 0.478 | + +### Stream Churn (100 cycles) + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 3.433 | 3.433 | 3.433 | 3.433 | +| sqlite_async | 16.332 | 16.332 | 16.332 | 16.332 | + + +### No-Streams Write Throughput (200 inserts, no active streams) + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 5.976 | 6.640 | 5.976 | 6.640 | +| sqlite_async | 10.428 | 11.678 | 10.428 | 11.678 | + + +### Growing-Stream Invalidation (batch-insert 100 into watched stream) + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 0.173 | 0.193 | 0.173 | 0.193 | + + +### Stream Subscription Rate (500 subscribe+cancel cycles) + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 14.421 | 16.022 | 14.421 | 16.022 | + + +## Streaming (Column Granularity) + +10 concurrent streams read `SELECT id, a, b FROM wide ...`. The writer issues 500 updates — first against a **disjoint** column (`c`, not in the projection), then against an **overlapping** column (`a`, in the projection). **`Re-emit ratio` = `disjoint / overlapping` is the primary metric**: it shows how effectively the library suppresses re-emission on writes that don't affect the query's result. Absolute counts are coalescing-dependent and not directly comparable across libraries. For resqlite, this emission metric can reflect writer-side column-level invalidation, experiment 075's native result-hash short-circuit, or both. Use A11c (Many-Streams Writer Throughput) when the question is specifically writer-side dispatch elision. + +### Disjoint column writes (SET c = ?) + +| Library | Re-emits (total) | Wall drain (ms) | Re-emit ratio | +|---|---|---|---| +| resqlite | 0 | 79.5 | 0.000 | +| sqlite_async | 3825 | 1560.8 | 0.951 | +| drift | 5000 | 1594.0 | 1.000 | + +### Overlapping column writes (SET a = ?) + +| Library | Re-emits (total) | Wall drain (ms) | Re-emit ratio | +|---|---|---|---| +| resqlite | 5000 | 88.4 | 0.000 | +| sqlite_async | 4021 | 1556.4 | 0.951 | +| drift | 5000 | 1566.8 | 1.000 | + +## Keyed PK Subscriptions (v1) + +50 reactive streams each watch one PK. 200 random-PK writes across a 10K-row table. The committed PRNG seed produces 3 hits on watched PKs, so both miss-path and hit-path are exercised each run. With keyed invalidation, a library fires only on those hits. With table-level invalidation, every write triggers a re-query on all 50 streams (10K re-queries, most suppressed by hash but still costly). + +### 50 streams × 200 random-PK writes + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | Total emits | Observed hits | +|---|---|---|---|---|---|---| +| resqlite stream() | 212.08 | 212.10 | 0.00 | 0.00 | 0 | 3 | +| sqlite_async stream() | 445.65 | 446.84 | 0.00 | 0.01 | 1195 | 3 | +| drift stream() | 659.43 | 664.34 | 0.06 | 0.08 | 10000 | 3 | + +**Total emits**: post-baseline emissions summed across all 50 streams. **Observed hits**: how many of the 200 random writes actually targeted a watched PK. Perfect behavior: emissions == hits. Emissions < hits means hash suppression elided some writes whose row value did not change. Emissions > hits means over-fire. + +Wall time is dominated by re-query work. A library with keyed-PK invalidation (Track D's planned `watchRow()`) can avoid re-querying for writes whose PK is unwatched, reducing wall time substantially even when emission counts already look clean due to hash suppression. + +## Chat Sim (v1) + +Mixed R/W workload: 500 users, 100 conversations, 10K seed messages (Zipfian distribution). 10K ops: 5% message inserts, 5% conversation last_msg_at updates, 45% fetch-last-20 with user JOIN, 45% fetch-user-by-PK. Measures each op type separately so per-library wall/main tradeoffs are legible. + +### Insert message + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 0.049 | 0.093 | 0.000 | 0.000 | +| sqlite3 | 0.026 | 0.048 | 0.026 | 0.048 | +| sqlite_async | 0.088 | 0.145 | 0.000 | 0.000 | +| drift | 0.079 | 0.131 | 0.000 | 0.000 | + +### Update conversation + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 0.039 | 0.080 | 0.000 | 0.000 | +| sqlite3 | 0.018 | 0.027 | 0.018 | 0.027 | +| sqlite_async | 0.073 | 0.125 | 0.000 | 0.000 | +| drift | 0.067 | 0.117 | 0.000 | 0.000 | + +### Fetch last-20 messages (JOIN users) + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 0.050 | 0.085 | 0.000 | 0.000 | +| sqlite3 | 0.051 | 0.057 | 0.051 | 0.057 | +| sqlite_async | 0.102 | 0.158 | 0.001 | 0.001 | +| drift | 0.097 | 0.127 | 0.001 | 0.001 | + +### Fetch user by PK + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 0.019 | 0.049 | 0.000 | 0.000 | +| sqlite3 | 0.010 | 0.012 | 0.010 | 0.012 | +| sqlite_async | 0.043 | 0.075 | 0.000 | 0.000 | +| drift | 0.042 | 0.064 | 0.000 | 0.000 | + +**Interpretation.** Each op type is timed independently. A library that dominates on one op type (e.g. reads) may lose on another (e.g. inserts under commit pressure). For Flutter-facing usage, the `Main med` column is the key number: it's the time spent on the UI thread per op. + +## Feed Paging (v1) + +100K posts. Part A: 20 keyset-paged queries of 50 posts each, all three peers. Part B: one reactive stream on latest-50 with 100 concurrent like_count writes, resqlite + sqlite_async. Models an infinite-scroll feed with live updates. + +### Keyset pagination (20 pages × 50 rows) + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | +|---|---|---|---|---| +| resqlite | 0.071 | 0.094 | 0.001 | 0.002 | +| sqlite3 | 0.112 | 0.116 | 0.112 | 0.116 | +| sqlite_async | 0.132 | 0.148 | 0.002 | 0.002 | +| drift | 0.194 | 0.253 | 0.002 | 0.003 | + +Keyset pagination walks backwards through the feed via `(created_at, id) < (?, ?)` rather than `OFFSET`, which scales with position rather than degrading on deep pages. Per-page timing is reported; reading the p90 catches occasional slow pages that would be invisible in a wall-aggregate. + +### Reactive feed with 100 concurrent writes + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | Emissions | +|---|---|---|---|---|---| +| resqlite | 106.908 | 107.198 | 0.000 | 0.000 | 0 | +| sqlite_async | 212.821 | 213.096 | 0.000 | 0.000 | 40 | +| drift | 225.667 | 226.407 | 0.000 | 0.000 | 100 | + +One stream on latest-50. 100 `like_count` writes against random posts — most do not intersect the watched page. `Main med` is aggregate listener-callback time (UI thread cost, see METHODOLOGY.md § Measurement). `Emissions` is post-baseline; a library with hash-based unchanged suppression can stay near 0 when the watched page does not change. + +## High-Cardinality Stream Fan-out (v1) + +100 reactive streams each watching one of 100 owner partitions of a 10K-item table. 200 random-item writes target random items. Models Flutter list views with many simultaneous row watchers (detail screens, reactive timelines). Originally exposed a write-burst pool-saturation pathology; that was fixed in PR #17 by adding per-stream re-query coalescing in the stream engine. This benchmark remains as its regression guard. + +### 100 streams × 200 writes + +| Library | Wall med (ms) | Wall p90 (ms) | Main med (ms) | Main p90 (ms) | Init drain (ms) | Write burst (ms) | Emissions | +|---|---|---|---|---|---|---|---| +| resqlite | 230.33 | 230.33 | 0.00 | 0.00 | 12.38 | 218.00 | 2 | +| sqlite_async | 500.46 | 500.46 | 0.00 | 0.00 | 23.76 | 476.70 | 1183 | +| drift | 3077.56 | 3077.56 | 2.53 | 2.53 | 17.91 | 3059.56 | 20000 | + +**Init drain**: median wall time from subscribing all 100 streams to the last one producing its initial emission. Exposes cold-start cost of the subscriber fleet. + +**Write burst**: median wall time from first write to last emission settled after 200 writes. Dominated by re-query cost × stream count × write count for libraries without per-row invalidation; hash suppression (resqlite exp 031/033) elides emissions but the re-query itself still runs. + +**Wall / Main** columns are end-to-end (init + writes + settle). `Main` is aggregate listener-callback time — the UI thread cost. + +## Memory + +Process RSS delta around each workload. Values are a **lower bound** on real allocation volume because the Dart VM retains heap pages after GC. A visible reduction here implies the underlying allocation win is at least that large. + +### Select 10k rows → Maps + +| Library | RSS delta med (MB) | RSS delta p90 (MB) | 95% CI (MB) | MDE (MB) | +|---|---|---|---|---| +| resqlite select() | 6.14 | 24.32 | 3.03..9.67 | ±3.32 | +| sqlite3 select() | 1.98 | 9.56 | 0.00..4.53 | ±2.27 | +| sqlite_async select() | 1.00 | 1.66 | 0.96..1.00 | ±0.02 | +| drift select() | 5.25 | 19.36 | 0.59..12.28 | ±5.85 | + +### Select 10k rows → JSON Bytes + +| Library | RSS delta med (MB) | RSS delta p90 (MB) | 95% CI (MB) | MDE (MB) | +|---|---|---|---|---| +| resqlite selectBytes() | 0.24 | 8.97 | 0.00..1.16 | ±0.58 | +| resqlite + jsonEncode | 7.83 | 67.64 | 0.00..47.54 | ±23.77 | +| sqlite3 + jsonEncode | 7.68 | 58.14 | 0.00..14.66 | ±7.33 | +| sqlite_async + jsonEncode | 2.38 | 26.09 | 0.00..10.20 | ±5.10 | +| drift + jsonEncode | 0.00 | 31.68 | 0.00..1.94 | ±0.97 | + +### Batch insert 10k rows + +| Library | RSS delta med (MB) | RSS delta p90 (MB) | 95% CI (MB) | MDE (MB) | +|---|---|---|---|---| +| resqlite executeBatch() | 6.73 | 12.64 | 0.00..9.56 | ±4.78 | +| sqlite3 executeBatch() | 0.00 | 0.62 | 0.00..0.00 | ±0.00 | +| sqlite_async executeBatch() | 0.29 | 4.98 | 0.00..0.54 | ±0.27 | +| drift batch() | 0.01 | 2.00 | 0.00..0.11 | ±0.06 | + +### Streaming fan-out (10 streams × 100 writes) + +| Library | RSS delta med (MB) | RSS delta p90 (MB) | 95% CI (MB) | MDE (MB) | +|---|---|---|---|---| +| resqlite stream() | 0.00 | 0.13 | 0.00..0.12 | ±0.06 | +| sqlite_async watch() | 0.00 | 0.91 | 0.00..0.03 | ±0.02 | + +## SQLite Diagnostics + +resqlite-only internal SQLite counters captured via `Database.diagnostics()` after representative workloads. Values reflect the writer plus idle readers in this connection pool; they are not process-global SQLite totals. + +### Warm read working set (20000 rows + 2000 point lookups) + +| Library | SQLite total (KiB) | Page cache (KiB) | Schema (KiB) | Stmt (KiB) | WAL (KiB) | Readers busy | +|---|---|---|---|---|---|---| +| resqlite | 3210.4 | 3189.5 | 5.3 | 15.6 | 2048.0 | 0 | + +### Statement cache footprint (48 distinct SELECT texts) + +| Library | SQLite total (KiB) | Page cache (KiB) | Schema (KiB) | Stmt (KiB) | WAL (KiB) | Readers busy | +|---|---|---|---|---|---|---| +| resqlite | 3299.6 | 3189.5 | 5.3 | 104.8 | 2048.0 | 0 | + +### WAL after write burst (1000 inserted rows) + +| Library | SQLite total (KiB) | Page cache (KiB) | Schema (KiB) | Stmt (KiB) | WAL (KiB) | Readers busy | +|---|---|---|---|---|---|---| +| resqlite | 260.9 | 240.0 | 5.3 | 15.6 | 161.0 | 0 | + +## Repeat Stability + +These rows summarize resqlite wall medians across repeated full-suite runs. +Use this section to judge whether small deltas are real or just noise. + +| Benchmark | Median (ms) | 95% CI (ms) | MDE_ci | Range | MAD | Stability | +|---|---|---|---|---|---|---| +| Chat Sim (v1) / Fetch last-20 messages (JOIN users) / resqlite | 0.05 | 0.04..0.05 | 8.7% | 17.4% | 2.2% | stable | +| Chat Sim (v1) / Fetch last-20 messages (JOIN users) / resqlite [main] | 0.00 | 0.00..0.00 | 0.0% | 0.0% | 0.0% | stable | +| Chat Sim (v1) / Fetch user by PK / resqlite | 0.02 | 0.02..0.02 | 11.1% | 22.2% | 5.6% | moderate | +| Chat Sim (v1) / Fetch user by PK / resqlite [main] | 0.00 | 0.00..0.00 | 0.0% | 0.0% | 0.0% | stable | +| Chat Sim (v1) / Insert message / resqlite | 0.04 | 0.04..0.06 | 20.5% | 40.9% | 9.1% | noisy | +| Chat Sim (v1) / Insert message / resqlite [main] | 0.00 | 0.00..0.00 | 0.0% | 0.0% | 0.0% | stable | +| Chat Sim (v1) / Update conversation / resqlite | 0.04 | 0.03..0.05 | 16.7% | 33.3% | 5.6% | moderate | +| Chat Sim (v1) / Update conversation / resqlite [main] | 0.00 | 0.00..0.00 | 0.0% | 0.0% | 0.0% | stable | +| Concurrent Reads (1000 rows per query) / 1× concurrency / resqlite | 0.47 | 0.43..0.50 | 7.4% | 14.9% | 6.4% | moderate | +| Concurrent Reads (1000 rows per query) / 1× concurrency / resqlite ... | 0.47 | 0.43..0.50 | 7.4% | 14.9% | 6.4% | moderate | +| Concurrent Reads (1000 rows per query) / 2× concurrency / resqlite | 0.50 | 0.48..0.61 | 13.0% | 26.0% | 2.0% | stable | +| Concurrent Reads (1000 rows per query) / 2× concurrency / resqlite ... | 0.25 | 0.24..0.30 | 12.0% | 24.0% | 4.0% | moderate | +| Concurrent Reads (1000 rows per query) / 4× concurrency / resqlite | 0.54 | 0.52..0.58 | 5.6% | 11.1% | 3.7% | moderate | +| Concurrent Reads (1000 rows per query) / 4× concurrency / resqlite ... | 0.13 | 0.13..0.14 | 3.8% | 7.7% | 0.0% | stable | +| Concurrent Reads (1000 rows per query) / 8× concurrency / resqlite | 1.11 | 0.98..1.64 | 29.7% | 59.5% | 7.2% | moderate | +| Concurrent Reads (1000 rows per query) / 8× concurrency / resqlite ... | 0.14 | 0.12..0.20 | 28.6% | 57.1% | 7.1% | moderate | +| Feed Paging (v1) / Keyset pagination (20 pages × 50 rows) / resqlite | 0.07 | 0.07..0.08 | 5.6% | 11.1% | 1.4% | stable | +| Feed Paging (v1) / Keyset pagination (20 pages × 50 rows) / resqlit... | 0.00 | 0.00..0.01 | 100.0% | 200.0% | 0.0% | stable | +| Feed Paging (v1) / Reactive feed with 100 concurrent writes / resqlite | 106.63 | 106.30..110.16 | 1.8% | 3.6% | 0.3% | stable | +| Feed Paging (v1) / Reactive feed with 100 concurrent writes / resql... | 0.00 | 0.00..0.00 | 0.0% | 0.0% | 0.0% | stable | +| High-Cardinality Stream Fan-out (v1) / 100 streams × 200 writes / r... | 229.17 | 228.00..237.51 | 2.1% | 4.1% | 0.5% | stable | +| High-Cardinality Stream Fan-out (v1) / 100 streams × 200 writes / r... | 0.00 | 0.00..0.01 | 0.0% | 0.0% | 0.0% | stable | +| Keyed PK Subscriptions (v1) / 50 streams × 200 random-PK writes / r... | 211.92 | 210.21..214.34 | 1.0% | 1.9% | 0.6% | stable | +| Keyed PK Subscriptions (v1) / 50 streams × 200 random-PK writes / r... | 0.00 | 0.00..0.00 | 0.0% | 0.0% | 0.0% | stable | +| Parameterized Queries / 100 queries × ~500 rows each / resqlite | 20.98 | 20.69..21.40 | 1.7% | 3.4% | 1.4% | stable | +| Parameterized Queries / 100 queries × ~500 rows each / resqlite [main] | 20.98 | 20.69..21.40 | 1.7% | 3.4% | 1.4% | stable | +| Point Query Throughput / resqlite qps | 64561.00 | 58030.00..70059.00 | 9.3% | 18.6% | 2.3% | stable | +| Scaling (10 → 20,000 rows) / 10 rows / resqlite | 0.02 | 0.02..0.03 | 16.7% | 33.3% | 12.5% | noisy | +| Scaling (10 → 20,000 rows) / 10 rows / resqlite + jsonEncode | 0.04 | 0.04..0.05 | 13.6% | 27.3% | 2.3% | stable | +| Scaling (10 → 20,000 rows) / 10 rows / resqlite + jsonEncode [main] | 0.04 | 0.04..0.05 | 13.6% | 27.3% | 2.3% | stable | +| Scaling (10 → 20,000 rows) / 10 rows / resqlite [main] | 0.00 | 0.00..0.00 | 50.0% | 100.0% | 0.0% | stable | +| Scaling (10 → 20,000 rows) / 10 rows / resqlite selectBytes() | 0.03 | 0.02..0.04 | 26.8% | 53.6% | 7.1% | moderate | +| Scaling (10 → 20,000 rows) / 10 rows / resqlite selectBytes() [main] | 0.03 | 0.02..0.04 | 26.8% | 53.6% | 7.1% | moderate | +| Scaling (10 → 20,000 rows) / 100 rows / resqlite | 0.07 | 0.07..0.08 | 8.1% | 16.2% | 4.1% | moderate | +| Scaling (10 → 20,000 rows) / 100 rows / resqlite + jsonEncode | 0.26 | 0.24..0.27 | 4.9% | 9.8% | 1.2% | stable | +| Scaling (10 → 20,000 rows) / 100 rows / resqlite + jsonEncode [main] | 0.26 | 0.24..0.27 | 4.9% | 9.8% | 1.2% | stable | +| Scaling (10 → 20,000 rows) / 100 rows / resqlite [main] | 0.01 | 0.01..0.01 | 7.1% | 14.3% | 0.0% | stable | +| Scaling (10 → 20,000 rows) / 100 rows / resqlite selectBytes() | 0.09 | 0.08..0.10 | 8.8% | 17.6% | 3.5% | moderate | +| Scaling (10 → 20,000 rows) / 100 rows / resqlite selectBytes() [main] | 0.09 | 0.08..0.10 | 8.8% | 17.6% | 3.5% | moderate | +| Scaling (10 → 20,000 rows) / 1000 rows / resqlite | 0.53 | 0.51..0.55 | 4.2% | 8.3% | 3.2% | moderate | +| Scaling (10 → 20,000 rows) / 1000 rows / resqlite + jsonEncode | 2.52 | 2.20..2.55 | 6.9% | 13.9% | 1.0% | stable | +| Scaling (10 → 20,000 rows) / 1000 rows / resqlite + jsonEncode [main] | 2.52 | 2.20..2.55 | 6.9% | 13.9% | 1.0% | stable | +| Scaling (10 → 20,000 rows) / 1000 rows / resqlite [main] | 0.07 | 0.07..0.07 | 2.1% | 4.2% | 0.0% | stable | +| Scaling (10 → 20,000 rows) / 1000 rows / resqlite selectBytes() | 0.70 | 0.66..0.78 | 8.5% | 17.0% | 2.4% | stable | +| Scaling (10 → 20,000 rows) / 1000 rows / resqlite selectBytes() [main] | 0.70 | 0.66..0.78 | 8.5% | 17.0% | 2.4% | stable | +| Scaling (10 → 20,000 rows) / 10000 rows / resqlite | 9.21 | 7.62..10.66 | 16.5% | 33.0% | 11.1% | noisy | +| Scaling (10 → 20,000 rows) / 10000 rows / resqlite + jsonEncode | 32.35 | 29.65..50.48 | 32.2% | 64.4% | 8.3% | noisy | +| Scaling (10 → 20,000 rows) / 10000 rows / resqlite + jsonEncode [main] | 32.35 | 29.65..50.48 | 32.2% | 64.4% | 8.3% | noisy | +| Scaling (10 → 20,000 rows) / 10000 rows / resqlite [main] | 0.81 | 0.73..1.00 | 16.6% | 33.2% | 8.7% | noisy | +| Scaling (10 → 20,000 rows) / 10000 rows / resqlite selectBytes() | 7.36 | 6.94..8.08 | 7.7% | 15.5% | 5.6% | moderate | +| Scaling (10 → 20,000 rows) / 10000 rows / resqlite selectBytes() [m... | 7.36 | 6.94..8.08 | 7.7% | 15.5% | 5.6% | moderate | +| Scaling (10 → 20,000 rows) / 2000 rows / resqlite | 1.18 | 1.10..1.29 | 8.0% | 16.0% | 2.2% | stable | +| Scaling (10 → 20,000 rows) / 2000 rows / resqlite + jsonEncode | 6.50 | 5.78..10.81 | 38.7% | 77.4% | 4.9% | moderate | +| Scaling (10 → 20,000 rows) / 2000 rows / resqlite + jsonEncode [main] | 6.50 | 5.78..10.81 | 38.7% | 77.4% | 4.9% | moderate | +| Scaling (10 → 20,000 rows) / 2000 rows / resqlite [main] | 0.14 | 0.14..0.15 | 4.5% | 9.1% | 0.7% | stable | +| Scaling (10 → 20,000 rows) / 2000 rows / resqlite selectBytes() | 1.68 | 1.60..1.76 | 4.8% | 9.6% | 2.7% | stable | +| Scaling (10 → 20,000 rows) / 2000 rows / resqlite selectBytes() [main] | 1.68 | 1.60..1.76 | 4.8% | 9.6% | 2.7% | stable | +| Scaling (10 → 20,000 rows) / 20000 rows / resqlite | 20.90 | 19.17..25.45 | 15.0% | 30.0% | 4.0% | moderate | +| Scaling (10 → 20,000 rows) / 20000 rows / resqlite + jsonEncode | 68.09 | 65.59..72.92 | 5.4% | 10.8% | 1.2% | stable | +| Scaling (10 → 20,000 rows) / 20000 rows / resqlite + jsonEncode [main] | 68.09 | 65.59..72.92 | 5.4% | 10.8% | 1.2% | stable | +| Scaling (10 → 20,000 rows) / 20000 rows / resqlite [main] | 1.77 | 1.63..2.00 | 10.5% | 21.1% | 4.5% | moderate | +| Scaling (10 → 20,000 rows) / 20000 rows / resqlite selectBytes() | 16.83 | 13.88..18.25 | 13.0% | 25.9% | 8.4% | noisy | +| Scaling (10 → 20,000 rows) / 20000 rows / resqlite selectBytes() [m... | 16.83 | 13.88..18.25 | 13.0% | 25.9% | 8.4% | noisy | +| Scaling (10 → 20,000 rows) / 50 rows / resqlite | 0.04 | 0.04..0.05 | 8.9% | 17.8% | 6.7% | moderate | +| Scaling (10 → 20,000 rows) / 50 rows / resqlite + jsonEncode | 0.14 | 0.13..0.14 | 4.2% | 8.5% | 0.7% | stable | +| Scaling (10 → 20,000 rows) / 50 rows / resqlite + jsonEncode [main] | 0.14 | 0.13..0.14 | 4.2% | 8.5% | 0.7% | stable | +| Scaling (10 → 20,000 rows) / 50 rows / resqlite [main] | 0.00 | 0.00..0.00 | 0.0% | 0.0% | 0.0% | stable | +| Scaling (10 → 20,000 rows) / 50 rows / resqlite selectBytes() | 0.05 | 0.05..0.07 | 18.5% | 37.0% | 11.1% | noisy | +| Scaling (10 → 20,000 rows) / 50 rows / resqlite selectBytes() [main] | 0.05 | 0.05..0.07 | 18.5% | 37.0% | 11.1% | noisy | +| Scaling (10 → 20,000 rows) / 500 rows / resqlite | 0.28 | 0.27..0.28 | 2.5% | 5.1% | 1.8% | stable | +| Scaling (10 → 20,000 rows) / 500 rows / resqlite + jsonEncode | 1.19 | 1.13..1.30 | 7.2% | 14.4% | 1.7% | stable | +| Scaling (10 → 20,000 rows) / 500 rows / resqlite + jsonEncode [main] | 1.19 | 1.13..1.30 | 7.2% | 14.4% | 1.7% | stable | +| Scaling (10 → 20,000 rows) / 500 rows / resqlite [main] | 0.04 | 0.04..0.04 | 1.4% | 2.8% | 0.0% | stable | +| Scaling (10 → 20,000 rows) / 500 rows / resqlite selectBytes() | 0.35 | 0.34..0.38 | 5.3% | 10.5% | 2.6% | stable | +| Scaling (10 → 20,000 rows) / 500 rows / resqlite selectBytes() [main] | 0.35 | 0.34..0.38 | 5.3% | 10.5% | 2.6% | stable | +| Scaling (10 → 20,000 rows) / 5000 rows / resqlite | 3.41 | 3.36..3.59 | 3.3% | 6.7% | 1.5% | stable | +| Scaling (10 → 20,000 rows) / 5000 rows / resqlite + jsonEncode | 20.34 | 19.58..23.48 | 9.6% | 19.2% | 3.7% | moderate | +| Scaling (10 → 20,000 rows) / 5000 rows / resqlite + jsonEncode [main] | 20.34 | 19.58..23.48 | 9.6% | 19.2% | 3.7% | moderate | +| Scaling (10 → 20,000 rows) / 5000 rows / resqlite [main] | 0.37 | 0.36..0.39 | 3.5% | 7.1% | 1.9% | stable | +| Scaling (10 → 20,000 rows) / 5000 rows / resqlite selectBytes() | 4.22 | 3.78..5.03 | 14.8% | 29.5% | 9.4% | noisy | +| Scaling (10 → 20,000 rows) / 5000 rows / resqlite selectBytes() [main] | 4.22 | 3.78..5.03 | 14.8% | 29.5% | 9.4% | noisy | +| Schema Shapes (1000 rows) / Narrow (2 cols: id + int) / resqlite | 0.13 | 0.12..0.19 | 28.3% | 56.7% | 5.5% | moderate | +| Schema Shapes (1000 rows) / Narrow (2 cols: id + int) / resqlite [m... | 0.03 | 0.03..0.09 | 112.5% | 225.0% | 0.0% | stable | +| Schema Shapes (1000 rows) / Nullable (50% NULLs) / resqlite | 0.36 | 0.35..0.39 | 5.7% | 11.4% | 1.9% | stable | +| Schema Shapes (1000 rows) / Nullable (50% NULLs) / resqlite [main] | 0.07 | 0.07..0.07 | 2.2% | 4.3% | 1.4% | stable | +| Schema Shapes (1000 rows) / Numeric-heavy (5 numeric cols) / resqlite | 0.36 | 0.35..0.36 | 2.5% | 5.1% | 2.2% | stable | +| Schema Shapes (1000 rows) / Numeric-heavy (5 numeric cols) / resqli... | 0.07 | 0.07..0.07 | 3.5% | 7.0% | 0.0% | stable | +| Schema Shapes (1000 rows) / Text-heavy (4 long TEXT cols) / resqlite | 0.98 | 0.95..1.01 | 3.1% | 6.2% | 2.7% | stable | +| Schema Shapes (1000 rows) / Text-heavy (4 long TEXT cols) / resqlit... | 0.07 | 0.07..0.08 | 3.4% | 6.8% | 2.7% | stable | +| Schema Shapes (1000 rows) / Wide (20 cols: mixed types) / resqlite | 1.24 | 1.20..1.35 | 6.0% | 12.0% | 2.5% | stable | +| Schema Shapes (1000 rows) / Wide (20 cols: mixed types) / resqlite ... | 0.27 | 0.26..0.27 | 1.3% | 2.6% | 0.7% | stable | +| Select → JSON Bytes / 10 rows / resqlite + jsonEncode | 0.04 | 0.04..0.09 | 65.5% | 131.0% | 7.1% | moderate | +| Select → JSON Bytes / 10 rows / resqlite + jsonEncode [main] | 0.02 | 0.02..0.05 | 80.0% | 160.0% | 5.0% | moderate | +| Select → JSON Bytes / 10 rows / resqlite selectBytes() | 0.03 | 0.02..0.04 | 30.0% | 60.0% | 4.0% | moderate | +| Select → JSON Bytes / 10 rows / resqlite selectBytes() [main] | 0.00 | 0.00..0.00 | 0.0% | 0.0% | 0.0% | stable | +| Select → JSON Bytes / 100 rows / resqlite + jsonEncode | 0.26 | 0.25..0.34 | 15.8% | 31.6% | 0.8% | stable | +| Select → JSON Bytes / 100 rows / resqlite + jsonEncode [main] | 0.19 | 0.19..0.24 | 14.7% | 29.5% | 1.1% | stable | +| Select → JSON Bytes / 100 rows / resqlite selectBytes() | 0.09 | 0.08..0.10 | 8.7% | 17.4% | 1.2% | stable | +| Select → JSON Bytes / 100 rows / resqlite selectBytes() [main] | 0.00 | 0.00..0.00 | 0.0% | 0.0% | 0.0% | stable | +| Select → JSON Bytes / 1000 rows / resqlite + jsonEncode | 2.56 | 2.37..3.37 | 19.4% | 38.8% | 1.4% | stable | +| Select → JSON Bytes / 1000 rows / resqlite + jsonEncode [main] | 1.95 | 1.89..2.52 | 16.1% | 32.2% | 2.3% | stable | +| Select → JSON Bytes / 1000 rows / resqlite selectBytes() | 0.70 | 0.68..0.73 | 3.4% | 6.7% | 2.0% | stable | +| Select → JSON Bytes / 1000 rows / resqlite selectBytes() [main] | 0.00 | 0.00..0.00 | 0.0% | 0.0% | 0.0% | stable | +| Select → JSON Bytes / 10000 rows / resqlite + jsonEncode | 29.67 | 29.55..40.17 | 17.9% | 35.8% | 0.4% | stable | +| Select → JSON Bytes / 10000 rows / resqlite + jsonEncode [main] | 20.78 | 20.65..21.84 | 2.9% | 5.7% | 0.6% | stable | +| Select → JSON Bytes / 10000 rows / resqlite selectBytes() | 7.25 | 6.96..7.76 | 5.5% | 11.0% | 2.7% | stable | +| Select → JSON Bytes / 10000 rows / resqlite selectBytes() [main] | 0.01 | 0.01..0.01 | 25.0% | 50.0% | 12.5% | noisy | +| Select → Maps / 10 rows / resqlite select() | 0.03 | 0.02..0.14 | 232.0% | 464.0% | 8.0% | noisy | +| Select → Maps / 10 rows / resqlite select() [main] | 0.00 | 0.00..0.03 | 1200.0% | 2400.0% | 0.0% | stable | +| Select → Maps / 100 rows / resqlite select() | 0.07 | 0.07..0.20 | 100.7% | 201.5% | 1.5% | stable | +| Select → Maps / 100 rows / resqlite select() [main] | 0.01 | 0.01..0.01 | 42.9% | 85.7% | 14.3% | noisy | +| Select → Maps / 1000 rows / resqlite select() | 0.52 | 0.50..0.58 | 8.1% | 16.2% | 1.5% | stable | +| Select → Maps / 1000 rows / resqlite select() [main] | 0.06 | 0.06..0.08 | 16.4% | 32.8% | 1.6% | stable | +| Select → Maps / 10000 rows / resqlite select() | 8.31 | 7.57..8.96 | 8.4% | 16.8% | 4.8% | moderate | +| Select → Maps / 10000 rows / resqlite select() [main] | 0.68 | 0.65..1.01 | 26.8% | 53.6% | 5.0% | moderate | +| Streaming / Fan-out (10 streams) / resqlite | 0.34 | 0.31..0.43 | 18.4% | 36.8% | 8.8% | noisy | +| Streaming / Fan-out (10 streams) / resqlite [main] | 0.34 | 0.31..0.43 | 18.4% | 36.8% | 8.8% | noisy | +| Streaming / Growing-Stream Invalidation (batch-insert 100 into watc... | 0.19 | 0.17..0.39 | 57.9% | 115.7% | 2.6% | stable | +| Streaming / Growing-Stream Invalidation (batch-insert 100 into watc... | 0.19 | 0.17..0.39 | 57.9% | 115.7% | 2.6% | stable | +| Streaming / Initial Emission / resqlite stream() | 0.06 | 0.05..0.11 | 51.8% | 103.5% | 14.0% | noisy | +| Streaming / Initial Emission / resqlite stream() [main] | 0.06 | 0.05..0.11 | 51.8% | 103.5% | 14.0% | noisy | +| Streaming / Invalidation Latency / resqlite | 0.11 | 0.08..0.14 | 24.8% | 49.5% | 18.7% | noisy | +| Streaming / Invalidation Latency / resqlite [main] | 0.11 | 0.08..0.14 | 24.8% | 49.5% | 18.7% | noisy | +| Streaming / Long-Text Unchanged Fanout (8 unchanged streams, 256 ro... | 0.16 | 0.13..0.16 | 11.5% | 22.9% | 3.8% | moderate | +| Streaming / Long-Text Unchanged Fanout (8 unchanged streams, 256 ro... | 0.16 | 0.13..0.16 | 11.5% | 22.9% | 3.8% | moderate | +| Streaming / No-Streams Write Throughput (200 inserts, no active str... | 6.32 | 5.98..8.70 | 21.5% | 43.1% | 3.3% | moderate | +| Streaming / No-Streams Write Throughput (200 inserts, no active str... | 6.32 | 5.98..8.70 | 21.5% | 43.1% | 3.3% | moderate | +| Streaming / Stream Churn (100 cycles) / resqlite | 4.01 | 3.24..6.48 | 40.5% | 80.9% | 14.6% | noisy | +| Streaming / Stream Churn (100 cycles) / resqlite [main] | 4.01 | 3.24..6.48 | 40.5% | 80.9% | 14.6% | noisy | +| Streaming / Stream Subscription Rate (500 subscribe+cancel cycles) ... | 15.47 | 14.17..19.20 | 16.2% | 32.5% | 6.8% | moderate | +| Streaming / Stream Subscription Rate (500 subscribe+cancel cycles) ... | 15.47 | 14.17..19.20 | 16.2% | 32.5% | 6.8% | moderate | +| Streaming / Unchanged Fanout Throughput (1 canary + 10 unchanged st... | 0.33 | 0.31..0.72 | 63.2% | 126.5% | 4.3% | moderate | +| Streaming / Unchanged Fanout Throughput (1 canary + 10 unchanged st... | 0.33 | 0.31..0.72 | 63.2% | 126.5% | 4.3% | moderate | +| Write Performance / Batch Insert (100 rows) / resqlite executeBatch() | 0.10 | 0.10..0.10 | 2.5% | 5.1% | 1.0% | stable | +| Write Performance / Batch Insert (100 rows) / resqlite executeBatch... | 0.10 | 0.10..0.10 | 2.5% | 5.1% | 1.0% | stable | +| Write Performance / Batch Insert (1000 rows) / resqlite executeBatch() | 0.60 | 0.58..1.07 | 40.0% | 79.9% | 1.2% | stable | +| Write Performance / Batch Insert (1000 rows) / resqlite executeBatc... | 0.60 | 0.58..1.07 | 40.0% | 79.9% | 1.2% | stable | +| Write Performance / Batch Insert (10000 rows) / resqlite executeBat... | 5.79 | 5.45..7.31 | 16.1% | 32.1% | 5.8% | moderate | +| Write Performance / Batch Insert (10000 rows) / resqlite executeBat... | 5.79 | 5.45..7.31 | 16.1% | 32.1% | 5.8% | moderate | +| Write Performance / Batched Write Inside Transaction (100 rows) / r... | 1.43 | 1.24..2.25 | 35.4% | 70.7% | 13.4% | noisy | +| Write Performance / Batched Write Inside Transaction (100 rows) / r... | 1.43 | 1.24..2.25 | 35.4% | 70.7% | 13.4% | noisy | +| Write Performance / Batched Write Inside Transaction (100 rows) / r... | 0.14 | 0.13..0.18 | 18.0% | 36.0% | 7.9% | moderate | +| Write Performance / Batched Write Inside Transaction (100 rows) / r... | 0.14 | 0.13..0.18 | 18.0% | 36.0% | 7.9% | moderate | +| Write Performance / Batched Write Inside Transaction (1000 rows) / ... | 13.56 | 12.46..16.98 | 16.7% | 33.3% | 8.1% | noisy | +| Write Performance / Batched Write Inside Transaction (1000 rows) / ... | 13.56 | 12.46..16.98 | 16.7% | 33.3% | 8.1% | noisy | +| Write Performance / Batched Write Inside Transaction (1000 rows) / ... | 0.65 | 0.62..0.83 | 16.3% | 32.6% | 4.5% | moderate | +| Write Performance / Batched Write Inside Transaction (1000 rows) / ... | 0.65 | 0.62..0.83 | 16.3% | 32.6% | 4.5% | moderate | +| Write Performance / Interactive Transaction (insert + select + cond... | 0.10 | 0.09..0.11 | 11.9% | 23.8% | 8.9% | noisy | +| Write Performance / Interactive Transaction (insert + select + cond... | 0.10 | 0.09..0.11 | 11.9% | 23.8% | 8.9% | noisy | +| Write Performance / Nested Transactions (savepoints) / resqlite nes... | 0.23 | 0.19..0.27 | 17.0% | 33.9% | 13.7% | noisy | +| Write Performance / Nested Transactions (savepoints) / resqlite nes... | 0.23 | 0.19..0.27 | 17.0% | 33.9% | 13.7% | noisy | +| Write Performance / Nested Transactions (savepoints) / resqlite nes... | 2.77 | 2.25..2.94 | 12.4% | 24.8% | 6.1% | moderate | +| Write Performance / Nested Transactions (savepoints) / resqlite nes... | 2.77 | 2.25..2.94 | 12.4% | 24.8% | 6.1% | moderate | +| Write Performance / Single Inserts (100 sequential) / resqlite exec... | 3.12 | 2.63..3.64 | 16.1% | 32.2% | 8.7% | noisy | +| Write Performance / Single Inserts (100 sequential) / resqlite exec... | 3.12 | 2.63..3.64 | 16.1% | 32.2% | 8.7% | noisy | +| Write Performance / Transaction Read (1000 rows) / resqlite tx.sele... | 0.28 | 0.27..0.37 | 18.2% | 36.3% | 2.5% | stable | +| Write Performance / Transaction Read (1000 rows) / resqlite tx.sele... | 0.28 | 0.27..0.37 | 18.2% | 36.3% | 2.5% | stable | +| Write Performance / Transaction Read (500 rows) / resqlite tx.select() | 0.16 | 0.15..0.18 | 7.9% | 15.9% | 3.0% | moderate | +| Write Performance / Transaction Read (500 rows) / resqlite tx.selec... | 0.16 | 0.15..0.18 | 7.9% | 15.9% | 3.0% | moderate | +| Write Performance / Wide Batch Insert (10000 rows x 20 params) / re... | 19.52 | 19.18..25.12 | 15.2% | 30.5% | 1.4% | stable | +| Write Performance / Wide Batch Insert (10000 rows x 20 params) / re... | 19.52 | 19.18..25.12 | 15.2% | 30.5% | 1.4% | stable | + + +## Comparison + +Automatic comparison skipped because `2026-06-09T11-20-00-exp150-nullable-batch-packing.md` was not captured in a compatible environment: +- baseline sidecar is missing environment metadata + +Use `--compare-to=benchmark/results/2026-06-09T11-20-00-exp150-nullable-batch-packing.md` to run an explicit reference comparison anyway. + + diff --git a/experiments/160-stream-delta-ivm.md b/experiments/160-stream-delta-ivm.md new file mode 100644 index 00000000..ffdd2ce1 --- /dev/null +++ b/experiments/160-stream-delta-ivm.md @@ -0,0 +1,179 @@ +# Experiment 160: Tier-1 incremental stream maintenance (row deltas) + +**Date:** 2026-06-10 +**Status:** In Review +**Direction:** `stream-rerun-dispatch` + +## Problem + +Every stream invalidation re-executes the full query. Hash suppression +(exp 075/077) saves the decode and the emission, never the execution: a +write that provably cannot affect a stream's result still costs a +reader-pool round-trip (12 µs dispatch floor) plus a full SQLite +re-execution per affected stream. On A11c overlap that is 50 re-queries +per write — 25,000 per 500-write burst — of which exp 136 measured the +completion-handler chain alone at 28.6% of total wall. Exp 134 proved the +win class (keyed-PK writer-burst wall halved when re-queries were elided +by rowid precision) but was rejected because its *write-path SQL text +recognizer* was too fragile to trust. signals.json directed: revive +row-level precision "only through explicit API/design or real workload +evidence", and exp 149 (residual split) concluded the next stream work +needed "a workload-shape or scheduling-model change rather than another +sub-bucket optimization". + +## Hypothesis + +The preupdate hook already observes every modified row's old and new +values — today they are discarded. If the writer ships bounded per-row +deltas alongside the dirty-table set, the stream engine can maintain +admitted streams' materialized results *incrementally*: prove most writes +irrelevant with a few integer comparisons (no reader dispatch at all), +patch in-window changes locally, and fall back to the existing re-query +path for anything unprovable. Unlike exp 134's recognizer, admission is +decided once at stream registration against a sound-by-construction +grammar, and every uncertainty degrades to performance loss, never +correctness loss. No public API changes — `db.stream()` is unchanged. + +## Approach + +**Native capture** (`native/resqlite.c`): the preupdate hook serializes +op + rowids + old/new column values into a bounded buffer (256 rows / 32 +columns / 256 KB per drain cycle; overflow or OOM poisons the cycle). +Savepoint rollback poisons the cycle too — stale deltas are unsafe where +stale dirty-tables are merely conservative. Drained alongside the dirty +sets at statement end / outermost commit; discarded on rollback. + +**Plumbing**: `ExecuteResponse`/`BatchResponse` carry the raw bytes +(`null` = unreliable → fallback); the engine decodes them lazily, once +per write cycle, and only when at least one admitted stream watches a +dirty table. + +**Tier-1 admission** (`lib/src/stream_ivm.dart`): at registration — +asynchronously, off the hot path — the engine fetches +`PRAGMA table_info` (cached per table) and classifies the stream's SQL +against a deliberately tiny grammar: +`SELECT FROM [WHERE col op (int|?) [AND ...]] +[ORDER BY [ASC]]` — comparisons on INTEGER values only, table must +have a single-column INTEGER PRIMARY KEY (rowid alias) included in the +projection, and result order must be fully determined (ORDER BY pk, or a +pk-equality predicate). TEXT comparisons are excluded in tier 1 because +column collations are not mirrored. Anything outside the grammar stays on +the re-query path forever. + +**Maintenance**: per delta row, evaluate the predicate conjunction +against old and new values (NULL cells fail predicates, matching SQL +semantics; non-INTEGER cells in a compared column bail). Proven miss → +nothing. In-window patch / entry / departure → clone-on-write the cached +rows (previously emitted lists are never mutated), emit, and store a hash +sentinel (−1) so the next fallback re-query can never be suppressed +against a pre-patch baseline. Any inconsistency (cache disagrees with a +delta, schema column-count drift, malformed buffer) bails: the cache is +dropped and the entry re-queries exactly as today. + +**Honest divergence from exp 134's rejection**: this is still a SQL +recognizer. The differences that change the calculus: it runs once at +registration (not per write), admits a closed grammar where SQLite +semantics are provably mirrored (INTEGER-only comparisons, BINARY-free), +fails closed (unparsed → fallback; unprovable at apply time → bail), and +is paired with maintained-vs-requery equivalence tests. The risk profile +is "misses an optimization", not "skips a required re-query". + +## Results + +### Engagement check (profile counters, A11c-overlap shape) + +50 streams × 500 writes, every write touching a projected column: +`ivm_skipped=24,500 ivm_applied=500 ivm_bail=0` — zero reader-pool +re-queries for the entire burst +(`benchmark/profile/ivm_engage_check.dart`). + +### Writer wall split audit (exp 147 harness, single pass, back-to-back) + +| workload | baseline wall | IVM wall | residual_us | emissions | +|---|---:|---:|---|---:| +| A11c baseline (0 streams) | 96.5 ms | 75.4 ms | 68,417 → 52,998 | — | +| A11c disjoint | 100.6 ms | 104.2 ms | 55,669 → 62,360 | 0 → 0 | +| A11c overlap | 186.5 ms | 132.0 ms | 135,071 → 47,966 | 44 → 500 | +| keyed PK subscriptions | 46.9 ms | 25.5 ms | 28,340 → 10,287 | 3 → 3 | + +A11c overlap **−29%** wall with **11× more emissions delivered** (500 vs +44): under the baseline, re-query latency causes most per-write changes +to coalesce or hash-suppress; IVM delivers each write's patch +synchronously — the behavior an infinitely-fast re-query would produce. +Keyed-PK **−46%**, reproducing exp 134's archived result through the +explicit classifier. Disjoint is neutral: column elision (exp 106) +already skips those streams before IVM is consulted. `invalidate_us` +rises on overlap (29,974 → 67,340) because patch+emit work now runs +inline in the invalidation pass — more than paid for by the residual +collapse (135,071 → 47,966). + +### Tracelite A/B (stream-rerun-dispatch direction, formal gate) + +**Pass 1 (baseline first, candidate second), 3 runs per side:** + +| scenario | delta | 95% CI | p | per-run medians (baseline → candidate) | +|---|---:|---|---|---| +| many-streams-writer-throughput | **−18.5%** | −122..−96.3 ms | 3.4e-6 | 591.6/583.5/578.6 → 476.9/491.0/480.3 ms | +| keyed-pk-subscriptions | **−14.1%** | −60.0..−27.4 ms | 1.6e-10 | 280.9/301.7/283.7 → 267.3/261.5/261.4 ms | +| high-cardinality-fanout | +2.11% | +2.59..+12.8 ms | 0.009 | 361.5/363.9/366.4 → 379.4/367.4/372.4 ms | + +Both wins are consistent across every run with tight CVs — and the +keyed-PK *within-run CV itself drops* from 0.13–0.15 to 0.02–0.05: +removing reader-pool contention removes its variance. The +high-cardinality +2.11% flag has a CI excluding zero but was collected +with the candidate phase second; per the exp 159 journal lesson, an +order-flipped second pass adjudicates it. + +**Pass 2 (order flipped: exp-160 collected first, main second).** With +the sign inverted to match pass 1's orientation (main slower → win): + +| scenario | main vs exp-160 | 95% CI | per-run medians (exp-160 → main) | +|---|---:|---|---| +| many-streams-writer-throughput | **+22.2% slower** | +100..+113 ms | 477.7/483.8/477.0 → 584.5/582.7/587.5 ms | +| keyed-pk-subscriptions | **+9.4% slower** | +13.0..+37.0 ms | 261.6/265.8/268.4 → 279.3/282.8/279.2 ms | +| high-cardinality-fanout | +0.80% | −3.63..+9.56 ms (straddles zero) | neutral | + +Both wins reproduce with the collection order flipped — exp-160's +absolute medians are essentially identical across passes (many-streams +477–491 ms, keyed-PK 261–268 ms), as are main's (579–592 / 279–302 ms). +The keyed-PK variance reduction follows the code, not the phase: exp-160 +runs at CV 0.01–0.05 in both passes while main runs at CV 0.10–0.15. The +pass-1 high-cardinality +2.11% flag **did not reproduce** (CI straddles +zero, p=0.868) — drift, exactly the exp 159 journal pattern. + +## Decision + +**In Review (accept-shaped).** Two large, twice-reproduced measured- +elapsed wins on the canonical stream gate — many-streams −18.5%/+22.2% +(pass 1/pass 2 orientation), keyed-PK −14.1%/+9.4% — with the third +scenario neutral after order-flipped adjudication, zero bails on every +measured workload, dedicated equivalence tests (26) and the full suite +green, and no public API change. This is the first stream experiment to +remove re-query *execution* rather than tuning its constant: 98% of +A11c-overlap invalidation decisions become proven misses that never +touch the reader pool. + +## Future Notes + +- **Emission cadence**: IVM emits per write where the re-query path + naturally coalesces bursts behind reader latency. Subscribers see more + timely (and more numerous) emissions. If a workload prefers coalescing, + a microtask-batched emit (exp 045 pattern applied to IVM emissions) is + the tunable — measure before adding it. +- **Tier-2 candidates, in evidence order**: TEXT equality under known + BINARY collation (needs `PRAGMA table_info` + collation introspection); + LIMIT windows with a K+buffer cache (unlocks top-K queries; boundary + departures currently can't exist because LIMIT is not admitted); + SQLite-side predicate evaluation against a bound delta row (exact + semantics for arbitrary predicates at O(1) per delta). +- **Capture overhead**: writes pay two bounded row-value serializations + per modified row even with no admitted streams. Batches poison the + buffer at 256 rows and stop paying. If release-suite write benchmarks + ever flag it, gate capture on a writer-side "admitted streams exist" + flag (one control message when the first stream is admitted). +- **Schema changes**: `ALTER TABLE ADD/DROP COLUMN` changes the capture + column count and demotes admitted streams to fallback on their next + delta (correct, self-healing). The cached `PRAGMA table_info` is keyed + per table and cleared on engine close; a same-count column rename + leaves predicates evaluating the same cid positions, which is + positionally correct until the stream's own SQL breaks on re-query. diff --git a/experiments/README.md b/experiments/README.md index 63e60fcf..54a65c11 100644 --- a/experiments/README.md +++ b/experiments/README.md @@ -61,6 +61,7 @@ moved them. | # | Experiment | Impact | PR | |---|---|---|---| +| [160](160-stream-delta-ivm.md) | Tier-1 incremental stream maintenance (row deltas) | **−18.5% many-streams / −14.1% keyed-PK** on the Tracelite stream gate (CIs −122..−96 ms and −60..−27 ms). Preupdate hook captures bounded per-row old/new values; a registration-time classifier admits single-table integer-predicate streams whose results the engine then maintains locally — A11c-overlap engagement: 24,500/25,000 invalidation decisions proven misses with zero reader re-queries, 0 bails. Everything outside the tier-1 grammar stays on the re-query path | | | [159](159-writer-pipelining.md) | Writer request pipelining + persistent reply port | **−36% to −45%** on concurrent standalone writes (new focused burst benchmark) by holding the write lock only across the send and letting the worker's port FIFO pipeline requests; persistent reply port + cached SendPort + sync completers remove a port allocate/register/close cycle and two scheduling hops from every write round-trip. Stream-dispatch Tracelite guardrails neutral on the clean order-flipped pass; exp 147 `residual_us` drops on all four audit workloads | | | [083](083-stream-rerun-pre-dispatch-queue.md) | Stream rerun pre-dispatch queue | Eliminates the measured `A11` / `A11b` reader-pool wait bottleneck by coalescing reruns before pool admission | [#25](https://github.com/danReynolds/resqlite/pull/25) | | [097](097-one-pass-initial-stream-hash.md) | One-pass initial stream decode and hash | 14-16% faster setup-heavy streaming benchmarks by avoiding the initial stream query replay | | diff --git a/experiments/signals.json b/experiments/signals.json index 2135752f..c71a2821 100644 --- a/experiments/signals.json +++ b/experiments/signals.json @@ -647,6 +647,21 @@ "when a tracelite gate flags a regression with elevated within-run CVs relative to the other phase, re-run with collection order flipped before treating the flag as real" ] }, + "160": { + "directions": ["stream-rerun-dispatch"], + "outcomeClass": "in_review_accepted", + "changedBeliefs": [ + "Row-level dirty precision is mergeable when admission is a registration-time, fail-closed classifier instead of exp 134's write-path SQL recognizer: tier-1 (single table, integer predicates, INTEGER-PK ordering) covers the canonical stream workloads and turns 98% of A11c-overlap invalidation decisions into proven misses with zero reader dispatches and zero bails", + "The preupdate hook's old/new row values are capturable within bounded cost (256 rows / 32 cols / 256 KB per cycle, poisoned by savepoint rollback) and ship as raw bytes decoded lazily on the main isolate", + "Maintained streams change emission cadence: each write's patch is delivered synchronously instead of coalescing behind re-query latency (A11c overlap 44 -> 500 emissions during the same burst) — the semantics an infinitely-fast re-query would produce", + "Tracelite stream gate, both collection orders: many-streams -18.5% / +22.2% main-slower (CIs -122..-96 ms and +100..+113 ms), keyed-PK -14.1% / +9.4% (CIs -60..-27 ms and +13..+37 ms) with keyed-PK within-run CV dropping 0.13-0.15 -> 0.02-0.05 under IVM in both passes; the standard-order pass's high-cardinality +2.11% flag did not reproduce order-flipped (CI -3.6..+9.6 ms) — drift, the exp 159 journal pattern" + ], + "nextSignals": [ + "delta capture currently runs unconditionally (bounded); if any write-path benchmark ever flags it, gate capture on a writer-side admitted-streams flag", + "tier-2 candidates in evidence order: TEXT equality under introspected BINARY collation, LIMIT windows with K+buffer caches, SQLite-side predicate evaluation against bound delta rows", + "if subscribers prefer coalesced emissions over per-write patches, apply the exp 045 microtask-batch pattern to IVM emissions and measure" + ] + }, "147": { "directions": ["stream-rerun-dispatch", "measurement-system"], "outcomeClass": "in_review_measurement", diff --git a/lib/src/database.dart b/lib/src/database.dart index 9707a487..e10d8ac0 100644 --- a/lib/src/database.dart +++ b/lib/src/database.dart @@ -405,6 +405,7 @@ final class Database { ProfileCounters.recordWriterSqlite(response.writerSqliteUs); streamEngine.onDependencyChanges( response.modifications, + deltas: response.deltas, traceCorrelationId: correlationId, ); @@ -468,6 +469,7 @@ final class Database { ProfileCounters.recordWriterSqlite(response.writerSqliteUs); streamEngine.onDependencyChanges( response.modifications, + deltas: response.deltas, traceCorrelationId: correlationId, ); } diff --git a/lib/src/native/resqlite_bindings.dart b/lib/src/native/resqlite_bindings.dart index 5aee18fb..a2a3e4b3 100644 --- a/lib/src/native/resqlite_bindings.dart +++ b/lib/src/native/resqlite_bindings.dart @@ -586,6 +586,67 @@ TableDependencies getDirtyTableDependencies(ffi.Pointer dbHandle) { void discardDirtyTableDependencies(ffi.Pointer dbHandle) { resqliteGetDirtyTables(dbHandle, _dirtyTablesBuf, 64); resqliteGetDirtyColumns(dbHandle, _columnTablesBuf, _columnNamesBuf, 64); + resqliteDiscardDeltas(dbHandle); +} + +// --------------------------------------------------------------------------- +// Row delta capture (exp 160) +// --------------------------------------------------------------------------- + +@ffi.Native< + ffi.Int Function( + ffi.Pointer, + ffi.Pointer>, + ffi.Pointer, + ffi.Pointer, + ) +>(symbol: 'resqlite_get_deltas', isLeaf: true) +external int resqliteGetDeltas( + ffi.Pointer db, + ffi.Pointer> outBuf, + ffi.Pointer outLen, + ffi.Pointer outRows, +); + +@ffi.Native)>( + symbol: 'resqlite_discard_deltas', + isLeaf: true, +) +external void resqliteDiscardDeltas(ffi.Pointer db); + +@ffi.Native)>( + symbol: 'resqlite_deltas_mark_unreliable', + isLeaf: true, +) +external void resqliteDeltasMarkUnreliable(ffi.Pointer db); + +/// Persistent out-parameter slots for the delta drain, mirroring the +/// `_dirtyTablesBuf` pattern. +final ffi.Pointer> _deltaBufSlot = + calloc>(); +final ffi.Pointer _deltaLenSlot = calloc(); +final ffi.Pointer _deltaRowsSlot = calloc(); + +final Uint8List _emptyDeltaBytes = Uint8List(0); + +/// Drain accumulated row deltas for the completed write cycle. +/// +/// Returns `null` when capture was unreliable for this cycle (caps +/// exceeded, allocation failure, or a savepoint rollback poisoned the +/// buffer) — callers fall back to plain re-query invalidation. Returns an +/// empty list when the cycle modified no rows. The bytes are copied out of +/// the native buffer, so they remain valid across subsequent writes. +Uint8List? drainRowDeltas(ffi.Pointer dbHandle) { + final reliable = resqliteGetDeltas( + dbHandle, + _deltaBufSlot, + _deltaLenSlot, + _deltaRowsSlot, + ); + if (reliable == 0) return null; + final len = _deltaLenSlot.value; + if (len == 0) return _emptyDeltaBytes; + return Uint8List.fromList(_deltaBufSlot.value.asTypedList(len)); } /// Read a sqlite3_db_status aggregate across the writer and any idle diff --git a/lib/src/profile_counters.dart b/lib/src/profile_counters.dart index 6c14b946..d5fd2822 100644 --- a/lib/src/profile_counters.dart +++ b/lib/src/profile_counters.dart @@ -169,6 +169,15 @@ class ProfileCounters { /// one per initial result emission. static int streamEmitCount = 0; + /// Incremental stream maintenance outcomes (exp 160). One increment per + /// stream entry per write cycle: `ivmSkippedTotal` — every delta was + /// proven irrelevant, no re-query dispatched; `ivmAppliedTotal` — the + /// cached result was patched and emitted locally; `ivmBailTotal` — the + /// entry fell back to the normal re-query path. + static int ivmSkippedTotal = 0; + static int ivmAppliedTotal = 0; + static int ivmBailTotal = 0; + /// Take a named snapshot of all counter values. static Map snapshot() => { 'rows_decoded': rowsDecoded, @@ -186,6 +195,9 @@ class ProfileCounters { 'completion_handler_count': completionHandlerCount, 'stream_emit_us': streamEmitUs, 'stream_emit_count': streamEmitCount, + 'ivm_skipped_total': ivmSkippedTotal, + 'ivm_applied_total': ivmAppliedTotal, + 'ivm_bail_total': ivmBailTotal, }; /// Compute `after - before` for every key present in both snapshots. @@ -220,5 +232,8 @@ class ProfileCounters { completionHandlerCount = 0; streamEmitUs = 0; streamEmitCount = 0; + ivmSkippedTotal = 0; + ivmAppliedTotal = 0; + ivmBailTotal = 0; } } diff --git a/lib/src/row_deltas.dart b/lib/src/row_deltas.dart new file mode 100644 index 00000000..dfa13abc --- /dev/null +++ b/lib/src/row_deltas.dart @@ -0,0 +1,141 @@ +/// Row delta value types and decoder (exp 160). +/// +/// The writer's preupdate hook captures bounded per-row old/new values into +/// a native buffer (see `native/resqlite.h` for the byte layout); the write +/// worker drains it and ships the raw bytes to the main isolate, where the +/// stream engine decodes them lazily — only when at least one stream is +/// admitted for incremental maintenance. +library; + +import 'dart:convert' show utf8; +import 'dart:typed_data'; + +/// SQLite preupdate op codes, as captured by the hook. +const int deltaOpDelete = 9; // SQLITE_DELETE +const int deltaOpInsert = 18; // SQLITE_INSERT +const int deltaOpUpdate = 23; // SQLITE_UPDATE + +/// One modified row: full old/new column values indexed by the table's +/// column order (cid order from `PRAGMA table_info`). +final class RowDelta { + const RowDelta({ + required this.op, + required this.table, + required this.oldRowid, + required this.newRowid, + required this.oldValues, + required this.newValues, + }); + + final int op; + final String table; + final int oldRowid; + final int newRowid; + + /// Pre-write values; `null` for INSERT. + final List? oldValues; + + /// Post-write values; `null` for DELETE. + final List? newValues; +} + +/// Decode a drained delta buffer. +/// +/// Returns `null` on any structural inconsistency — callers treat that +/// exactly like an unreliable capture and fall back to re-query. +List? decodeRowDeltas(Uint8List bytes) { + final data = ByteData.sublistView(bytes); + final deltas = []; + var off = 0; + + bool canRead(int n) => off + n <= bytes.length; + + List? readCells(int colCount) { + final values = List.filled(colCount, null); + for (var i = 0; i < colCount; i++) { + if (!canRead(1)) return null; + final tag = data.getUint8(off); + off += 1; + switch (tag) { + case 0: + values[i] = null; + case 1: + if (!canRead(8)) return null; + values[i] = data.getInt64(off, Endian.little); + off += 8; + case 2: + if (!canRead(8)) return null; + values[i] = data.getFloat64(off, Endian.little); + off += 8; + case 3 || 4: + if (!canRead(4)) return null; + final len = data.getInt32(off, Endian.little); + off += 4; + if (len < 0 || !canRead(len)) return null; + if (tag == 3) { + values[i] = utf8.decode( + Uint8List.sublistView(bytes, off, off + len), + ); + } else { + values[i] = Uint8List.fromList( + Uint8List.sublistView(bytes, off, off + len), + ); + } + off += len; + default: + return null; + } + } + return values; + } + + while (off < bytes.length) { + if (!canRead(1 + 4)) return null; + final op = data.getUint8(off); + off += 1; + final tableLen = data.getInt32(off, Endian.little); + off += 4; + if (tableLen < 0 || !canRead(tableLen)) return null; + final table = utf8.decode( + Uint8List.sublistView(bytes, off, off + tableLen), + ); + off += tableLen; + if (!canRead(8 + 8 + 4 + 1 + 1)) return null; + final oldRowid = data.getInt64(off, Endian.little); + off += 8; + final newRowid = data.getInt64(off, Endian.little); + off += 8; + final colCount = data.getInt32(off, Endian.little); + off += 4; + final hasOld = data.getUint8(off) != 0; + off += 1; + final hasNew = data.getUint8(off) != 0; + off += 1; + if (colCount < 0) return null; + + List? oldValues; + List? newValues; + if (hasOld) { + oldValues = readCells(colCount); + if (oldValues == null) return null; + } + if (hasNew) { + newValues = readCells(colCount); + if (newValues == null) return null; + } + if (!hasOld && !hasNew) return null; + + deltas.add( + RowDelta( + op: op, + table: table, + oldRowid: oldRowid, + newRowid: newRowid, + oldValues: oldValues, + newValues: newValues, + ), + ); + } + + return deltas; +} diff --git a/lib/src/stream_engine.dart b/lib/src/stream_engine.dart index 4d93c928..e657fbef 100644 --- a/lib/src/stream_engine.dart +++ b/lib/src/stream_engine.dart @@ -1,5 +1,6 @@ import 'dart:collection'; import 'dart:async'; +import 'dart:typed_data'; import 'dependency_tracking.dart' show @@ -11,9 +12,17 @@ import 'dependency_tracking.dart' import 'profile_counters.dart'; import 'profile_mode.dart'; import 'reader/reader_pool.dart'; +import 'row_deltas.dart'; +import 'stream_ivm.dart'; import 'tracelite_profile.dart'; import 'extensions/set.dart'; +/// Hash sentinel stored after an incrementally-patched emission. The +/// native result hash is 63-bit non-negative, so `-1` can never match — +/// the next fallback re-query always decodes and re-emits rather than +/// risking a stale suppression against a pre-patch baseline. +const int _ivmStaleHash = -1; + // --------------------------------------------------------------------------- // Stream dependency tracking contract // ([EXP-106](../../experiments/106-column-level-deps.md) polish) @@ -110,8 +119,14 @@ final class StreamEngine { /// [TableDependencies.none] means there are no stream-visible changes for /// this writer response. [TableDependencies.unknown] means native dirty-table /// tracking was unreliable, so every active stream must re-query. + /// + /// [deltas] carries the write cycle's raw row-delta bytes when capture + /// was reliable (exp 160). Streams admitted for incremental maintenance + /// consume them to skip or locally patch instead of re-querying; all + /// other streams ignore them. Future onDependencyChanges( TableDependencies changes, { + Uint8List? deltas, int? traceCorrelationId, }) async { if (_entries.isEmpty) { @@ -178,6 +193,9 @@ final class StreamEngine { } for (final entry in dirtyEntries) { + if (deltas != null && _tryIncrementalMaintain(entry, deltas)) { + continue; + } entry.dirty = true; if (traceCorrelationId != null) { entry.pendingTraceCorrelationId = traceCorrelationId; @@ -188,6 +206,9 @@ final class StreamEngine { } _flushQueue(); + _decodedDeltas = null; + _decodedDeltasSource = null; + _deltasByTable = null; } finally { if (kProfileMode) { invalidateSw!.stop(); @@ -224,6 +245,92 @@ final class StreamEngine { } } + /// Per-write-cycle memo for the decoded delta buffer, so N admitted + /// streams on the same table decode it once. Keyed by buffer identity; + /// cleared at the end of each [onDependencyChanges] pass. + List? _decodedDeltas; + Object? _decodedDeltasSource; + Map>? _deltasByTable; + + /// `PRAGMA table_info` results per table, shared across admissions. + final Map>>> _tableInfoCache = {}; + + /// Attempt to maintain [entry]'s cached result from [deltas] instead of + /// re-querying. Returns true when the entry is fully handled for this + /// write cycle (deltas proven irrelevant, or result patched + emitted). + bool _tryIncrementalMaintain(StreamEntry entry, Uint8List deltas) { + final ivm = entry.ivm; + if (ivm == null || entry.inFlight || entry.dirty) return false; + + if (!identical(_decodedDeltasSource, deltas)) { + _decodedDeltasSource = deltas; + _decodedDeltas = decodeRowDeltas(deltas); + _deltasByTable = null; + final decoded = _decodedDeltas; + if (decoded != null) { + final byTable = >{}; + for (final delta in decoded) { + (byTable[delta.table] ??= []).add(delta); + } + _deltasByTable = byTable; + } + } + final byTable = _deltasByTable; + if (byTable == null) return false; // malformed buffer — fall back + + final tableDeltas = byTable[ivm.shape.table]; + if (tableDeltas == null || tableDeltas.isEmpty) { + // The table was reported dirty but capture saw no rows for it — + // contradicts a reliable buffer, so trust the dirty set. + return false; + } + + if (ivm.rows == null) { + final last = entry.lastResult; + if (last == null || !ivm.rebuild(last)) { + // The cached result cannot be keyed (non-int pk, unexpected + // ordering). This is structural, not transient — demote. + entry.ivm = null; + return false; + } + } + + switch (ivm.apply(tableDeltas)) { + case IvmOutcome.unchanged: + if (kProfileMode) ProfileCounters.ivmSkippedTotal++; + return true; + case IvmOutcome.applied: + final rows = ivm.rows!; + entry.lastResult = rows; + entry.lastResultHash = _ivmStaleHash; + entry.lastRowCount = rows.length; + entry.emit(rows); + if (kProfileMode) ProfileCounters.ivmAppliedTotal++; + return true; + case IvmOutcome.bail: + if (kProfileMode) ProfileCounters.ivmBailTotal++; + return false; + } + } + + /// Try to admit [entry] for incremental maintenance. Best-effort and + /// asynchronous; until (and unless) it completes, the entry stays on the + /// plain re-query path. + Future _admitIvm(StreamEntry entry, String table) async { + try { + final escaped = table.replaceAll('"', '""'); + final info = await (_tableInfoCache[table] ??= _pool.select( + 'PRAGMA table_info("$escaped")', + )); + if (entry.subscribers.isEmpty || entry.ivm != null) return; + final shape = classifyIvmQuery(entry.sql, entry.params, table, info); + if (shape == null) return; + entry.ivm = IvmState(shape); + } catch (_) { + // Classification is best-effort; the entry stays on re-query. + } + } + void _flushQueue() { if (_requeryQueue.isEmpty) { return; @@ -253,6 +360,10 @@ final class StreamEngine { _tableIndex.clear(); _unknownDepsEntries.clear(); _requeryQueue.clear(); + _tableInfoCache.clear(); + _decodedDeltas = null; + _decodedDeltasSource = null; + _deltasByTable = null; } /// Create a new stream entry and return a subscriber stream. @@ -305,6 +416,12 @@ final class StreamEngine { entry.dependencies = { for (final dependency in tables) dependency.table: dependency, }; + + // Single-table queries are candidates for incremental + // maintenance (exp 160). Admission is async and best-effort. + if (tables.length == 1) { + unawaited(_admitIvm(entry, tables.single.table)); + } } // If an invalidation occurred while performing the entry's initial query then the entry @@ -362,6 +479,10 @@ final class StreamEngine { entry.lastResultHash = newHash; entry.lastRowCount = newRowCount; entry.lastResult = rows; + // A fresh query result invalidates the maintained cache; it is + // rebuilt lazily from lastResult on the next applicable delta. + entry.ivm?.rows = null; + entry.ivm?.keys = null; entry.emit(rows); } catch (e, st) { @@ -484,6 +605,10 @@ final class StreamEntry { /// to a triggering write in tracelite. int? pendingTraceCorrelationId; + /// Incremental maintenance state (exp 160). Non-null once the query has + /// been admitted by the tier-1 classifier; null streams always re-query. + IvmState? ivm; + @override int get hashCode => key; diff --git a/lib/src/stream_ivm.dart b/lib/src/stream_ivm.dart new file mode 100644 index 00000000..d6b06f7e --- /dev/null +++ b/lib/src/stream_ivm.dart @@ -0,0 +1,651 @@ +/// Tier-1 incremental view maintenance for streams (exp 160). +/// +/// A stream is *admitted* when its query falls inside a deliberately tiny +/// grammar whose semantics this module can mirror exactly: +/// +/// SELECT FROM
+/// [WHERE (AND ...)*] +/// [ORDER BY [ASC]] +/// +/// with every comparison on INTEGER-typed values only, the table having a +/// single-column `INTEGER PRIMARY KEY` (a rowid alias), the key column in +/// the projection, and either an `ORDER BY ` or a `pk = ?` equality +/// predicate (so result order is fully determined). Everything outside the +/// grammar simply stays on the existing re-query path — a classifier miss +/// costs performance, never correctness. +/// +/// For admitted streams, the engine maintains the materialized result by +/// applying the writer's row deltas: +/// +/// * a delta row failing the predicate before AND after the write is a +/// *proven miss* — no reader dispatch, no SQLite, no hash; +/// * in-window updates / entries / departures patch the cached rows +/// locally and emit; +/// * anything unprovable (type mismatch, cache inconsistency, schema +/// drift) bails to the normal re-query path. +library; + +import 'row_deltas.dart'; + +// --------------------------------------------------------------------------- +// Query shape +// --------------------------------------------------------------------------- + +final class IvmPredicate { + const IvmPredicate(this.columnIndex, this.op, this.value); + + /// Table column index (cid from `PRAGMA table_info`). + final int columnIndex; + + /// One of `=`, `<`, `<=`, `>`, `>=` (== normalizes to =). + final String op; + + /// Comparison constant, resolved from a literal or the stream's fixed + /// bind parameters at classification time. + final int value; + + bool evaluate(int cell) => switch (op) { + '=' => cell == value, + '<' => cell < value, + '<=' => cell <= value, + '>' => cell > value, + '>=' => cell >= value, + _ => false, + }; +} + +final class IvmShape { + const IvmShape({ + required this.table, + required this.predicates, + required this.projection, + required this.pkColumnIndex, + required this.pkOutputName, + required this.tableColumnCount, + }); + + final String table; + final List predicates; + + /// Output column name → table column index, in projection order. + final List<(String, int)> projection; + + /// Table column index of the INTEGER PRIMARY KEY (rowid alias). + final int pkColumnIndex; + + /// The pk's name as it appears in emitted row maps. + final String pkOutputName; + + /// `PRAGMA table_info` row count at classification time. A delta whose + /// column count differs means the schema changed under us — demote. + final int tableColumnCount; +} + +// --------------------------------------------------------------------------- +// Classifier +// --------------------------------------------------------------------------- + +/// Classify a stream query against the tier-1 grammar. +/// +/// [tableInfo] is the result of `PRAGMA table_info(table)`. Returns `null` +/// when the query is not admissible. +IvmShape? classifyIvmQuery( + String sql, + List params, + String table, + List> tableInfo, +) { + // Table metadata: name (declared case) → cid, and the single INTEGER + // PRIMARY KEY column. + if (tableInfo.isEmpty || tableInfo.length > 64) return null; + final cidByLowerName = {}; + final declaredNameByCid = {}; + var pkCid = -1; + var pkCount = 0; + for (final col in tableInfo) { + final cid = col['cid']; + final name = col['name']; + final type = col['type']; + final pk = col['pk']; + if (cid is! int || name is! String || pk is! int) return null; + cidByLowerName[name.toLowerCase()] = cid; + declaredNameByCid[cid] = name; + if (pk > 0) { + pkCount++; + if (pk == 1 && type is String && type.toUpperCase() == 'INTEGER') { + pkCid = cid; + } + } + } + // Exactly one pk column, and it must be the INTEGER rowid alias. + if (pkCount != 1 || pkCid < 0) return null; + + final tokens = _tokenize(sql); + if (tokens == null) return null; + final cursor = _Cursor(tokens); + + if (!cursor.takeKeyword('select')) return null; + + // Projection. + final projection = <(String, int)>[]; + if (cursor.takeSymbol('*')) { + for (var cid = 0; cid < tableInfo.length; cid++) { + final name = declaredNameByCid[cid]; + if (name == null) return null; + projection.add((name, cid)); + } + } else { + while (true) { + final ident = cursor.takeIdent(); + if (ident == null) return null; + final cid = cidByLowerName[ident.toLowerCase()]; + if (cid == null) return null; + // Bare column reference: SQLite names the result column as written. + projection.add((ident, cid)); + if (!cursor.takeSymbol(',')) break; + } + } + + if (!cursor.takeKeyword('from')) return null; + final fromIdent = cursor.takeIdent(); + if (fromIdent == null || fromIdent.toLowerCase() != table.toLowerCase()) { + return null; + } + + // WHERE conjunction. + final predicates = []; + var paramIndex = 0; + var hasPkEquality = false; + if (cursor.takeKeyword('where')) { + while (true) { + final ident = cursor.takeIdent(); + if (ident == null) return null; + final cid = cidByLowerName[ident.toLowerCase()]; + if (cid == null) return null; + final op = cursor.takeOperator(); + if (op == null) return null; + final int value; + if (cursor.takeSymbol('?')) { + if (paramIndex >= params.length) return null; + final param = params[paramIndex++]; + if (param is! int) return null; + value = param; + } else { + final literal = cursor.takeIntLiteral(); + if (literal == null) return null; + value = literal; + } + predicates.add(IvmPredicate(cid, op, value)); + if (op == '=' && cid == pkCid) hasPkEquality = true; + if (!cursor.takeKeyword('and')) break; + } + } + // Every bind parameter must be consumed by the WHERE clause — a `?` + // anywhere else was already rejected by the grammar, but a surplus + // parameter means the SQL used it somewhere we didn't model. + if (paramIndex != params.length) return null; + + // ORDER BY — only the pk, only ascending. + var orderedByPk = false; + if (cursor.takeKeyword('order')) { + if (!cursor.takeKeyword('by')) return null; + final ident = cursor.takeIdent(); + if (ident == null) return null; + if (cidByLowerName[ident.toLowerCase()] != pkCid) return null; + cursor.takeKeyword('asc'); // optional + orderedByPk = true; + } + + cursor.takeSymbol(';'); + if (!cursor.atEnd) return null; + + // Result order must be fully determined: either ordered by pk, or pinned + // to at most one row by a pk equality predicate. + if (!orderedByPk && !hasPkEquality) return null; + + // The pk must be projected so cached rows can be keyed. + String? pkOutputName; + for (final (name, cid) in projection) { + if (cid == pkCid) { + pkOutputName = name; + break; + } + } + if (pkOutputName == null) return null; + + return IvmShape( + table: table, + predicates: predicates, + projection: projection, + pkColumnIndex: pkCid, + pkOutputName: pkOutputName, + tableColumnCount: tableInfo.length, + ); +} + +// --------------------------------------------------------------------------- +// Tokenizer — strict by construction: any unexpected character rejects. +// --------------------------------------------------------------------------- + +final class _Token { + const _Token(this.kind, this.text, [this.intValue = 0]); + final int kind; // 0 ident/keyword, 1 int literal, 2 symbol/operator + final String text; + final int intValue; +} + +List<_Token>? _tokenize(String sql) { + final tokens = <_Token>[]; + var i = 0; + while (i < sql.length) { + final c = sql.codeUnitAt(i); + // Whitespace. + if (c == 0x20 || c == 0x09 || c == 0x0a || c == 0x0d) { + i++; + continue; + } + // Identifier / keyword. + if (_isIdentStart(c)) { + final start = i; + while (i < sql.length && _isIdentChar(sql.codeUnitAt(i))) { + i++; + } + tokens.add(_Token(0, sql.substring(start, i))); + continue; + } + // Integer literal (optionally negative). + if (_isDigit(c) || + (c == 0x2d /* - */ && + i + 1 < sql.length && + _isDigit(sql.codeUnitAt(i + 1)))) { + final start = i; + i++; + while (i < sql.length && _isDigit(sql.codeUnitAt(i))) { + i++; + } + final text = sql.substring(start, i); + final value = int.tryParse(text); + if (value == null) return null; + tokens.add(_Token(1, text, value)); + continue; + } + // Operators / symbols. + switch (c) { + case 0x2a: // * + tokens.add(const _Token(2, '*')); + i++; + case 0x2c: // , + tokens.add(const _Token(2, ',')); + i++; + case 0x3f: // ? + tokens.add(const _Token(2, '?')); + i++; + case 0x3b: // ; + tokens.add(const _Token(2, ';')); + i++; + case 0x3d: // = or == + i++; + if (i < sql.length && sql.codeUnitAt(i) == 0x3d) i++; + tokens.add(const _Token(2, '=')); + case 0x3c: // < or <= + i++; + if (i < sql.length && sql.codeUnitAt(i) == 0x3d) { + tokens.add(const _Token(2, '<=')); + i++; + } else { + tokens.add(const _Token(2, '<')); + } + case 0x3e: // > or >= + i++; + if (i < sql.length && sql.codeUnitAt(i) == 0x3d) { + tokens.add(const _Token(2, '>=')); + i++; + } else { + tokens.add(const _Token(2, '>')); + } + default: + // Anything else (quotes, dots, parens, arithmetic, ...) is outside + // the grammar. + return null; + } + } + return tokens; +} + +bool _isDigit(int c) => c >= 0x30 && c <= 0x39; +bool _isIdentStart(int c) => + (c >= 0x41 && c <= 0x5a) || (c >= 0x61 && c <= 0x7a) || c == 0x5f; +bool _isIdentChar(int c) => _isIdentStart(c) || _isDigit(c); + +/// SQL keywords that may never appear as bare identifiers in an admitted +/// query — their presence anywhere outside their grammar position rejects. +const _reservedAsIdent = { + 'select', + 'from', + 'where', + 'and', + 'or', + 'not', + 'order', + 'by', + 'asc', + 'desc', + 'limit', + 'offset', + 'group', + 'having', + 'join', + 'left', + 'inner', + 'outer', + 'cross', + 'union', + 'except', + 'intersect', + 'distinct', + 'as', + 'in', + 'between', + 'like', + 'is', + 'null', + 'case', + 'when', + 'collate', + 'glob', + 'exists', +}; + +final class _Cursor { + _Cursor(this.tokens); + final List<_Token> tokens; + int pos = 0; + + bool get atEnd => pos >= tokens.length; + + bool takeKeyword(String keyword) { + if (atEnd) return false; + final t = tokens[pos]; + if (t.kind == 0 && t.text.toLowerCase() == keyword) { + pos++; + return true; + } + return false; + } + + String? takeIdent() { + if (atEnd) return null; + final t = tokens[pos]; + if (t.kind != 0) return null; + if (_reservedAsIdent.contains(t.text.toLowerCase())) return null; + pos++; + return t.text; + } + + bool takeSymbol(String symbol) { + if (atEnd) return false; + final t = tokens[pos]; + if (t.kind == 2 && t.text == symbol) { + pos++; + return true; + } + return false; + } + + String? takeOperator() { + if (atEnd) return null; + final t = tokens[pos]; + if (t.kind == 2 && + (t.text == '=' || + t.text == '<' || + t.text == '<=' || + t.text == '>' || + t.text == '>=')) { + pos++; + return t.text; + } + return null; + } + + int? takeIntLiteral() { + if (atEnd) return null; + final t = tokens[pos]; + if (t.kind != 1) return null; + pos++; + return t.intValue; + } +} + +// --------------------------------------------------------------------------- +// Maintained state + delta application +// --------------------------------------------------------------------------- + +enum IvmOutcome { + /// All deltas were proven irrelevant — nothing to emit, no re-query. + unchanged, + + /// The cached result was patched; the caller should emit it. + applied, + + /// Something was unprovable or inconsistent — fall back to re-query. + bail, +} + +final class IvmState { + IvmState(this.shape); + + final IvmShape shape; + + /// Materialized rows, ascending by pk. `null` until built from the + /// entry's last result (and after any bail/fallback re-query). + List>? rows; + + /// Sorted pk values parallel to [rows]. + List? keys; + + /// Build the maintained cache from the last emitted result. Returns + /// false (leaving the cache unset) when the rows cannot be keyed. + bool rebuild(List> lastResult) { + final newRows = List>.generate( + lastResult.length, + (i) => Map.of(lastResult[i]), + growable: true, + ); + final newKeys = []; + for (final row in newRows) { + final key = row[shape.pkOutputName]; + if (key is! int) return false; + if (newKeys.isNotEmpty && key <= newKeys.last) { + // Result was not strictly ascending by pk — never patch it. + return false; + } + newKeys.add(key); + } + rows = newRows; + keys = newKeys; + return true; + } + + /// Apply [deltas] (already filtered to this shape's table). + /// + /// On [IvmOutcome.applied], [rows] is a *fresh* list (previously emitted + /// lists are never mutated). On [IvmOutcome.bail], the cache is cleared. + IvmOutcome apply(List deltas) { + var currentRows = rows; + var currentKeys = keys; + if (currentRows == null || currentKeys == null) return _bail(); + + var mutated = false; + List> workRows = currentRows; + List workKeys = currentKeys; + + void ensureMutable() { + if (!mutated) { + workRows = List>.of(workRows); + workKeys = List.of(workKeys); + mutated = true; + } + } + + bool? evalPredicates(List values) { + for (final pred in shape.predicates) { + final cell = values[pred.columnIndex]; + if (cell == null) return false; // SQL comparison with NULL → no match + if (cell is! int) return null; // unprovable type + if (!pred.evaluate(cell)) return false; + } + return true; + } + + bool applyOne( + bool hasOld, + bool hasNew, + int oldKey, + int newKey, + List? oldValues, + List? newValues, + ) { + final pOldOrNull = hasOld ? evalPredicates(oldValues!) : false; + final pNewOrNull = hasNew ? evalPredicates(newValues!) : false; + if (pOldOrNull == null || pNewOrNull == null) return false; + final pOld = pOldOrNull; + final pNew = pNewOrNull; + + final oldIdx = _binarySearch(workKeys, oldKey); + if (!pOld && !pNew) { + // Proven miss — but if the cache claims the row is present, it is + // out of sync with reality. + if (hasOld && oldIdx >= 0) return false; + return true; + } + + if (pOld && pNew) { + // Row stays in the result; keys are equal here (rowid changes were + // split by the caller). + if (oldIdx < 0) return false; + final patched = _projectRow(newValues); + if (patched == null) return false; + if (_projectedEquals(workRows[oldIdx], patched)) return true; + ensureMutable(); + workRows[oldIdx] = patched; + return true; + } + + if (pOld) { + // Row leaves the result. + if (oldIdx < 0) return false; + ensureMutable(); + workRows.removeAt(oldIdx); + workKeys.removeAt(oldIdx); + return true; + } + + // Row enters the result. + final newIdx = _binarySearch(workKeys, newKey); + if (newIdx >= 0) return false; + final inserted = _projectRow(newValues!); + if (inserted == null) return false; + ensureMutable(); + final insertAt = -(newIdx + 1); + workRows.insert(insertAt, inserted); + workKeys.insert(insertAt, newKey); + return true; + } + + for (final delta in deltas) { + // Schema drift guard: the capture's column count must match the + // shape's view of the table. + final values = delta.newValues ?? delta.oldValues; + if (values == null || values.length != shape.tableColumnCount) { + return _bail(); + } + if (delta.oldValues != null && + delta.oldValues!.length != shape.tableColumnCount) { + return _bail(); + } + + final bool ok; + if (delta.op == deltaOpUpdate && delta.oldRowid != delta.newRowid) { + // Rowid changed: split into departure + entry. + ok = + applyOne( + true, + false, + delta.oldRowid, + delta.oldRowid, + delta.oldValues, + null, + ) && + applyOne( + false, + true, + delta.newRowid, + delta.newRowid, + null, + delta.newValues, + ); + } else { + ok = applyOne( + delta.oldValues != null, + delta.newValues != null, + delta.oldRowid, + delta.newRowid, + delta.oldValues, + delta.newValues, + ); + } + if (!ok) return _bail(); + } + + if (!mutated) return IvmOutcome.unchanged; + rows = workRows; + keys = workKeys; + return IvmOutcome.applied; + } + + IvmOutcome _bail() { + rows = null; + keys = null; + return IvmOutcome.bail; + } + + /// Build an emitted row from table-indexed [values] via the projection. + /// Returns null when the pk cell is not an int (unprovable). + Map? _projectRow(List values) { + if (values[shape.pkColumnIndex] is! int) return null; + final row = {}; + for (final (name, cid) in shape.projection) { + row[name] = values[cid]; + } + return row; + } + + /// Whether two projected rows are provably identical. Non-primitive + /// values (blobs) compare by identity, which can only produce a false + /// "changed" — never a false "unchanged". + bool _projectedEquals(Map a, Map b) { + for (final (name, _) in shape.projection) { + if (a[name] != b[name]) return false; + } + return true; + } +} + +/// Standard binary search: index when found, `-(insertion point) - 1` +/// when absent. +int _binarySearch(List keys, int key) { + var lo = 0; + var hi = keys.length - 1; + while (lo <= hi) { + final mid = (lo + hi) >> 1; + final v = keys[mid]; + if (v < key) { + lo = mid + 1; + } else if (v > key) { + hi = mid - 1; + } else { + return mid; + } + } + return -(lo + 1); +} diff --git a/lib/src/writer/write_worker.dart b/lib/src/writer/write_worker.dart index c8caab68..c6e71d4b 100644 --- a/lib/src/writer/write_worker.dart +++ b/lib/src/writer/write_worker.dart @@ -10,6 +10,7 @@ library; import 'dart:developer' show Timeline; import 'dart:ffi' as ffi; import 'dart:isolate'; +import 'dart:typed_data'; import 'package:ffi/ffi.dart'; @@ -100,11 +101,18 @@ final class ExecuteResponse { this.result, this.modifications, { this.writerSqliteUs = 0, + this.deltas, }); final WriteResult result; final TableDependencies modifications; final int writerSqliteUs; + + /// Raw row-delta bytes for this write cycle (exp 160). `null` when + /// capture was unreliable or not applicable (inside a transaction); + /// empty when the cycle modified no rows. Decoded lazily by the stream + /// engine only when an incrementally-maintained stream exists. + final Uint8List? deltas; } /// Response to [QueryRequest] (transaction reads). @@ -116,10 +124,18 @@ final class QueryResponse { /// Response to [BatchRequest] and [CommitRequest]. final class BatchResponse { - const BatchResponse(this.modifications, {this.writerSqliteUs = 0}); + const BatchResponse( + this.modifications, { + this.writerSqliteUs = 0, + this.deltas, + }); final TableDependencies modifications; final int writerSqliteUs; + + /// Raw row-delta bytes for this write cycle (exp 160). See + /// [ExecuteResponse.deltas]. + final Uint8List? deltas; } // --------------------------------------------------------------------------- @@ -255,14 +271,21 @@ void _handleExecute(_WriterState state, ExecuteRequest msg) { final sqliteSw = kProfileMode ? (Stopwatch()..start()) : null; final result = executeWrite(state.dbHandle, msg.sql, msg.params); final writerSqliteUs = _stopSqliteTimer(sqliteSw); - // Dirty tables and columns are only collected outside transactions. - // Inside a transaction they accumulate in the C-level dirty sets until - // the outermost transaction completes. - final modifications = state.txDepth > 0 + // Dirty tables, columns, and row deltas are only collected outside + // transactions. Inside a transaction they accumulate in the C-level + // accumulators until the outermost transaction completes. + final inTx = state.txDepth > 0; + final modifications = inTx ? TableDependencies.none : getDirtyTableDependencies(state.dbHandle); + final deltas = inTx ? null : drainRowDeltas(state.dbHandle); msg.replyPort.send( - ExecuteResponse(result, modifications, writerSqliteUs: writerSqliteUs), + ExecuteResponse( + result, + modifications, + writerSqliteUs: writerSqliteUs, + deltas: deltas, + ), ); } @@ -286,6 +309,7 @@ void _handleBatch(_WriterState state, BatchRequest msg) { BatchResponse( getDirtyTableDependencies(state.dbHandle), writerSqliteUs: writerSqliteUs, + deltas: drainRowDeltas(state.dbHandle), ), ); } @@ -392,6 +416,7 @@ void _handleCommit(_WriterState state, CommitRequest msg) { BatchResponse( getDirtyTableDependencies(state.dbHandle), writerSqliteUs: writerSqliteUs, + deltas: drainRowDeltas(state.dbHandle), ), ); } else { @@ -422,6 +447,9 @@ void _handleCommit(_WriterState state, CommitRequest msg) { resqliteExec(state.dbHandle, releaseSp); calloc.free(rollbackSp); calloc.free(releaseSp); + // The accumulated deltas may include rows the ROLLBACK TO just + // undid; the surviving outer transaction must not apply them. + resqliteDeltasMarkUnreliable(state.dbHandle); state.txDepth = newDepth; throw ResqliteTransactionException( errMsg, @@ -467,6 +495,10 @@ void _handleRollback(_WriterState state, RollbackRequest msg) { final rc2 = resqliteExec(state.dbHandle, releaseSp); calloc.free(rollbackSp); calloc.free(releaseSp); + // The accumulated deltas may include rows the ROLLBACK TO just undid; + // the surviving outer transaction must not apply them. Dirty tables + // stay accumulated — over-invalidation is safe, stale deltas are not. + resqliteDeltasMarkUnreliable(state.dbHandle); state.txDepth = newDepth; if (rc1 != 0) { throw ResqliteTransactionException( diff --git a/lib/src/writer/writer.dart b/lib/src/writer/writer.dart index 265a2fa0..8443c8b7 100644 --- a/lib/src/writer/writer.dart +++ b/lib/src/writer/writer.dart @@ -306,6 +306,7 @@ final class Writer { if (Transaction.current == null) { _streamEngine.onDependencyChanges( response.modifications, + deltas: response.deltas, traceCorrelationId: traceCorrelationId, ); } diff --git a/native/resqlite.c b/native/resqlite.c index 403eb052..3cc54396 100644 --- a/native/resqlite.c +++ b/native/resqlite.c @@ -376,6 +376,14 @@ struct resqlite_db { resqlite_cached_stmt* writer_active_entry; int writer_checkpoint_running; + // Row deltas accumulated by the preupdate hook (exp 160). Bounded and + // best-effort: any cap overflow, allocation failure, or savepoint + // rollback flips `delta_reliable` and empties the buffer, and the Dart + // layer falls back to plain re-query invalidation for that write cycle. + resqlite_buf delta_buf; + int delta_rows; + int delta_reliable; + // Reader pool. resqlite_reader readers[MAX_READERS]; int reader_count; @@ -477,6 +485,113 @@ static void dirty_columns_add_for_active_stmt(resqlite_db* sdb, } } +// Poison the current delta capture cycle. The next drain reports +// unreliable once, then capture resumes fresh. +static void delta_mark_unreliable(resqlite_db* sdb) { + sdb->delta_reliable = 0; + sdb->delta_buf.len = 0; + sdb->delta_rows = 0; +} + +// Serialize one sqlite3_value into the delta buffer (tag + payload). +static int delta_write_value(resqlite_buf* b, sqlite3_value* v) { + if (!v) return -1; + switch (sqlite3_value_type(v)) { + case SQLITE_NULL: + return buf_write_byte(b, 0); + case SQLITE_INTEGER: + if (buf_write_byte(b, 1) != 0) return -1; + return buf_write_i64(b, sqlite3_value_int64(v)); + case SQLITE_FLOAT: + if (buf_write_byte(b, 2) != 0) return -1; + return buf_write_f64(b, sqlite3_value_double(v)); + case SQLITE_TEXT: { + const unsigned char* t = sqlite3_value_text(v); + int n = sqlite3_value_bytes(v); + if (!t && n > 0) return -1; + if (buf_write_byte(b, 3) != 0) return -1; + if (buf_write_i32(b, n) != 0) return -1; + return n > 0 ? buf_write(b, t, n) : 0; + } + case SQLITE_BLOB: { + const void* p = sqlite3_value_blob(v); + int n = sqlite3_value_bytes(v); + if (!p && n > 0) return -1; + if (buf_write_byte(b, 4) != 0) return -1; + if (buf_write_i32(b, n) != 0) return -1; + return n > 0 ? buf_write(b, p, n) : 0; + } + } + return -1; +} + +// Capture one modified row's old/new values from inside the preupdate +// callback (the only window where sqlite3_preupdate_old/new are valid). +// Row appends are atomic: any failure truncates back to the row start and +// poisons the cycle. +static void delta_capture( + resqlite_db* sdb, + sqlite3* db, + int op, + const char* table_name, + sqlite3_int64 old_rowid, + sqlite3_int64 new_rowid +) { + if (!sdb->delta_reliable) return; + if (!table_name) { delta_mark_unreliable(sdb); return; } + if (sdb->delta_rows >= RESQLITE_MAX_DELTA_ROWS || + sdb->delta_buf.len > RESQLITE_MAX_DELTA_BYTES) { + delta_mark_unreliable(sdb); + return; + } + int col_count = sqlite3_preupdate_count(db); + if (col_count <= 0 || col_count > RESQLITE_MAX_DELTA_ROW_COLUMNS) { + delta_mark_unreliable(sdb); + return; + } + + int row_start = sdb->delta_buf.len; + int table_len = (int)strlen(table_name); + unsigned char has_old = (op == SQLITE_UPDATE || op == SQLITE_DELETE); + unsigned char has_new = (op == SQLITE_UPDATE || op == SQLITE_INSERT); + + resqlite_buf* b = &sdb->delta_buf; + int ok = buf_write_byte(b, (unsigned char)op) == 0 && + buf_write_i32(b, table_len) == 0 && + buf_write(b, table_name, table_len) == 0 && + buf_write_i64(b, old_rowid) == 0 && + buf_write_i64(b, new_rowid) == 0 && + buf_write_i32(b, col_count) == 0 && + buf_write_byte(b, has_old) == 0 && + buf_write_byte(b, has_new) == 0; + + if (ok && has_old) { + for (int i = 0; ok && i < col_count; i++) { + sqlite3_value* v = NULL; + if (sqlite3_preupdate_old(db, i, &v) != SQLITE_OK || + delta_write_value(b, v) != 0) { + ok = 0; + } + } + } + if (ok && has_new) { + for (int i = 0; ok && i < col_count; i++) { + sqlite3_value* v = NULL; + if (sqlite3_preupdate_new(db, i, &v) != SQLITE_OK || + delta_write_value(b, v) != 0) { + ok = 0; + } + } + } + + if (!ok) { + sdb->delta_buf.len = row_start; + delta_mark_unreliable(sdb); + return; + } + sdb->delta_rows++; +} + static void preupdate_hook( void* user_data, sqlite3* db, @@ -486,10 +601,11 @@ static void preupdate_hook( sqlite3_int64 old_rowid, sqlite3_int64 new_rowid ) { - (void)db; (void)op; (void)db_name; (void)old_rowid; (void)new_rowid; + (void)db_name; resqlite_db* sdb = (resqlite_db*)user_data; resqlite_dirty_set_add(&sdb->dirty_tables, table_name); dirty_columns_add_for_active_stmt(sdb, table_name); + delta_capture(sdb, db, op, table_name, old_rowid, new_rowid); } static int writer_wal_hook( @@ -632,6 +748,14 @@ static resqlite_db* resqlite_open_impl(const char* path, int max_readers, resqlite_column_set_init(&db->dirty_columns); resqlite_column_set_init(&db->writer_authz_scratch); db->writer_active_entry = NULL; + if (buf_init(&db->delta_buf, 4096) != 0) { + // Allocation failure at open: deltas stay permanently unreliable + // (every drain reports fallback); core functionality is unaffected. + db->delta_buf.data = NULL; + db->delta_buf.cap = 0; + } + db->delta_rows = 0; + db->delta_reliable = db->delta_buf.data != NULL; db->writer_mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); db->pool_mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); @@ -799,6 +923,8 @@ void resqlite_close(resqlite_db* db) { resqlite_dirty_set_free(&db->dirty_tables); resqlite_column_set_free(&db->dirty_columns); resqlite_column_set_free(&db->writer_authz_scratch); + free(db->delta_buf.data); + db->delta_buf.data = NULL; sqlite3_close_v2(db->writer); sqlite3_mutex_leave(db->writer_mutex); @@ -908,6 +1034,7 @@ int resqlite_run_connection_setup( resqlite_dirty_set_reset(&db->dirty_tables); resqlite_column_set_reset(&db->dirty_columns); resqlite_column_set_reset(&db->writer_authz_scratch); + resqlite_discard_deltas(db); db->writer_active_entry = NULL; } @@ -1251,6 +1378,49 @@ void resqlite_reader_set_busy(resqlite_db* db, int reader_id, int busy) { memory_order_release); } +int resqlite_get_deltas( + resqlite_db* db, + const unsigned char** out_buf, + int* out_len, + int* out_rows +) { + if (out_buf) *out_buf = NULL; + if (out_len) *out_len = 0; + if (out_rows) *out_rows = 0; + if (!db || atomic_load_explicit(&db->closed, memory_order_acquire)) { + return 0; + } + + int reliable = db->delta_reliable; + if (reliable && out_buf && out_len && out_rows) { + *out_buf = db->delta_buf.data; + *out_len = db->delta_buf.len; + *out_rows = db->delta_rows; + } + + // Reset for the next capture cycle. Buffer bytes stay valid until the + // next writer activity — the caller copies before issuing more writes + // (same contract as resqlite_get_dirty_tables strings). + db->delta_buf.len = 0; + db->delta_rows = 0; + // A buffer that failed to allocate at open stays permanently unreliable. + db->delta_reliable = db->delta_buf.data != NULL; + + return reliable; +} + +void resqlite_discard_deltas(resqlite_db* db) { + if (!db) return; + db->delta_buf.len = 0; + db->delta_rows = 0; + db->delta_reliable = db->delta_buf.data != NULL; +} + +void resqlite_deltas_mark_unreliable(resqlite_db* db) { + if (!db) return; + delta_mark_unreliable(db); +} + // Polish (post-2026-04): returns RESQLITE_DEPENDENCY_COUNT_UNKNOWN when // the cached entry's read-table dependencies are unreliable (overflow / OOM // during prepare). Zero would mean "stream has no table deps" → silent stuck diff --git a/native/resqlite.h b/native/resqlite.h index 8560f70f..5b72543d 100644 --- a/native/resqlite.h +++ b/native/resqlite.h @@ -172,6 +172,49 @@ int resqlite_get_dirty_tables( int max_tables ); +// --------------------------------------------------------------------------- +// Row delta capture (exp 160 — incremental stream maintenance) +// --------------------------------------------------------------------------- + +// Bounded, best-effort capture of per-row old/new values inside the writer +// preupdate hook. Exceeding any cap (rows per drain cycle, columns per row, +// total bytes), an allocation failure, or a savepoint rollback flips the +// accumulator's reliable flag; Dart then falls back to plain re-query +// invalidation. Tables remain the correctness layer — deltas are purely an +// optimization input. +#define RESQLITE_MAX_DELTA_ROWS 256 +#define RESQLITE_MAX_DELTA_ROW_COLUMNS 32 +#define RESQLITE_MAX_DELTA_BYTES (256 * 1024) + +// Buffer layout (little-endian), one record per modified row: +// u8 op SQLITE_INSERT (18) | SQLITE_UPDATE (23) | SQLITE_DELETE (9) +// i32 table_len; table bytes (UTF-8, no NUL) +// i64 old_rowid; i64 new_rowid +// i32 col_count +// u8 has_old; u8 has_new +// col_count old cells when has_old, then col_count new cells when has_new +// Cell: u8 tag (0=NULL, 1=INTEGER, 2=FLOAT, 3=TEXT, 4=BLOB); +// INTEGER: i64; FLOAT: f64; TEXT/BLOB: i32 len + bytes. +// +// Drains the accumulator: writes the buffer pointer/length/row count and +// resets capture state for the next cycle. Returns 1 when the captured +// deltas are reliable, 0 when they must be discarded (caller falls back). +// Buffer bytes stay valid until the next writer activity — copy first. +int resqlite_get_deltas( + resqlite_db* db, + const unsigned char** out_buf, + int* out_len, + int* out_rows +); + +// Drop accumulated deltas (transaction rollback / discarded write cycle). +void resqlite_discard_deltas(resqlite_db* db); + +// Poison the current capture cycle (savepoint rollback: the buffer may +// contain rows whose changes were undone). The next drain reports +// unreliable, then capture resumes fresh. +void resqlite_deltas_mark_unreliable(resqlite_db* db); + // --------------------------------------------------------------------------- // Read dependency tracking (authorizer hook on readers) // --------------------------------------------------------------------------- diff --git a/test/profile_counters_test.dart b/test/profile_counters_test.dart index 3e478186..8ca93242 100644 --- a/test/profile_counters_test.dart +++ b/test/profile_counters_test.dart @@ -29,6 +29,9 @@ void main() { 'completion_handler_count': 0, 'stream_emit_us': 0, 'stream_emit_count': 0, + 'ivm_skipped_total': 0, + 'ivm_applied_total': 0, + 'ivm_bail_total': 0, }); ProfileCounters.reset(); @@ -49,6 +52,9 @@ void main() { 'completion_handler_count': 0, 'stream_emit_us': 0, 'stream_emit_count': 0, + 'ivm_skipped_total': 0, + 'ivm_applied_total': 0, + 'ivm_bail_total': 0, }); }); diff --git a/test/stream_ivm_test.dart b/test/stream_ivm_test.dart new file mode 100644 index 00000000..757db65a --- /dev/null +++ b/test/stream_ivm_test.dart @@ -0,0 +1,545 @@ +import 'dart:async'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:resqlite/resqlite.dart'; +import 'package:resqlite/src/row_deltas.dart'; +import 'package:resqlite/src/stream_ivm.dart'; +import 'package:test/test.dart'; + +final _itemsTableInfo = >[ + {'cid': 0, 'name': 'id', 'type': 'INTEGER', 'pk': 1}, + {'cid': 1, 'name': 'flag', 'type': 'INTEGER', 'pk': 0}, + {'cid': 2, 'name': 'score', 'type': 'INTEGER', 'pk': 0}, + {'cid': 3, 'name': 'name', 'type': 'TEXT', 'pk': 0}, +]; + +void main() { + group('classifyIvmQuery', () { + IvmShape? classify(String sql, [List params = const []]) => + classifyIvmQuery(sql, params, 'items', _itemsTableInfo); + + test('admits range + ORDER BY pk', () { + final shape = classify( + 'SELECT id, score, name FROM items WHERE id >= ? AND id < ? ORDER BY id', + [10, 20], + ); + expect(shape, isNotNull); + expect(shape!.predicates, hasLength(2)); + expect(shape.pkOutputName, 'id'); + expect(shape.projection.map((p) => p.$1), ['id', 'score', 'name']); + }); + + test('admits pk equality without ORDER BY', () { + expect(classify('SELECT * FROM items WHERE id = ?', [5]), isNotNull); + }); + + test('admits SELECT * with table-order projection', () { + final shape = classify('SELECT * FROM items WHERE id = 3'); + expect(shape, isNotNull); + expect(shape!.projection.map((p) => p.$1), [ + 'id', + 'flag', + 'score', + 'name', + ]); + }); + + test('admits integer equality on non-pk column with ORDER BY pk', () { + expect( + classify('SELECT id, name FROM items WHERE flag = 1 ORDER BY id'), + isNotNull, + ); + }); + + test('rejects everything outside the grammar', () { + final rejected = [ + // No deterministic order and no pk pin. + 'SELECT id FROM items WHERE flag = 1', + // Order by non-pk. + 'SELECT id, score FROM items WHERE id > 0 ORDER BY score', + 'SELECT id FROM items ORDER BY id DESC', + 'SELECT id FROM items ORDER BY id LIMIT 10', + 'SELECT id FROM items WHERE id > 0 OR flag = 1 ORDER BY id', + 'SELECT id FROM items WHERE name = ? ORDER BY id', // text param + 'SELECT count(*) FROM items WHERE id = 1', + 'SELECT i.id FROM items i WHERE id = 1', + 'SELECT id FROM items JOIN other ON 1 WHERE id = 1', + 'SELECT DISTINCT id FROM items WHERE id = 1', + 'SELECT id AS x FROM items WHERE id = 1', + 'SELECT id FROM other WHERE id = 1', // table mismatch + 'SELECT flag FROM items WHERE id = 1', // pk not projected + 'SELECT id FROM items WHERE missing = 1 ORDER BY id', + "SELECT id FROM items WHERE name = 'x' ORDER BY id", + 'SELECT id FROM items WHERE id IN (1, 2) ORDER BY id', + ]; + for (final sql in rejected) { + expect(classify(sql), isNull, reason: sql); + } + }); + + test('rejects text bind parameter values', () { + expect( + classify('SELECT id FROM items WHERE id = ? ORDER BY id', ['5']), + isNull, + ); + }); + + test('rejects tables without an INTEGER pk rowid alias', () { + final noPk = [ + {'cid': 0, 'name': 'a', 'type': 'INTEGER', 'pk': 0}, + ]; + expect( + classifyIvmQuery('SELECT a FROM t WHERE a = 1', const [], 't', noPk), + isNull, + ); + final textPk = [ + {'cid': 0, 'name': 'k', 'type': 'TEXT', 'pk': 1}, + ]; + expect( + classifyIvmQuery('SELECT k FROM t WHERE k = 1', const [], 't', textPk), + isNull, + ); + final compositePk = [ + {'cid': 0, 'name': 'a', 'type': 'INTEGER', 'pk': 1}, + {'cid': 1, 'name': 'b', 'type': 'INTEGER', 'pk': 2}, + ]; + expect( + classifyIvmQuery( + 'SELECT a, b FROM t WHERE a = 1', + const [], + 't', + compositePk, + ), + isNull, + ); + }); + }); + + group('IvmState.apply', () { + IvmState freshState() { + final shape = classifyIvmQuery( + 'SELECT id, score, name FROM items WHERE id >= 10 AND id < 20 ORDER BY id', + const [], + 'items', + _itemsTableInfo, + )!; + final state = IvmState(shape); + expect( + state.rebuild([ + {'id': 11, 'score': 5, 'name': 'a'}, + {'id': 15, 'score': 7, 'name': 'b'}, + ]), + isTrue, + ); + return state; + } + + RowDelta update(int rowid, List oldV, List newV) => + RowDelta( + op: deltaOpUpdate, + table: 'items', + oldRowid: rowid, + newRowid: rowid, + oldValues: oldV, + newValues: newV, + ); + + test('proven miss leaves the cache untouched', () { + final state = freshState(); + final before = state.rows; + final outcome = state.apply([ + update(50, [50, 0, 1, 'x'], [50, 0, 2, 'x']), + ]); + expect(outcome, IvmOutcome.unchanged); + expect(identical(state.rows, before), isTrue); + }); + + test('in-window patch emits a fresh list and preserves order', () { + final state = freshState(); + final before = state.rows; + final outcome = state.apply([ + update(15, [15, 0, 7, 'b'], [15, 0, 9, 'b2']), + ]); + expect(outcome, IvmOutcome.applied); + expect(identical(state.rows, before), isFalse); + expect(state.rows, [ + {'id': 11, 'score': 5, 'name': 'a'}, + {'id': 15, 'score': 9, 'name': 'b2'}, + ]); + // The pre-patch list (held by earlier subscribers) is unchanged. + expect(before!.last['score'], 7); + }); + + test('update touching only unprojected columns is unchanged', () { + final state = freshState(); + final outcome = state.apply([ + update(15, [15, 0, 7, 'b'], [15, 1, 7, 'b']), // flag not projected + ]); + expect(outcome, IvmOutcome.unchanged); + }); + + test('insert enters at the sorted position', () { + final state = freshState(); + final outcome = state.apply([ + RowDelta( + op: deltaOpInsert, + table: 'items', + oldRowid: 13, + newRowid: 13, + oldValues: null, + newValues: [13, 0, 1, 'c'], + ), + ]); + expect(outcome, IvmOutcome.applied); + expect(state.rows!.map((r) => r['id']), [11, 13, 15]); + }); + + test('delete departs exactly', () { + final state = freshState(); + final outcome = state.apply([ + RowDelta( + op: deltaOpDelete, + table: 'items', + oldRowid: 11, + newRowid: 11, + oldValues: [11, 0, 5, 'a'], + newValues: null, + ), + ]); + expect(outcome, IvmOutcome.applied); + expect(state.rows!.map((r) => r['id']), [15]); + }); + + test('rowid change splits into departure + entry', () { + final state = freshState(); + final outcome = state.apply([ + RowDelta( + op: deltaOpUpdate, + table: 'items', + oldRowid: 15, + newRowid: 12, + oldValues: [15, 0, 7, 'b'], + newValues: [12, 0, 7, 'b'], + ), + ]); + expect(outcome, IvmOutcome.applied); + expect(state.rows!.map((r) => r['id']), [11, 12]); + }); + + test('NULL predicate cell means the row does not match', () { + final state = freshState(); + final outcome = state.apply([ + update(50, [null, 0, 1, 'x'], [null, 0, 2, 'x']), + ]); + expect(outcome, IvmOutcome.unchanged); + }); + + test('non-int predicate cell bails', () { + final state = freshState(); + final outcome = state.apply([ + update(50, ['oops', 0, 1, 'x'], ['oops', 0, 2, 'x']), + ]); + expect(outcome, IvmOutcome.bail); + expect(state.rows, isNull); + }); + + test('cache inconsistency bails (entry already present)', () { + final state = freshState(); + final outcome = state.apply([ + RowDelta( + op: deltaOpInsert, + table: 'items', + oldRowid: 15, + newRowid: 15, + oldValues: null, + newValues: [15, 0, 1, 'dup'], + ), + ]); + expect(outcome, IvmOutcome.bail); + }); + + test('schema drift (column count mismatch) bails', () { + final state = freshState(); + final outcome = state.apply([ + update(15, [15, 0, 7], [15, 0, 9]), + ]); + expect(outcome, IvmOutcome.bail); + }); + + test('rebuild rejects unkeyable or unordered results', () { + final shape = classifyIvmQuery( + 'SELECT id, score, name FROM items WHERE id >= 0 ORDER BY id', + const [], + 'items', + _itemsTableInfo, + )!; + expect( + IvmState(shape).rebuild([ + {'id': 'nope', 'score': 1, 'name': 'a'}, + ]), + isFalse, + ); + expect( + IvmState(shape).rebuild([ + {'id': 5, 'score': 1, 'name': 'a'}, + {'id': 3, 'score': 1, 'name': 'b'}, + ]), + isFalse, + ); + }); + }); + + group('end-to-end incremental streams', () { + late Directory tempDir; + late Database db; + + setUp(() async { + tempDir = await Directory.systemTemp.createTemp('resqlite_ivm_test_'); + db = await Database.open('${tempDir.path}/test.db'); + await db.execute( + 'CREATE TABLE items(id INTEGER PRIMARY KEY, flag INTEGER NOT NULL, ' + 'score INTEGER NOT NULL, name TEXT NOT NULL, weight REAL, ' + 'blob_col BLOB)', + ); + await db.executeBatch( + 'INSERT INTO items(id, flag, score, name) VALUES (?, ?, ?, ?)', + [ + for (var i = 0; i < 50; i++) [i, i % 2, i * 10, 'row_$i'], + ], + ); + }); + + tearDown(() async { + await db.close(); + try { + await tempDir.delete(recursive: true); + } on PathNotFoundException { + // ignore + } + }); + + /// Allow initial emission + async IVM admission + any in-flight + /// invalidations to settle. + Future settle() => Future.delayed( + const Duration(milliseconds: 60), + ); + + /// Assert the latest emission matches a fresh query of the same SQL. + Future expectMatchesSelect( + List>> emissions, + String sql, + List params, + ) async { + final fresh = await db.select(sql, params); + expect(emissions, isNotEmpty); + expect(emissions.last, fresh); + } + + test('range stream tracks misses, patches, entries, departures', () async { + const sql = + 'SELECT id, score, name FROM items ' + 'WHERE id >= ? AND id < ? ORDER BY id'; + final params = [10, 20]; + final emissions = >>[]; + final sub = db.stream(sql, params).listen(emissions.add); + addTearDown(sub.cancel); + await settle(); + expect(emissions, hasLength(1)); + + // Miss: out-of-range write must not emit. + await db.execute('UPDATE items SET score = 999 WHERE id = 40'); + await settle(); + expect(emissions, hasLength(1)); + + // Patch: in-range projected column. + await db.execute('UPDATE items SET score = 111 WHERE id = 15'); + await settle(); + expect(emissions, hasLength(2)); + await expectMatchesSelect(emissions, sql, params); + expect( + emissions.last.firstWhere((r) => r['id'] == 15)['score'], + 111, + ); + + // Entry: insert into the range. + await db.execute( + "INSERT INTO items(id, flag, score, name) VALUES (1000, 0, 5, 'x')", + ); + await settle(); + expect(emissions, hasLength(2)); // out of range — no emission + await db.execute('DELETE FROM items WHERE id = 1000'); + await settle(); + expect(emissions, hasLength(2)); + + await db.execute('DELETE FROM items WHERE id = 12'); + await settle(); + expect(emissions, hasLength(3)); + await expectMatchesSelect(emissions, sql, params); + expect(emissions.last.map((r) => r['id']), isNot(contains(12))); + + // Rowid change: moves a row out of the range. + await db.execute('UPDATE items SET id = 500 WHERE id = 15'); + await settle(); + await expectMatchesSelect(emissions, sql, params); + expect(emissions.last.map((r) => r['id']), isNot(contains(15))); + + // Rowid change: moves a row into the range. + await db.execute('UPDATE items SET id = 12 WHERE id = 500'); + await settle(); + await expectMatchesSelect(emissions, sql, params); + expect(emissions.last.map((r) => r['id']), contains(12)); + }); + + test('keyed stream only wakes for its row', () async { + const sql = 'SELECT * FROM items WHERE id = ?'; + final emissions = >>[]; + final sub = db.stream(sql, [15]).listen(emissions.add); + addTearDown(sub.cancel); + await settle(); + expect(emissions, hasLength(1)); + + for (var i = 0; i < 10; i++) { + await db.execute('UPDATE items SET score = ? WHERE id = ?', [ + i, + 20 + i, + ]); + } + await settle(); + expect(emissions, hasLength(1)); // all misses + + await db.execute('UPDATE items SET name = ? WHERE id = ?', ['hit', 15]); + await settle(); + expect(emissions, hasLength(2)); + await expectMatchesSelect(emissions, sql, [15]); + }); + + test('REAL, NULL, and BLOB projected values survive patching', () async { + const sql = + 'SELECT id, weight, blob_col FROM items ' + 'WHERE id >= 0 AND id < 5 ORDER BY id'; + final emissions = >>[]; + final sub = db.stream(sql, const []).listen(emissions.add); + addTearDown(sub.cancel); + await settle(); + + await db.execute( + 'UPDATE items SET weight = 2.5, blob_col = ? WHERE id = 3', + [ + Uint8List.fromList([1, 2, 3]), + ], + ); + await settle(); + await expectMatchesSelect(emissions, sql, const []); + final patched = emissions.last.firstWhere((r) => r['id'] == 3); + expect(patched['weight'], 2.5); + expect(patched['blob_col'], [1, 2, 3]); + }); + + test('transaction deltas apply once on commit', () async { + const sql = + 'SELECT id, score, name FROM items ' + 'WHERE id >= 10 AND id < 20 ORDER BY id'; + final emissions = >>[]; + final sub = db.stream(sql, const []).listen(emissions.add); + addTearDown(sub.cancel); + await settle(); + expect(emissions, hasLength(1)); + + await db.transaction((tx) async { + await tx.execute('UPDATE items SET score = 1 WHERE id = 11'); + await tx.execute('UPDATE items SET score = 2 WHERE id = 13'); + await tx.execute('DELETE FROM items WHERE id = 17'); + }); + await settle(); + expect(emissions, hasLength(2)); + await expectMatchesSelect(emissions, sql, const []); + }); + + test('rolled-back savepoint never leaks deltas', () async { + const sql = + 'SELECT id, score, name FROM items ' + 'WHERE id >= 10 AND id < 20 ORDER BY id'; + final emissions = >>[]; + final sub = db.stream(sql, const []).listen(emissions.add); + addTearDown(sub.cancel); + await settle(); + + await db.transaction((tx) async { + await tx.execute('UPDATE items SET score = 1 WHERE id = 11'); + try { + await tx.transaction((tx2) async { + await tx2.execute('UPDATE items SET score = 666 WHERE id = 13'); + throw StateError('undo savepoint'); + }); + } on StateError { + // expected + } + await tx.execute('UPDATE items SET score = 2 WHERE id = 14'); + }); + await settle(); + await expectMatchesSelect(emissions, sql, const []); + final byId = {for (final r in emissions.last) r['id']: r}; + expect(byId[11]!['score'], 1); + expect(byId[13]!['score'], 130); // savepoint rollback restored it + expect(byId[14]!['score'], 2); + }); + + test('delta overflow falls back to re-query and stays correct', () async { + const sql = + 'SELECT id, score, name FROM items ' + 'WHERE id >= 0 AND id < 5000 ORDER BY id'; + final emissions = >>[]; + final sub = db.stream(sql, const []).listen(emissions.add); + addTearDown(sub.cancel); + await settle(); + + // 400 rows > RESQLITE_MAX_DELTA_ROWS (256) — capture goes unreliable. + await db.executeBatch( + 'INSERT INTO items(id, flag, score, name) VALUES (?, ?, ?, ?)', + [ + for (var i = 100; i < 500; i++) [i, 0, 1, 'bulk_$i'], + ], + ); + await settle(); + await expectMatchesSelect(emissions, sql, const []); + expect(emissions.last, hasLength(450)); + }); + + test('unadmitted text-predicate stream still works via re-query', () async { + const sql = "SELECT id, name FROM items WHERE name = 'row_7'"; + final emissions = >>[]; + final sub = db.stream(sql, const []).listen(emissions.add); + addTearDown(sub.cancel); + await settle(); + expect(emissions, hasLength(1)); + + await db.execute("UPDATE items SET name = 'row_7' WHERE id = 8"); + await settle(); + await expectMatchesSelect(emissions, sql, const []); + expect(emissions.last, hasLength(2)); + }); + + test('two partitioned streams wake independently', () async { + const sqlA = + 'SELECT id, score FROM items WHERE id >= 0 AND id < 10 ORDER BY id'; + const sqlB = + 'SELECT id, score FROM items WHERE id >= 10 AND id < 20 ORDER BY id'; + final emissionsA = >>[]; + final emissionsB = >>[]; + final subA = db.stream(sqlA, const []).listen(emissionsA.add); + final subB = db.stream(sqlB, const []).listen(emissionsB.add); + addTearDown(subA.cancel); + addTearDown(subB.cancel); + await settle(); + expect(emissionsA, hasLength(1)); + expect(emissionsB, hasLength(1)); + + await db.execute('UPDATE items SET score = 1 WHERE id = 5'); + await settle(); + expect(emissionsA, hasLength(2)); + expect(emissionsB, hasLength(1)); + await expectMatchesSelect(emissionsA, sqlA, const []); + }); + }); +} From 9be72af4ab35f513f74f17914b8761680ba7432e Mon Sep 17 00:00:00 2001 From: Dan Reynolds Date: Wed, 10 Jun 2026 11:50:15 -0400 Subject: [PATCH 02/10] exp 160: admission audit + encapsulation pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the app-shaped streaming workload the suite was missing: benchmark/profile/ivm_admission_audit.dart runs eight reactive-UI stream shapes (chat panes, conversation list, unread badge, user cards, transcripts, feed page, drafts) over chat+feed schemas with a chat-shaped write burst. Tier-1 admits 30/62 distinct entries and resolves 3,000 invalidation decisions without reader re-query (0 bails), while the unadmitted DESC+LIMIT shapes — the highest-churn screens — still generate ~3,100 reader replies: the quantified case for tier-2 (composite-key ordering + LIMIT K+buffer). Encapsulation: per-cycle delta decode moves out of the stream engine into RowDeltaBatch (lazy, grouped by table, no cross-cycle state) — the engine's only remaining IVM state is StreamEntry.ivm plus the table_info admission cache. Adds ivm_admitted/rejected_total classifier counters. Co-Authored-By: Claude Fable 5 --- benchmark/profile/ivm_admission_audit.dart | 313 ++++++++++++++++++ .../exp-160-stream-delta-ivm-aggregate.md | 14 + experiments/160-stream-delta-ivm.md | 35 ++ lib/src/profile_counters.dart | 11 + lib/src/row_deltas.dart | 32 ++ lib/src/stream_engine.dart | 56 +--- test/profile_counters_test.dart | 4 + 7 files changed, 427 insertions(+), 38 deletions(-) create mode 100644 benchmark/profile/ivm_admission_audit.dart diff --git a/benchmark/profile/ivm_admission_audit.dart b/benchmark/profile/ivm_admission_audit.dart new file mode 100644 index 00000000..e9158634 --- /dev/null +++ b/benchmark/profile/ivm_admission_audit.dart @@ -0,0 +1,313 @@ +// Profile-mode audit: how much of an app-shaped reactive query mix does +// the exp 160 tier-1 classifier admit, and what do its streams do during +// a realistic write burst? +// +// The tracelite app-shaped scenarios (chat-sim, feed-paging) exercise +// select(), not stream(), so no existing workload answers the question +// "what fraction of real reactive UI queries benefit from incremental +// maintenance?". This harness builds the missing workload: chat + feed +// schemas mirroring the tracelite scenarios, a stream mix drawn from the +// screens a real app would keep live, and a chat-shaped write burst. +// +// Run: +// dart run -DRESQLITE_PROFILE=true \ +// benchmark/profile/ivm_admission_audit.dart +// +// Reports per-query admission verdicts (via the classifier directly, so +// rejection reasons can be annotated) plus the engine's profile counters +// across the burst: admitted/rejected, skipped/applied/bailed, and the +// re-query traffic that remains. + +// ignore_for_file: avoid_print + +import 'dart:async'; +import 'dart:io'; +import 'dart:math' as math; + +import 'package:resqlite/resqlite.dart'; +import 'package:resqlite/src/profile_counters.dart'; +import 'package:resqlite/src/stream_ivm.dart'; + +const _userCount = 200; +const _conversationCount = 50; +const _seedMessages = 2000; +const _feedItems = 1000; +const _writeCount = 300; + +final class _StreamSpec { + const _StreamSpec(this.label, this.table, this.sql, this.paramsFor); + + final String label; + final String table; + final String sql; + final List Function(int instance) paramsFor; +} + +/// The reactive screens a chat + feed app keeps live, 10 instances each +/// (different conversations / users / authors). +const int _instancesPerSpec = 10; + +final _specs = <_StreamSpec>[ + _StreamSpec( + 'message pane (JOIN + DESC + LIMIT)', + 'messages', + 'SELECT m.id, m.body, m.sent_at, u.name, u.avatar_url ' + 'FROM messages m JOIN users u ON u.id = m.sender_id ' + 'WHERE m.conv_id = ? ORDER BY m.sent_at DESC LIMIT 20', + _convParam, + ), + _StreamSpec( + 'message pane, denormalized (DESC + LIMIT)', + 'messages', + 'SELECT id, sender_id, body, sent_at FROM messages ' + 'WHERE conv_id = ? ORDER BY sent_at DESC LIMIT 20', + _convParam, + ), + _StreamSpec( + 'conversation list (DESC + LIMIT)', + 'conversations', + 'SELECT id, last_msg_at FROM conversations ' + 'ORDER BY last_msg_at DESC LIMIT 30', + _noParams, + ), + _StreamSpec( + 'unread badge (aggregate)', + 'messages', + 'SELECT COUNT(*) AS unread FROM messages WHERE conv_id = ? AND id > ?', + (i) => [_convParam(i).first, 0], + ), + _StreamSpec( + 'user card (pk equality)', + 'users', + 'SELECT id, name, avatar_url FROM users WHERE id = ?', + (i) => [i + 1], + ), + _StreamSpec( + 'full conversation transcript (int equality + ORDER BY pk)', + 'messages', + 'SELECT id, sender_id, body, sent_at FROM messages ' + 'WHERE conv_id = ? ORDER BY id', + _convParam, + ), + _StreamSpec( + 'feed page (DESC + LIMIT)', + 'feed_items', + 'SELECT id, author_id, created_at, body, like_count FROM feed_items ' + 'ORDER BY created_at DESC, id DESC LIMIT 50', + _noParams, + ), + _StreamSpec( + 'author drafts (int equality + ORDER BY pk)', + 'feed_items', + 'SELECT id, body, like_count FROM feed_items ' + 'WHERE author_id = ? ORDER BY id', + (i) => [i + 1], + ), +]; + +List _convParam(int instance) => [instance + 1]; +List _noParams(int instance) => const []; + +Future main() async { + final dir = await Directory.systemTemp.createTemp('resqlite_ivm_audit_'); + final db = await Database.open('${dir.path}/app.db'); + + await _setupSchema(db); + + // Direct classifier verdict per spec, against the real table_info. + final verdicts = {}; + for (final spec in _specs) { + final info = await db.select('PRAGMA table_info("${spec.table}")'); + final shape = classifyIvmQuery(spec.sql, spec.paramsFor(0), spec.table, info); + verdicts[spec.label] = shape != null; + } + + // Install the stream mix and wait for initial emissions. + final emissions = {for (final s in _specs) s.label: 0}; + final subs = >[]; + final initial = >[]; + for (final spec in _specs) { + for (var i = 0; i < _instancesPerSpec; i++) { + final first = Completer(); + initial.add(first.future); + subs.add( + db.stream(spec.sql, spec.paramsFor(i)).listen((_) { + if (!first.isCompleted) { + first.complete(); + } else { + emissions[spec.label] = emissions[spec.label]! + 1; + } + }), + ); + } + } + await Future.wait(initial).timeout(const Duration(seconds: 30)); + // Let async admission (PRAGMA round trips) settle before counting. + await Future.delayed(const Duration(milliseconds: 200)); + + // Admission happens at registration — snapshot before resetting for + // the burst counters. + final admissionSnap = ProfileCounters.snapshot(); + + // Chat-shaped write burst: 70% new message + conversation bump, + // 15% feed like, 10% profile edit, 5% message edit. New messages get + // sent_at above every seed row so DESC panes see them enter their + // windows, as they would in a real app. + final prng = math.Random(160); + ProfileCounters.reset(); + final sw = Stopwatch()..start(); + var nextMessageId = _seedMessages + 1; + for (var w = 0; w < _writeCount; w++) { + final roll = prng.nextInt(100); + if (roll < 70) { + final conv = prng.nextInt(_conversationCount) + 1; + await db.execute( + 'INSERT INTO messages(id, conv_id, sender_id, body, sent_at) ' + 'VALUES (?, ?, ?, ?, ?)', + [ + nextMessageId, + conv, + prng.nextInt(_userCount) + 1, + 'msg', + _seedMessages + w + 1, + ], + ); + nextMessageId++; + await db.execute( + 'UPDATE conversations SET last_msg_at = ? WHERE id = ?', + [_seedMessages + w + 1, conv], + ); + } else if (roll < 85) { + await db.execute( + 'UPDATE feed_items SET like_count = like_count + 1 WHERE id = ?', + [prng.nextInt(_feedItems) + 1], + ); + } else if (roll < 95) { + await db.execute('UPDATE users SET name = ? WHERE id = ?', [ + 'renamed_$w', + prng.nextInt(_userCount) + 1, + ]); + } else { + await db.execute('UPDATE messages SET body = ? WHERE id = ?', [ + 'edited_$w', + prng.nextInt(_seedMessages) + 1, + ]); + } + await Future.delayed(Duration.zero); + } + sw.stop(); + + // Quiet-window drain so trailing re-queries land before the snapshot. + var last = emissions.values.fold(0, (a, b) => a + b); + final deadline = DateTime.now().add(const Duration(seconds: 30)); + while (DateTime.now().isBefore(deadline)) { + await Future.delayed(const Duration(milliseconds: 100)); + final now = emissions.values.fold(0, (a, b) => a + b); + if (now == last) break; + last = now; + } + final snap = ProfileCounters.snapshot(); + + print('# IVM admission audit (app-shaped stream mix)\n'); + print( + '| stream | instances | tier-1 admitted | burst emissions |', + ); + print('|---|---:|---|---:|'); + for (final spec in _specs) { + print( + '| ${spec.label} | $_instancesPerSpec ' + '| ${verdicts[spec.label]! ? 'yes' : 'no'} ' + '| ${emissions[spec.label]} |', + ); + } + final admittedSpecs = verdicts.values.where((v) => v).length; + print('\nSpecs admitted: $admittedSpecs/${_specs.length} ' + '(${admittedSpecs * _instancesPerSpec}/${_specs.length * _instancesPerSpec} stream instances)'); + print('Burst wall: ${(sw.elapsedMicroseconds / 1000).toStringAsFixed(2)} ms ' + 'for $_writeCount write ops (writes issue 1-2 statements each)'); + print('\nEngine admission counters (at registration):\n'); + for (final key in ['ivm_admitted_total', 'ivm_rejected_total']) { + print('- `$key`: ${admissionSnap[key]}'); + } + print('\nEngine counters across the burst:\n'); + for (final key in [ + 'ivm_skipped_total', + 'ivm_applied_total', + 'ivm_bail_total', + 'invalidate_count', + 'completion_handler_count', + ]) { + print('- `$key`: ${snap[key]}'); + } + final decisions = + (snap['ivm_skipped_total'] ?? 0) + (snap['ivm_applied_total'] ?? 0); + print( + '\nPer-stream invalidation decisions resolved without a reader ' + 're-query: $decisions', + ); + + for (final sub in subs) { + await sub.cancel(); + } + await db.close(); + await dir.delete(recursive: true); + exit(0); +} + +Future _setupSchema(Database db) async { + await db.execute( + 'CREATE TABLE users(id INTEGER PRIMARY KEY, name TEXT NOT NULL, ' + 'avatar_url TEXT NOT NULL)', + ); + await db.execute( + 'CREATE TABLE conversations(id INTEGER PRIMARY KEY, ' + 'last_msg_at INTEGER NOT NULL)', + ); + await db.execute( + 'CREATE TABLE messages(id INTEGER PRIMARY KEY, conv_id INTEGER NOT NULL, ' + 'sender_id INTEGER NOT NULL, body TEXT NOT NULL, sent_at INTEGER NOT NULL)', + ); + await db.execute( + 'CREATE INDEX messages_conv_sent ON messages(conv_id, sent_at DESC)', + ); + await db.execute( + 'CREATE TABLE feed_items(id INTEGER PRIMARY KEY, ' + 'author_id INTEGER NOT NULL, created_at INTEGER NOT NULL, ' + 'body TEXT NOT NULL, like_count INTEGER NOT NULL)', + ); + await db.executeBatch( + 'INSERT INTO users(id, name, avatar_url) VALUES (?, ?, ?)', + [ + for (var i = 1; i <= _userCount; i++) + [i, 'user_$i', 'https://example.com/a/$i.png'], + ], + ); + await db.executeBatch( + 'INSERT INTO conversations(id, last_msg_at) VALUES (?, ?)', + [ + for (var i = 1; i <= _conversationCount; i++) [i, 0], + ], + ); + await db.executeBatch( + 'INSERT INTO messages(id, conv_id, sender_id, body, sent_at) ' + 'VALUES (?, ?, ?, ?, ?)', + [ + for (var i = 1; i <= _seedMessages; i++) + [ + i, + (i % _conversationCount) + 1, + (i % _userCount) + 1, + 'seed message $i', + i, + ], + ], + ); + await db.executeBatch( + 'INSERT INTO feed_items(id, author_id, created_at, body, like_count) ' + 'VALUES (?, ?, ?, ?, ?)', + [ + for (var i = 1; i <= _feedItems; i++) + [i, (i % _userCount) + 1, i, 'post $i', 0], + ], + ); +} diff --git a/benchmark/profile/results/exp-160-stream-delta-ivm-aggregate.md b/benchmark/profile/results/exp-160-stream-delta-ivm-aggregate.md index 2b26bb27..27d5d3f2 100644 --- a/benchmark/profile/results/exp-160-stream-delta-ivm-aggregate.md +++ b/benchmark/profile/results/exp-160-stream-delta-ivm-aggregate.md @@ -68,3 +68,17 @@ Pass 2 (order flipped): see experiment writeup. Raw per-run JSONs and tracelite artifacts are local-only under `build/tracelite-experiments/`. + +## Admission audit (app-shaped stream mix) + +`dart run -DRESQLITE_PROFILE=true benchmark/profile/ivm_admission_audit.dart` + +8 reactive-UI stream shapes x 10 instances over chat+feed schemas; 300 +chat-shaped write ops. Tier-1 admits 3/8 shapes (30/62 distinct entries +after dedup): pk-equality cards, int-equality transcripts, and +ORDER-BY-pk lists. Burst counters: ivm_skipped=2,948, ivm_applied=52, +ivm_bail=0 (3,000 decisions resolved without reader re-query); +completion_handler_count=3,138 reader replies remain, generated almost +entirely by the unadmitted DESC+LIMIT shapes (message panes, +conversation list, feed page). Tier-2 composite-key ordering + LIMIT +K+buffer targets exactly that remainder. diff --git a/experiments/160-stream-delta-ivm.md b/experiments/160-stream-delta-ivm.md index ffdd2ce1..87300f14 100644 --- a/experiments/160-stream-delta-ivm.md +++ b/experiments/160-stream-delta-ivm.md @@ -153,6 +153,41 @@ remove re-query *execution* rather than tuning its constant: 98% of A11c-overlap invalidation decisions become proven misses that never touch the reader pool. +### Admission on an app-shaped stream mix + +`benchmark/profile/ivm_admission_audit.dart` (new) builds the workload +the suite was missing: chat + feed schemas mirroring the tracelite +app-shaped scenarios (which exercise `select()`, not `stream()`), eight +reactive-UI stream shapes × 10 instances, and a chat-shaped write burst +(70% new message + conversation bump, 15% feed like, 10% profile edit, +5% message edit). + +| stream shape | admitted | burst emissions | +|---|---|---:| +| message pane (JOIN + DESC + LIMIT) | no | 56 | +| message pane, denormalized (DESC + LIMIT) | no | 47 | +| conversation list (DESC + LIMIT) | no | 650 | +| unread badge (aggregate) | no | 46 | +| user card (pk equality) | **yes** | 1 | +| full transcript (int equality + ORDER BY pk) | **yes** | 48 | +| feed page (DESC + LIMIT) | no | 20 | +| author drafts (int equality + ORDER BY pk) | **yes** | 3 | + +Counters: 30/62 distinct entries admitted (`ivm_admitted_total=30`, +`ivm_rejected_total=32`; dedup collapses identical-SQL instances). Burst: +`ivm_skipped=2,948 / applied=52 / bail=0` — 3,000 invalidation decisions +resolved without a reader re-query — against `completion_handler_count` +≈ 3,138 reader replies still generated by the unadmitted entries. + +The read: tier-1 admits roughly half the distinct entries in a realistic +mix, but the **rejected set holds the highest-churn screens** (message +panes, conversation list, feed — all `ORDER BY DESC LIMIT N` +shapes). Tier-2's first two steps (composite-key ordering with pk +tiebreak + LIMIT windows with a K+buffer cache) would convert exactly +those, flipping most of the remaining ~3,100 re-queries per burst into +skips/patches. That is the quantified case for tier-2 — and the bound on +what tier-1 alone can claim in real apps. + ## Future Notes - **Emission cadence**: IVM emits per write where the re-query path diff --git a/lib/src/profile_counters.dart b/lib/src/profile_counters.dart index d5fd2822..a5b73c2d 100644 --- a/lib/src/profile_counters.dart +++ b/lib/src/profile_counters.dart @@ -178,6 +178,13 @@ class ProfileCounters { static int ivmAppliedTotal = 0; static int ivmBailTotal = 0; + /// Classifier admission outcomes, one increment per registered stream + /// once its dependencies are known: admitted = the tier-1 classifier + /// accepted the query; rejected = outside the grammar (including + /// multi-table queries that never reach the classifier). + static int ivmAdmittedTotal = 0; + static int ivmRejectedTotal = 0; + /// Take a named snapshot of all counter values. static Map snapshot() => { 'rows_decoded': rowsDecoded, @@ -198,6 +205,8 @@ class ProfileCounters { 'ivm_skipped_total': ivmSkippedTotal, 'ivm_applied_total': ivmAppliedTotal, 'ivm_bail_total': ivmBailTotal, + 'ivm_admitted_total': ivmAdmittedTotal, + 'ivm_rejected_total': ivmRejectedTotal, }; /// Compute `after - before` for every key present in both snapshots. @@ -235,5 +244,7 @@ class ProfileCounters { ivmSkippedTotal = 0; ivmAppliedTotal = 0; ivmBailTotal = 0; + ivmAdmittedTotal = 0; + ivmRejectedTotal = 0; } } diff --git a/lib/src/row_deltas.dart b/lib/src/row_deltas.dart index dfa13abc..2d3fae1c 100644 --- a/lib/src/row_deltas.dart +++ b/lib/src/row_deltas.dart @@ -39,6 +39,38 @@ final class RowDelta { final List? newValues; } +/// Lazily-decoded view over one write cycle's delta bytes, grouped by +/// table. +/// +/// Decodes at most once no matter how many admitted streams consult it; +/// a malformed buffer reports `null` for every table so all consumers +/// fall back to re-query. Construct one per write cycle and discard — +/// it carries no cross-cycle state. +final class RowDeltaBatch { + RowDeltaBatch(this._bytes); + + final Uint8List _bytes; + bool _decoded = false; + Map>? _byTable; + + /// Deltas for [table]; `null` when the buffer is malformed or holds no + /// rows for the table. + List? forTable(String table) { + if (!_decoded) { + _decoded = true; + final rows = decodeRowDeltas(_bytes); + if (rows != null) { + final byTable = >{}; + for (final delta in rows) { + (byTable[delta.table] ??= []).add(delta); + } + _byTable = byTable; + } + } + return _byTable?[table]; + } +} + /// Decode a drained delta buffer. /// /// Returns `null` on any structural inconsistency — callers treat that diff --git a/lib/src/stream_engine.dart b/lib/src/stream_engine.dart index e657fbef..5c0241b1 100644 --- a/lib/src/stream_engine.dart +++ b/lib/src/stream_engine.dart @@ -192,8 +192,9 @@ final class StreamEngine { } } + final deltaBatch = deltas == null ? null : RowDeltaBatch(deltas); for (final entry in dirtyEntries) { - if (deltas != null && _tryIncrementalMaintain(entry, deltas)) { + if (deltaBatch != null && _tryIncrementalMaintain(entry, deltaBatch)) { continue; } entry.dirty = true; @@ -206,9 +207,6 @@ final class StreamEngine { } _flushQueue(); - _decodedDeltas = null; - _decodedDeltasSource = null; - _deltasByTable = null; } finally { if (kProfileMode) { invalidateSw!.stop(); @@ -245,43 +243,21 @@ final class StreamEngine { } } - /// Per-write-cycle memo for the decoded delta buffer, so N admitted - /// streams on the same table decode it once. Keyed by buffer identity; - /// cleared at the end of each [onDependencyChanges] pass. - List? _decodedDeltas; - Object? _decodedDeltasSource; - Map>? _deltasByTable; - /// `PRAGMA table_info` results per table, shared across admissions. final Map>>> _tableInfoCache = {}; - /// Attempt to maintain [entry]'s cached result from [deltas] instead of - /// re-querying. Returns true when the entry is fully handled for this - /// write cycle (deltas proven irrelevant, or result patched + emitted). - bool _tryIncrementalMaintain(StreamEntry entry, Uint8List deltas) { + /// Attempt to maintain [entry]'s cached result from the write cycle's + /// [batch] instead of re-querying. Returns true when the entry is fully + /// handled for this cycle (deltas proven irrelevant, or result patched + /// and emitted). + bool _tryIncrementalMaintain(StreamEntry entry, RowDeltaBatch batch) { final ivm = entry.ivm; if (ivm == null || entry.inFlight || entry.dirty) return false; - if (!identical(_decodedDeltasSource, deltas)) { - _decodedDeltasSource = deltas; - _decodedDeltas = decodeRowDeltas(deltas); - _deltasByTable = null; - final decoded = _decodedDeltas; - if (decoded != null) { - final byTable = >{}; - for (final delta in decoded) { - (byTable[delta.table] ??= []).add(delta); - } - _deltasByTable = byTable; - } - } - final byTable = _deltasByTable; - if (byTable == null) return false; // malformed buffer — fall back - - final tableDeltas = byTable[ivm.shape.table]; + final tableDeltas = batch.forTable(ivm.shape.table); if (tableDeltas == null || tableDeltas.isEmpty) { - // The table was reported dirty but capture saw no rows for it — - // contradicts a reliable buffer, so trust the dirty set. + // Malformed buffer, or the table was reported dirty with no + // captured rows — either way, trust the dirty set and re-query. return false; } @@ -324,10 +300,15 @@ final class StreamEngine { )); if (entry.subscribers.isEmpty || entry.ivm != null) return; final shape = classifyIvmQuery(entry.sql, entry.params, table, info); - if (shape == null) return; + if (shape == null) { + if (kProfileMode) ProfileCounters.ivmRejectedTotal++; + return; + } entry.ivm = IvmState(shape); + if (kProfileMode) ProfileCounters.ivmAdmittedTotal++; } catch (_) { // Classification is best-effort; the entry stays on re-query. + if (kProfileMode) ProfileCounters.ivmRejectedTotal++; } } @@ -361,9 +342,6 @@ final class StreamEngine { _unknownDepsEntries.clear(); _requeryQueue.clear(); _tableInfoCache.clear(); - _decodedDeltas = null; - _decodedDeltasSource = null; - _deltasByTable = null; } /// Create a new stream entry and return a subscriber stream. @@ -421,6 +399,8 @@ final class StreamEngine { // maintenance (exp 160). Admission is async and best-effort. if (tables.length == 1) { unawaited(_admitIvm(entry, tables.single.table)); + } else if (kProfileMode) { + ProfileCounters.ivmRejectedTotal++; } } diff --git a/test/profile_counters_test.dart b/test/profile_counters_test.dart index 8ca93242..de4fb44f 100644 --- a/test/profile_counters_test.dart +++ b/test/profile_counters_test.dart @@ -32,6 +32,8 @@ void main() { 'ivm_skipped_total': 0, 'ivm_applied_total': 0, 'ivm_bail_total': 0, + 'ivm_admitted_total': 0, + 'ivm_rejected_total': 0, }); ProfileCounters.reset(); @@ -55,6 +57,8 @@ void main() { 'ivm_skipped_total': 0, 'ivm_applied_total': 0, 'ivm_bail_total': 0, + 'ivm_admitted_total': 0, + 'ivm_rejected_total': 0, }); }); From e142936706498bebe6e4079ffb592ae3965fcb84 Mon Sep 17 00:00:00 2001 From: Dan Reynolds Date: Wed, 10 Jun 2026 13:38:52 -0400 Subject: [PATCH 03/10] Fix pre-existing diagnostics x reader data race surfaced by exp 160 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resqlite_db_status_total skips readers marked in_use, but that flag has been dead code since exp 030 moved workers to dedicated reader assignment — Database.diagnostics() was calling sqlite3_db_status on live NOMUTEX reader connections from the main isolate. SCHEMA_USED measures memory via the connection's pnBytesFreed dry-run mechanism; toggling it under a reader mid-query corrupts the reader's allocation accounting (flaky SEGV in sqlite3VdbeDelete, ~1-in-30 stream_test runs once exp 160's detached admission reads made readers reliably busy at diagnostics-poll time). Read workers now bracket each request with resqlite_reader_set_busy (atomic store, two leaf FFI calls per request, ~ns), making the existing busy guard real. The sacrifice path clears the bracket before Isolate.exit since exit skips finally. Bisection: crash gone with admission disabled (60/60); with the fix and admission enabled, 100/100 clean stress iterations. Co-Authored-By: Claude Fable 5 --- experiments/160-stream-delta-ivm.md | 20 ++++++++++++++++++++ experiments/JOURNAL.md | 20 ++++++++++++++++++++ native/resqlite.c | 6 ++++++ 3 files changed, 46 insertions(+) diff --git a/experiments/160-stream-delta-ivm.md b/experiments/160-stream-delta-ivm.md index 87300f14..b2ba6932 100644 --- a/experiments/160-stream-delta-ivm.md +++ b/experiments/160-stream-delta-ivm.md @@ -188,6 +188,26 @@ those, flipping most of the remaining ~3,100 re-queries per burst into skips/patches. That is the quantified case for tier-2 — and the bound on what tier-1 alone can claim in real apps. +### Surfaced pre-existing bug: diagnostics × reader data race + +CI caught a flaky reader-isolate SEGV (~1-in-30 stream_test runs) that +bisected to the admission PRAGMA — but the root cause predates this +experiment. `resqlite_db_status_total` skips readers whose `in_use` flag +is set, and that flag has been dead code since exp 030 moved workers to +dedicated reader assignment (exp 051 even documented the acquire path as +dead). `Database.diagnostics()` was therefore calling `sqlite3_db_status` +on live NOMUTEX reader connections from the main isolate; the +`SCHEMA_USED` op measures memory via the connection's `pnBytesFreed` +dry-run mechanism, and toggling that under a reader mid-query corrupts +the reader's own allocation accounting (crash: `sqlite3VdbeDelete` → +allocation-size read, null). Exp 160's detached admission reads made +"reader busy while a test polls `Diagnostics.streamLength`" the common +case, blowing the window open. Fixed here: read workers bracket each +request with `resqlite_reader_set_busy` (two ~ns leaf FFI calls), making +the existing busy guard real; the sacrifice path clears the bracket +before `Isolate.exit`. Validation: crash reproduced locally pre-fix; +100/100 clean stress iterations post-fix. + ## Future Notes - **Emission cadence**: IVM emits per write where the re-query path diff --git a/experiments/JOURNAL.md b/experiments/JOURNAL.md index 6dcd13b1..f1e04bae 100644 --- a/experiments/JOURNAL.md +++ b/experiments/JOURNAL.md @@ -209,6 +209,26 @@ elevated, re-run order-flipped before treating the flag as real — and before burning an ablation pass on a mechanism the workload may not even exercise.* +### A dead guard flag is a data race waiting for a traffic pattern + +[Exp 160](160-stream-delta-ivm.md)'s detached admission reads surfaced a +reader-isolate SEGV that root-caused to `Database.diagnostics()` reading +live NOMUTEX reader connections from the main isolate. The guard existed +— `resqlite_db_status_total` skips readers marked `in_use` — but +[exp 030](030-dedicated-reader-assignment.md) had made that flag dead +code long before, and [exp 051](051-lock-free-reader-pool.md) even +*documented* the acquire path as dead without anyone asking what the +flag was still guarding. The race stayed latent because nothing made +readers reliably busy at diagnostics-poll time until a new feature +changed the traffic pattern. + +*Reapplies whenever an experiment declares a code path dead. Before +archiving that fact, list what the dead path's side effects (flags, +locks, counters) were protecting — either remove the consumers or make +the signal real. A guard that silently stopped being maintained is +strictly worse than no guard, because readers of the consuming code +assume protection.* + ## How to add to this file Add an entry when an experiment surfaces a transferable lesson — something a diff --git a/native/resqlite.c b/native/resqlite.c index 3cc54396..d4e33790 100644 --- a/native/resqlite.c +++ b/native/resqlite.c @@ -1421,6 +1421,12 @@ void resqlite_deltas_mark_unreliable(resqlite_db* db) { delta_mark_unreliable(db); } +void resqlite_reader_set_busy(resqlite_db* db, int reader_id, int busy) { + if (!db || reader_id < 0 || reader_id >= db->reader_count) return; + atomic_store_explicit(&db->readers[reader_id].worker_busy, busy, + memory_order_release); +} + // Polish (post-2026-04): returns RESQLITE_DEPENDENCY_COUNT_UNKNOWN when // the cached entry's read-table dependencies are unreliable (overflow / OOM // during prepare). Zero would mean "stream has no table deps" → silent stuck From 59d56283cc2a9e7cf796f5959638e8e001bf88b9 Mon Sep 17 00:00:00 2001 From: Dan Reynolds Date: Wed, 10 Jun 2026 13:58:16 -0400 Subject: [PATCH 04/10] Harden IVM test waits against suite load The end-to-end tests used fixed 60ms settles, which flaked under full- suite concurrency when an initial query or fallback re-query took longer (observed: the first emission assertion seeing an empty list). Positive emission-count assertions now poll until the expected count (15s deadline) before asserting exactly; must-not-emit assertions wait for a quiet window (two consecutive 50ms windows with no new emissions) so in- flight work drains first. 8/8 clean full-suite runs after; ~1-in-3 flaked before. Co-Authored-By: Claude Fable 5 --- test/stream_ivm_test.dart | 89 +++++++++++++++++++++++++++------------ 1 file changed, 61 insertions(+), 28 deletions(-) diff --git a/test/stream_ivm_test.dart b/test/stream_ivm_test.dart index 757db65a..bef08b8a 100644 --- a/test/stream_ivm_test.dart +++ b/test/stream_ivm_test.dart @@ -319,11 +319,42 @@ void main() { } }); - /// Allow initial emission + async IVM admission + any in-flight - /// invalidations to settle. - Future settle() => Future.delayed( - const Duration(milliseconds: 60), - ); + /// Wait until [emissions] has stopped changing across consecutive + /// quiet windows, so in-flight admissions/re-queries drain before a + /// "no emission happened" assertion. + Future settleQuiet(List emissions) async { + final deadline = DateTime.now().add(const Duration(seconds: 15)); + var last = emissions.length; + var quietWindows = 0; + while (quietWindows < 2) { + if (DateTime.now().isAfter(deadline)) break; + await Future.delayed(const Duration(milliseconds: 50)); + if (emissions.length == last) { + quietWindows++; + } else { + quietWindows = 0; + last = emissions.length; + } + } + } + /// Wait until [emissions] reaches at least [count], then let any + /// trailing work drain. Use before positive emission-count + /// assertions — fixed delays flake under full-suite load where a + /// re-query (or the initial query) can take longer than any chosen + /// constant. + Future settleTo(List emissions, int count) async { + final deadline = DateTime.now().add(const Duration(seconds: 15)); + while (emissions.length < count) { + if (DateTime.now().isAfter(deadline)) { + fail( + 'expected $count emissions within 15s, saw ${emissions.length}', + ); + } + await Future.delayed(const Duration(milliseconds: 5)); + } + await settleQuiet(emissions); + } + /// Assert the latest emission matches a fresh query of the same SQL. Future expectMatchesSelect( @@ -344,17 +375,17 @@ void main() { final emissions = >>[]; final sub = db.stream(sql, params).listen(emissions.add); addTearDown(sub.cancel); - await settle(); + await settleTo(emissions, 1); expect(emissions, hasLength(1)); // Miss: out-of-range write must not emit. await db.execute('UPDATE items SET score = 999 WHERE id = 40'); - await settle(); + await settleQuiet(emissions); expect(emissions, hasLength(1)); // Patch: in-range projected column. await db.execute('UPDATE items SET score = 111 WHERE id = 15'); - await settle(); + await settleTo(emissions, 2); expect(emissions, hasLength(2)); await expectMatchesSelect(emissions, sql, params); expect( @@ -366,27 +397,27 @@ void main() { await db.execute( "INSERT INTO items(id, flag, score, name) VALUES (1000, 0, 5, 'x')", ); - await settle(); + await settleQuiet(emissions); expect(emissions, hasLength(2)); // out of range — no emission await db.execute('DELETE FROM items WHERE id = 1000'); - await settle(); + await settleQuiet(emissions); expect(emissions, hasLength(2)); await db.execute('DELETE FROM items WHERE id = 12'); - await settle(); + await settleTo(emissions, 3); expect(emissions, hasLength(3)); await expectMatchesSelect(emissions, sql, params); expect(emissions.last.map((r) => r['id']), isNot(contains(12))); // Rowid change: moves a row out of the range. await db.execute('UPDATE items SET id = 500 WHERE id = 15'); - await settle(); + await settleTo(emissions, 4); await expectMatchesSelect(emissions, sql, params); expect(emissions.last.map((r) => r['id']), isNot(contains(15))); // Rowid change: moves a row into the range. await db.execute('UPDATE items SET id = 12 WHERE id = 500'); - await settle(); + await settleTo(emissions, 5); await expectMatchesSelect(emissions, sql, params); expect(emissions.last.map((r) => r['id']), contains(12)); }); @@ -396,7 +427,7 @@ void main() { final emissions = >>[]; final sub = db.stream(sql, [15]).listen(emissions.add); addTearDown(sub.cancel); - await settle(); + await settleTo(emissions, 1); expect(emissions, hasLength(1)); for (var i = 0; i < 10; i++) { @@ -405,11 +436,11 @@ void main() { 20 + i, ]); } - await settle(); + await settleQuiet(emissions); expect(emissions, hasLength(1)); // all misses await db.execute('UPDATE items SET name = ? WHERE id = ?', ['hit', 15]); - await settle(); + await settleTo(emissions, 2); expect(emissions, hasLength(2)); await expectMatchesSelect(emissions, sql, [15]); }); @@ -421,7 +452,7 @@ void main() { final emissions = >>[]; final sub = db.stream(sql, const []).listen(emissions.add); addTearDown(sub.cancel); - await settle(); + await settleTo(emissions, 1); await db.execute( 'UPDATE items SET weight = 2.5, blob_col = ? WHERE id = 3', @@ -429,7 +460,7 @@ void main() { Uint8List.fromList([1, 2, 3]), ], ); - await settle(); + await settleTo(emissions, 2); await expectMatchesSelect(emissions, sql, const []); final patched = emissions.last.firstWhere((r) => r['id'] == 3); expect(patched['weight'], 2.5); @@ -443,7 +474,7 @@ void main() { final emissions = >>[]; final sub = db.stream(sql, const []).listen(emissions.add); addTearDown(sub.cancel); - await settle(); + await settleTo(emissions, 1); expect(emissions, hasLength(1)); await db.transaction((tx) async { @@ -451,7 +482,7 @@ void main() { await tx.execute('UPDATE items SET score = 2 WHERE id = 13'); await tx.execute('DELETE FROM items WHERE id = 17'); }); - await settle(); + await settleTo(emissions, 2); expect(emissions, hasLength(2)); await expectMatchesSelect(emissions, sql, const []); }); @@ -463,7 +494,7 @@ void main() { final emissions = >>[]; final sub = db.stream(sql, const []).listen(emissions.add); addTearDown(sub.cancel); - await settle(); + await settleTo(emissions, 1); await db.transaction((tx) async { await tx.execute('UPDATE items SET score = 1 WHERE id = 11'); @@ -477,7 +508,7 @@ void main() { } await tx.execute('UPDATE items SET score = 2 WHERE id = 14'); }); - await settle(); + await settleTo(emissions, 2); await expectMatchesSelect(emissions, sql, const []); final byId = {for (final r in emissions.last) r['id']: r}; expect(byId[11]!['score'], 1); @@ -492,7 +523,7 @@ void main() { final emissions = >>[]; final sub = db.stream(sql, const []).listen(emissions.add); addTearDown(sub.cancel); - await settle(); + await settleTo(emissions, 1); // 400 rows > RESQLITE_MAX_DELTA_ROWS (256) — capture goes unreliable. await db.executeBatch( @@ -501,7 +532,7 @@ void main() { for (var i = 100; i < 500; i++) [i, 0, 1, 'bulk_$i'], ], ); - await settle(); + await settleTo(emissions, 2); await expectMatchesSelect(emissions, sql, const []); expect(emissions.last, hasLength(450)); }); @@ -511,11 +542,11 @@ void main() { final emissions = >>[]; final sub = db.stream(sql, const []).listen(emissions.add); addTearDown(sub.cancel); - await settle(); + await settleTo(emissions, 1); expect(emissions, hasLength(1)); await db.execute("UPDATE items SET name = 'row_7' WHERE id = 8"); - await settle(); + await settleTo(emissions, 2); await expectMatchesSelect(emissions, sql, const []); expect(emissions.last, hasLength(2)); }); @@ -531,12 +562,14 @@ void main() { final subB = db.stream(sqlB, const []).listen(emissionsB.add); addTearDown(subA.cancel); addTearDown(subB.cancel); - await settle(); + await settleTo(emissionsA, 1); + await settleTo(emissionsB, 1); expect(emissionsA, hasLength(1)); expect(emissionsB, hasLength(1)); await db.execute('UPDATE items SET score = 1 WHERE id = 5'); - await settle(); + await settleTo(emissionsA, 2); + await settleQuiet(emissionsB); expect(emissionsA, hasLength(2)); expect(emissionsB, hasLength(1)); await expectMatchesSelect(emissionsA, sqlA, const []); From d2538916e24a4001d7872273a41be6e77b58fdc1 Mon Sep 17 00:00:00 2001 From: Dan Reynolds Date: Wed, 10 Jun 2026 14:14:25 -0400 Subject: [PATCH 05/10] Deduplicate resqlite_reader_set_busy after rebase The rebase onto main (which landed the diagnostics race fix via #156) re-applied the function on top of main's copy. Co-Authored-By: Claude Fable 5 --- native/resqlite.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/native/resqlite.c b/native/resqlite.c index d4e33790..3cc54396 100644 --- a/native/resqlite.c +++ b/native/resqlite.c @@ -1421,12 +1421,6 @@ void resqlite_deltas_mark_unreliable(resqlite_db* db) { delta_mark_unreliable(db); } -void resqlite_reader_set_busy(resqlite_db* db, int reader_id, int busy) { - if (!db || reader_id < 0 || reader_id >= db->reader_count) return; - atomic_store_explicit(&db->readers[reader_id].worker_busy, busy, - memory_order_release); -} - // Polish (post-2026-04): returns RESQLITE_DEPENDENCY_COUNT_UNKNOWN when // the cached entry's read-table dependencies are unreliable (overflow / OOM // during prepare). Zero would mean "stream has no table deps" → silent stuck From 124fbaecdcdea9ad1a02ebf581658ea982c655b3 Mon Sep 17 00:00:00 2001 From: Dan Reynolds Date: Wed, 10 Jun 2026 14:16:13 -0400 Subject: [PATCH 06/10] Regenerate experiment history post-rebase Co-Authored-By: Claude Fable 5 --- docs/experiments/history.json | 345 +++++++++++++++++++++++++++++++++- 1 file changed, 344 insertions(+), 1 deletion(-) diff --git a/docs/experiments/history.json b/docs/experiments/history.json index 6595a25d..7b0d53ca 100644 --- a/docs/experiments/history.json +++ b/docs/experiments/history.json @@ -1,5 +1,5 @@ { - "generated": "2026-06-10T11:35:50.140198", + "generated": "2026-06-10T14:15:07.234065", "runs": [ { "id": "baseline-before-authorizer-hooks", @@ -17447,6 +17447,330 @@ "repeatCount": 5, "sqlite3SingleInsertMedianMs": 1.401, "sqlite3SingleInsertP90Ms": 2.092 + }, + { + "id": "exp160-stream-delta-ivm", + "date": "2026-06-10", + "timestamp": "2026-06-10T11:25:55", + "label": "exp160-stream-delta-ivm", + "environment": { + "cwd": "/Users/dan/Coding/resqlite/.claude/worktrees/exp-160-stream-delta-ivm", + "hostname": "macbookpro.lan", + "os": "macos", + "osVersion": "Version 26.2 (Build 25C56)", + "dartVersion": "3.12.1", + "runtime": "dart-runtime", + "executable": "dart", + "resolvedExecutable": "/usr/local/Cellar/dart/3.12.1/libexec/bin/dart", + "script": "file:///Users/dan/Coding/resqlite/.claude/worktrees/exp-160-stream-delta-ivm/benchmark/run_release.dart", + "processors": 10, + "ci": false, + "githubActions": false, + "gitSha": "b32177adfadd7926fc3f3a61ae0c8e938e4c8c31", + "gitBranch": "exp-160-stream-delta-ivm", + "gitDirty": true, + "benchmarkMode": "release", + "includeSlow": false + }, + "metrics": { + "Select → Maps / 10 rows / resqlite select()": 0.021, + "Select → Maps / 10 rows / resqlite select() [main]": 0.001, + "Select → Maps / 100 rows / resqlite select()": 0.066, + "Select → Maps / 100 rows / resqlite select() [main]": 0.007, + "Select → Maps / 1000 rows / resqlite select()": 0.512, + "Select → Maps / 1000 rows / resqlite select() [main]": 0.064, + "Select → Maps / 10000 rows / resqlite select()": 8.708, + "Select → Maps / 10000 rows / resqlite select() [main]": 0.717, + "Select → JSON Bytes / 10 rows / resqlite + jsonEncode": 0.039, + "Select → JSON Bytes / 10 rows / resqlite + jsonEncode [main]": 0.02, + "Select → JSON Bytes / 10 rows / resqlite selectBytes()": 0.022, + "Select → JSON Bytes / 10 rows / resqlite selectBytes() [main]": 0.0, + "Select → JSON Bytes / 100 rows / resqlite + jsonEncode": 0.258, + "Select → JSON Bytes / 100 rows / resqlite + jsonEncode [main]": 0.191, + "Select → JSON Bytes / 100 rows / resqlite selectBytes()": 0.086, + "Select → JSON Bytes / 100 rows / resqlite selectBytes() [main]": 0.0, + "Select → JSON Bytes / 1000 rows / resqlite + jsonEncode": 2.372, + "Select → JSON Bytes / 1000 rows / resqlite + jsonEncode [main]": 1.888, + "Select → JSON Bytes / 1000 rows / resqlite selectBytes()": 0.7, + "Select → JSON Bytes / 1000 rows / resqlite selectBytes() [main]": 0.0, + "Select → JSON Bytes / 10000 rows / resqlite + jsonEncode": 32.654, + "Select → JSON Bytes / 10000 rows / resqlite + jsonEncode [main]": 20.903, + "Select → JSON Bytes / 10000 rows / resqlite selectBytes()": 7.114, + "Select → JSON Bytes / 10000 rows / resqlite selectBytes() [main]": 0.006, + "Schema Shapes (1000 rows) / Narrow (2 cols: id + int) / resqlite": 0.123, + "Schema Shapes (1000 rows) / Narrow (2 cols: id + int) / resqlite [main]": 0.028, + "Schema Shapes (1000 rows) / Wide (20 cols: mixed types) / resqlite": 1.237, + "Schema Shapes (1000 rows) / Wide (20 cols: mixed types) / resqlite [main]": 0.269, + "Schema Shapes (1000 rows) / Text-heavy (4 long TEXT cols) / resqlite": 0.947, + "Schema Shapes (1000 rows) / Text-heavy (4 long TEXT cols) / resqlite [main]": 0.072, + "Schema Shapes (1000 rows) / Numeric-heavy (5 numeric cols) / resqlite": 0.353, + "Schema Shapes (1000 rows) / Numeric-heavy (5 numeric cols) / resqlite [main]": 0.071, + "Schema Shapes (1000 rows) / Nullable (50% NULLs) / resqlite": 0.347, + "Schema Shapes (1000 rows) / Nullable (50% NULLs) / resqlite [main]": 0.069, + "Scaling (10 → 20,000 rows) / 10 rows / resqlite": 0.021, + "Scaling (10 → 20,000 rows) / 10 rows / resqlite [main]": 0.001, + "Scaling (10 → 20,000 rows) / 50 rows / resqlite": 0.042, + "Scaling (10 → 20,000 rows) / 50 rows / resqlite [main]": 0.004, + "Scaling (10 → 20,000 rows) / 100 rows / resqlite": 0.065, + "Scaling (10 → 20,000 rows) / 100 rows / resqlite [main]": 0.007, + "Scaling (10 → 20,000 rows) / 500 rows / resqlite": 0.276, + "Scaling (10 → 20,000 rows) / 500 rows / resqlite [main]": 0.036, + "Scaling (10 → 20,000 rows) / 1000 rows / resqlite": 0.529, + "Scaling (10 → 20,000 rows) / 1000 rows / resqlite [main]": 0.071, + "Scaling (10 → 20,000 rows) / 2000 rows / resqlite": 1.099, + "Scaling (10 → 20,000 rows) / 2000 rows / resqlite [main]": 0.14, + "Scaling (10 → 20,000 rows) / 5000 rows / resqlite": 3.361, + "Scaling (10 → 20,000 rows) / 5000 rows / resqlite [main]": 0.36, + "Scaling (10 → 20,000 rows) / 10000 rows / resqlite": 8.588, + "Scaling (10 → 20,000 rows) / 10000 rows / resqlite [main]": 0.737, + "Scaling (10 → 20,000 rows) / 20000 rows / resqlite": 19.172, + "Scaling (10 → 20,000 rows) / 20000 rows / resqlite [main]": 1.625, + "Scaling (10 → 20,000 rows) / 10 rows / resqlite + jsonEncode": 0.04, + "Scaling (10 → 20,000 rows) / 10 rows / resqlite + jsonEncode [main]": 0.04, + "Scaling (10 → 20,000 rows) / 10 rows / resqlite selectBytes()": 0.021, + "Scaling (10 → 20,000 rows) / 10 rows / resqlite selectBytes() [main]": 0.021, + "Scaling (10 → 20,000 rows) / 50 rows / resqlite + jsonEncode": 0.131, + "Scaling (10 → 20,000 rows) / 50 rows / resqlite + jsonEncode [main]": 0.131, + "Scaling (10 → 20,000 rows) / 50 rows / resqlite selectBytes()": 0.048, + "Scaling (10 → 20,000 rows) / 50 rows / resqlite selectBytes() [main]": 0.048, + "Scaling (10 → 20,000 rows) / 100 rows / resqlite + jsonEncode": 0.24, + "Scaling (10 → 20,000 rows) / 100 rows / resqlite + jsonEncode [main]": 0.24, + "Scaling (10 → 20,000 rows) / 100 rows / resqlite selectBytes()": 0.082, + "Scaling (10 → 20,000 rows) / 100 rows / resqlite selectBytes() [main]": 0.082, + "Scaling (10 → 20,000 rows) / 500 rows / resqlite + jsonEncode": 1.125, + "Scaling (10 → 20,000 rows) / 500 rows / resqlite + jsonEncode [main]": 1.125, + "Scaling (10 → 20,000 rows) / 500 rows / resqlite selectBytes()": 0.342, + "Scaling (10 → 20,000 rows) / 500 rows / resqlite selectBytes() [main]": 0.342, + "Scaling (10 → 20,000 rows) / 1000 rows / resqlite + jsonEncode": 2.203, + "Scaling (10 → 20,000 rows) / 1000 rows / resqlite + jsonEncode [main]": 2.203, + "Scaling (10 → 20,000 rows) / 1000 rows / resqlite selectBytes()": 0.657, + "Scaling (10 → 20,000 rows) / 1000 rows / resqlite selectBytes() [main]": 0.657, + "Scaling (10 → 20,000 rows) / 2000 rows / resqlite + jsonEncode": 5.783, + "Scaling (10 → 20,000 rows) / 2000 rows / resqlite + jsonEncode [main]": 5.783, + "Scaling (10 → 20,000 rows) / 2000 rows / resqlite selectBytes()": 1.704, + "Scaling (10 → 20,000 rows) / 2000 rows / resqlite selectBytes() [main]": 1.704, + "Scaling (10 → 20,000 rows) / 5000 rows / resqlite + jsonEncode": 19.581, + "Scaling (10 → 20,000 rows) / 5000 rows / resqlite + jsonEncode [main]": 19.581, + "Scaling (10 → 20,000 rows) / 5000 rows / resqlite selectBytes()": 4.22, + "Scaling (10 → 20,000 rows) / 5000 rows / resqlite selectBytes() [main]": 4.22, + "Scaling (10 → 20,000 rows) / 10000 rows / resqlite + jsonEncode": 32.354, + "Scaling (10 → 20,000 rows) / 10000 rows / resqlite + jsonEncode [main]": 32.354, + "Scaling (10 → 20,000 rows) / 10000 rows / resqlite selectBytes()": 8.079, + "Scaling (10 → 20,000 rows) / 10000 rows / resqlite selectBytes() [main]": 8.079, + "Scaling (10 → 20,000 rows) / 20000 rows / resqlite + jsonEncode": 65.593, + "Scaling (10 → 20,000 rows) / 20000 rows / resqlite + jsonEncode [main]": 65.593, + "Scaling (10 → 20,000 rows) / 20000 rows / resqlite selectBytes()": 16.827, + "Scaling (10 → 20,000 rows) / 20000 rows / resqlite selectBytes() [main]": 16.827, + "Concurrent Reads (1000 rows per query) / 1× concurrency / resqlite": 0.43, + "Concurrent Reads (1000 rows per query) / 1× concurrency / resqlite [main]": 0.43, + "Concurrent Reads (1000 rows per query) / 2× concurrency / resqlite": 0.48, + "Concurrent Reads (1000 rows per query) / 2× concurrency / resqlite [main]": 0.24, + "Concurrent Reads (1000 rows per query) / 4× concurrency / resqlite": 0.52, + "Concurrent Reads (1000 rows per query) / 4× concurrency / resqlite [main]": 0.13, + "Concurrent Reads (1000 rows per query) / 8× concurrency / resqlite": 0.98, + "Concurrent Reads (1000 rows per query) / 8× concurrency / resqlite [main]": 0.12, + "Point Query Throughput / resqlite qps": 70059.0, + "Parameterized Queries / 100 queries × ~500 rows each / resqlite": 21.398, + "Parameterized Queries / 100 queries × ~500 rows each / resqlite [main]": 21.398, + "Write Performance / Single Inserts (100 sequential) / resqlite execute()": 2.635, + "Write Performance / Single Inserts (100 sequential) / resqlite execute() [main]": 2.635, + "Write Performance / Batch Insert (100 rows) / resqlite executeBatch()": 0.098, + "Write Performance / Batch Insert (100 rows) / resqlite executeBatch() [main]": 0.098, + "Write Performance / Batch Insert (1000 rows) / resqlite executeBatch()": 0.61, + "Write Performance / Batch Insert (1000 rows) / resqlite executeBatch() [main]": 0.61, + "Write Performance / Batch Insert (10000 rows) / resqlite executeBatch()": 5.746, + "Write Performance / Batch Insert (10000 rows) / resqlite executeBatch() [main]": 5.746, + "Write Performance / Wide Batch Insert (10000 rows x 20 params) / resqlite executeBatch()": 19.178, + "Write Performance / Wide Batch Insert (10000 rows x 20 params) / resqlite executeBatch() [main]": 19.178, + "Write Performance / Interactive Transaction (insert + select + conditional delete) / resqlite transaction()": 0.086, + "Write Performance / Interactive Transaction (insert + select + conditional delete) / resqlite transaction() [main]": 0.086, + "Write Performance / Batched Write Inside Transaction (100 rows) / resqlite tx.executeBatch()": 0.15, + "Write Performance / Batched Write Inside Transaction (100 rows) / resqlite tx.executeBatch() [main]": 0.15, + "Write Performance / Batched Write Inside Transaction (100 rows) / resqlite tx.execute() loop": 1.401, + "Write Performance / Batched Write Inside Transaction (100 rows) / resqlite tx.execute() loop [main]": 1.401, + "Write Performance / Batched Write Inside Transaction (1000 rows) / resqlite tx.executeBatch()": 0.621, + "Write Performance / Batched Write Inside Transaction (1000 rows) / resqlite tx.executeBatch() [main]": 0.621, + "Write Performance / Batched Write Inside Transaction (1000 rows) / resqlite tx.execute() loop": 13.556, + "Write Performance / Batched Write Inside Transaction (1000 rows) / resqlite tx.execute() loop [main]": 13.556, + "Write Performance / Transaction Read (500 rows) / resqlite tx.select()": 0.159, + "Write Performance / Transaction Read (500 rows) / resqlite tx.select() [main]": 0.159, + "Write Performance / Transaction Read (1000 rows) / resqlite tx.select()": 0.273, + "Write Performance / Transaction Read (1000 rows) / resqlite tx.select() [main]": 0.273, + "Write Performance / Nested Transactions (savepoints) / resqlite nested transaction() x50": 2.941, + "Write Performance / Nested Transactions (savepoints) / resqlite nested transaction() x50 [main]": 2.941, + "Write Performance / Nested Transactions (savepoints) / resqlite nested transaction() depth=5": 0.188, + "Write Performance / Nested Transactions (savepoints) / resqlite nested transaction() depth=5 [main]": 0.188, + "Streaming / Initial Emission / resqlite stream()": 0.054, + "Streaming / Initial Emission / resqlite stream() [main]": 0.054, + "Streaming / Invalidation Latency / resqlite": 0.127, + "Streaming / Invalidation Latency / resqlite [main]": 0.127, + "Streaming / Unchanged Fanout Throughput (1 canary + 10 unchanged streams) / resqlite": 0.325, + "Streaming / Unchanged Fanout Throughput (1 canary + 10 unchanged streams) / resqlite [main]": 0.325, + "Streaming / Long-Text Unchanged Fanout (8 unchanged streams, 256 rows x 4KB TEXT) / resqlite": 0.135, + "Streaming / Long-Text Unchanged Fanout (8 unchanged streams, 256 rows x 4KB TEXT) / resqlite [main]": 0.135, + "Streaming / Fan-out (10 streams) / resqlite": 0.31, + "Streaming / Fan-out (10 streams) / resqlite [main]": 0.31, + "Streaming / Stream Churn (100 cycles) / resqlite": 3.433, + "Streaming / Stream Churn (100 cycles) / resqlite [main]": 3.433, + "Streaming / No-Streams Write Throughput (200 inserts, no active streams) / resqlite": 5.976, + "Streaming / No-Streams Write Throughput (200 inserts, no active streams) / resqlite [main]": 5.976, + "Streaming / Growing-Stream Invalidation (batch-insert 100 into watched stream) / resqlite": 0.173, + "Streaming / Growing-Stream Invalidation (batch-insert 100 into watched stream) / resqlite [main]": 0.173, + "Streaming / Stream Subscription Rate (500 subscribe+cancel cycles) / resqlite": 14.421, + "Streaming / Stream Subscription Rate (500 subscribe+cancel cycles) / resqlite [main]": 14.421, + "Keyed PK Subscriptions (v1) / 50 streams × 200 random-PK writes / resqlite stream()": 212.08, + "Keyed PK Subscriptions (v1) / 50 streams × 200 random-PK writes / resqlite stream() [main]": 0.0, + "Chat Sim (v1) / Insert message / resqlite": 0.049, + "Chat Sim (v1) / Insert message / resqlite [main]": 0.0, + "Chat Sim (v1) / Update conversation / resqlite": 0.039, + "Chat Sim (v1) / Update conversation / resqlite [main]": 0.0, + "Chat Sim (v1) / Fetch last-20 messages (JOIN users) / resqlite": 0.05, + "Chat Sim (v1) / Fetch last-20 messages (JOIN users) / resqlite [main]": 0.0, + "Chat Sim (v1) / Fetch user by PK / resqlite": 0.019, + "Chat Sim (v1) / Fetch user by PK / resqlite [main]": 0.0, + "Feed Paging (v1) / Keyset pagination (20 pages × 50 rows) / resqlite": 0.071, + "Feed Paging (v1) / Keyset pagination (20 pages × 50 rows) / resqlite [main]": 0.001, + "Feed Paging (v1) / Reactive feed with 100 concurrent writes / resqlite": 106.908, + "Feed Paging (v1) / Reactive feed with 100 concurrent writes / resqlite [main]": 0.0, + "High-Cardinality Stream Fan-out (v1) / 100 streams × 200 writes / resqlite": 230.33, + "High-Cardinality Stream Fan-out (v1) / 100 streams × 200 writes / resqlite [main]": 0.0 + }, + "memoryMetrics": { + "Memory / Select 10k rows → Maps / resqlite select()": { + "rssDeltaMedMB": 6.14, + "rssDeltaP90MB": 24.32, + "ciLowMB": 3.03, + "ciHighMB": 9.67, + "mdeMB": 3.32 + }, + "Memory / Select 10k rows → Maps / sqlite3 select()": { + "rssDeltaMedMB": 1.98, + "rssDeltaP90MB": 9.56, + "ciLowMB": 0.0, + "ciHighMB": 4.53, + "mdeMB": 2.27 + }, + "Memory / Select 10k rows → Maps / sqlite_async select()": { + "rssDeltaMedMB": 1.0, + "rssDeltaP90MB": 1.66, + "ciLowMB": 0.96, + "ciHighMB": 1.0, + "mdeMB": 0.02 + }, + "Memory / Select 10k rows → Maps / drift select()": { + "rssDeltaMedMB": 5.25, + "rssDeltaP90MB": 19.36, + "ciLowMB": 0.59, + "ciHighMB": 12.28, + "mdeMB": 5.85 + }, + "Memory / Select 10k rows → JSON Bytes / resqlite selectBytes()": { + "rssDeltaMedMB": 0.24, + "rssDeltaP90MB": 8.97, + "ciLowMB": 0.0, + "ciHighMB": 1.16, + "mdeMB": 0.58 + }, + "Memory / Select 10k rows → JSON Bytes / resqlite + jsonEncode": { + "rssDeltaMedMB": 7.83, + "rssDeltaP90MB": 67.64, + "ciLowMB": 0.0, + "ciHighMB": 47.54, + "mdeMB": 23.77 + }, + "Memory / Select 10k rows → JSON Bytes / sqlite3 + jsonEncode": { + "rssDeltaMedMB": 7.68, + "rssDeltaP90MB": 58.14, + "ciLowMB": 0.0, + "ciHighMB": 14.66, + "mdeMB": 7.33 + }, + "Memory / Select 10k rows → JSON Bytes / sqlite_async + jsonEncode": { + "rssDeltaMedMB": 2.38, + "rssDeltaP90MB": 26.09, + "ciLowMB": 0.0, + "ciHighMB": 10.2, + "mdeMB": 5.1 + }, + "Memory / Select 10k rows → JSON Bytes / drift + jsonEncode": { + "rssDeltaMedMB": 0.0, + "rssDeltaP90MB": 31.68, + "ciLowMB": 0.0, + "ciHighMB": 1.94, + "mdeMB": 0.97 + }, + "Memory / Batch insert 10k rows / resqlite executeBatch()": { + "rssDeltaMedMB": 6.73, + "rssDeltaP90MB": 12.64, + "ciLowMB": 0.0, + "ciHighMB": 9.56, + "mdeMB": 4.78 + }, + "Memory / Batch insert 10k rows / sqlite3 executeBatch()": { + "rssDeltaMedMB": 0.0, + "rssDeltaP90MB": 0.62, + "ciLowMB": 0.0, + "ciHighMB": 0.0, + "mdeMB": 0.0 + }, + "Memory / Batch insert 10k rows / sqlite_async executeBatch()": { + "rssDeltaMedMB": 0.29, + "rssDeltaP90MB": 4.98, + "ciLowMB": 0.0, + "ciHighMB": 0.54, + "mdeMB": 0.27 + }, + "Memory / Batch insert 10k rows / drift batch()": { + "rssDeltaMedMB": 0.01, + "rssDeltaP90MB": 2.0, + "ciLowMB": 0.0, + "ciHighMB": 0.11, + "mdeMB": 0.06 + }, + "Memory / Streaming fan-out (10 streams × 100 writes) / resqlite stream()": { + "rssDeltaMedMB": 0.0, + "rssDeltaP90MB": 0.13, + "ciLowMB": 0.0, + "ciHighMB": 0.12, + "mdeMB": 0.06 + }, + "Memory / Streaming fan-out (10 streams × 100 writes) / sqlite_async watch()": { + "rssDeltaMedMB": 0.0, + "rssDeltaP90MB": 0.91, + "ciLowMB": 0.0, + "ciHighMB": 0.03, + "mdeMB": 0.02 + } + }, + "sqliteDiagnosticsMetrics": { + "SQLite Diagnostics / Warm read working set (20000 rows + 2000 point lookups) / resqlite": { + "sqliteTotalKiB": 3210.4, + "pageCacheKiB": 3189.5, + "schemaKiB": 5.3, + "stmtKiB": 15.6, + "walKiB": 2048.0, + "readersBusy": 0 + }, + "SQLite Diagnostics / Statement cache footprint (48 distinct SELECT texts) / resqlite": { + "sqliteTotalKiB": 3299.6, + "pageCacheKiB": 3189.5, + "schemaKiB": 5.3, + "stmtKiB": 104.8, + "walKiB": 2048.0, + "readersBusy": 0 + }, + "SQLite Diagnostics / WAL after write burst (1000 inserted rows) / resqlite": { + "sqliteTotalKiB": 260.9, + "pageCacheKiB": 240.0, + "schemaKiB": 5.3, + "stmtKiB": 15.6, + "walKiB": 161.0, + "readersBusy": 0 + } + }, + "repeatCount": 5, + "sqlite3SingleInsertMedianMs": 1.702, + "sqlite3SingleInsertP90Ms": 4.769 } ], "experiments": [ @@ -19644,6 +19968,25 @@ "timestamp": "2026-06-09T22:41:01", "source": "exact" } + }, + { + "id": "160", + "title": "Tier-1 incremental stream maintenance (row deltas)", + "date": "2026-06-10", + "status": "in_review", + "summary": "**−18.5% many-streams / −14.1% keyed-PK** on the Tracelite stream gate (CIs −122..−96 ms and −60..−27 ms). Preupdate hook captures bounded per-row old/new values; a registration-time classifier admits single-table integer-predicate streams whose results the engine then maintains locally — A11c-overlap engagement: 24,500/25,000 invalidation decisions proven misses with zero reader re-queries, 0 bails. Everything outside the tier-1 grammar stays on the re-query path", + "commit": null, + "problem": "Every stream invalidation re-executes the full query. Hash suppression\n(exp 075/077) saves the decode and the emission, never the execution: a\nwrite that provably cannot affect a stream's result still costs a\nreader-pool round-trip (12 µs dispatch floor) plus a full SQLite\nre-execution per affected stream. On A11c overlap that is 50 re-queries\nper write — 25,000 per 500-write burst — of which exp 136 measured the\ncompletion-handler chain alone at 28.6% of total wall. Exp 134 proved the\nwin class (keyed-PK writer-burst wall halved when re-queries were elided\nby rowid precision) but was rejected because its *write-path SQL text\nrecognizer* was too fragile to trust. signals.json directed: revive\nrow-level precision \"only through explicit API/design or real workload\nevidence\", and exp 149 (...", + "hypothesis": "The preupdate hook already observes every modified row's old and new\nvalues — today they are discarded. If the writer ships bounded per-row\ndeltas alongside the dirty-table set, the stream engine can maintain\nadmitted streams' materialized results *incrementally*: prove most writes\nirrelevant with a few integer comparisons (no reader dispatch at all),\npatch in-window changes locally, and fall back to the existing re-query\npath for anything unprovable. Unlike exp 134's recognizer, admission is\ndecided once at stream registration against a sound-by-construction\ngrammar, and every uncertainty degrades to performance loss, never\ncorrectness loss. No public API changes — `db.stream()` is unchanged.", + "approach": "**Native capture** (`native/resqlite.c`): the preupdate hook serializes\nop + rowids + old/new column values into a bounded buffer (256 rows / 32\ncolumns / 256 KB per drain cycle; overflow or OOM poisons the cycle).\nSavepoint rollback poisons the cycle too — stale deltas are unsafe where\nstale dirty-tables are merely conservative. Drained alongside the dirty\nsets at statement end / outermost commit; discarded on rollback.\n\n**Plumbing**: `ExecuteResponse`/`BatchResponse` carry the raw bytes\n(`null` = unreliable → fallback); the engine decodes them lazily, once\nper write cycle, and only when at least one admitted stream watches a\ndirty table.\n\n**Tier-1 admission** (`lib/src/stream_ivm.dart`): at registration —\nasynchronously, off the hot path — the engine fetches\n`PRAGMA table_info` (cache...", + "results": "### Engagement check (profile counters, A11c-overlap shape)\n\n50 streams × 500 writes, every write touching a projected column:\n`ivm_skipped=24,500 ivm_applied=500 ivm_bail=0` — zero reader-pool\nre-queries for the entire burst\n(`benchmark/profile/ivm_engage_check.dart`).\n\n### Writer wall split audit (exp 147 harness, single pass, back-to-back)\n\n| workload | baseline wall | IVM wall | residual_us | emissions |\n|---|---:|---:|---|---:|\n| A11c baseline (0 streams) | 96.5 ms | 75.4 ms | 68,417 → 52,998 | — |\n| A11c disjoint | 100.6 ms | 104.2 ms | 55,669 → 62,360 | 0 → 0 |\n| A11c overlap | 186.5 ms | 132.0 ms | 135,071 → 47,966 | 44 → 500 |\n| keyed PK subscriptions | 46.9 ms | 25.5 ms | 28,340 → 10,287 | 3 → 3 |\n\nA11c overlap **−29%** wall with **11× more emissions delivered** (500 vs\n44): u...", + "reasoning": "**In Review (accept-shaped).** Two large, twice-reproduced measured-\nelapsed wins on the canonical stream gate — many-streams −18.5%/+22.2%\n(pass 1/pass 2 orientation), keyed-PK −14.1%/+9.4% — with the third\nscenario neutral after order-flipped adjudication, zero bails on every\nmeasured workload, dedicated equivalence tests (26) and the full suite\ngreen, and no public API change. This is the first stream experiment to\nremove re-query *execution* rather than tuning its constant: 98% of\nA11c-overlap invalidation decisions become proven misses that never\ntouch the reader pool.\n\n### Admission on an app-shaped stream mix\n\n`benchmark/profile/ivm_admission_audit.dart` (new) builds the workload\nthe suite was missing: chat + feed schemas mirroring the tracelite\napp-shaped scenarios (which exerci...", + "benchmarkRun": { + "id": "exp160-stream-delta-ivm", + "date": "2026-06-10", + "timestamp": "2026-06-10T11:25:55", + "source": "exact" + } } ], "tracked": [ From 86015bb480a2b584c6108b0545ad3ab30d77a37d Mon Sep 17 00:00:00 2001 From: Dan Reynolds Date: Wed, 10 Jun 2026 15:57:41 -0400 Subject: [PATCH 07/10] exp 160 tiers 1.5-3: skip-only, maintained windows, incremental aggregates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generalizes the registration-time classifier into three fail-closed admission modes sharing one strict grammar (now with string literals, aggregate calls, AS aliases, DESC, LIMIT/OFFSET, DISTINCT): - Full maintenance gains composite ordering (ORDER BY intCol [DESC], pk [DESC] — explicit pk tiebreak required so tie order is exact) and LIMIT-K windows: a top-K cache with complete-set tracking; entries and in-window patches are O(delta), departures from a full incomplete window fall back (the replacement row is unknown), boundary-crossing moves fall back. TEXT equality predicates are admitted when the table's CREATE statement (from sqlite_master, cached per table) contains no COLLATE clause, making BINARY semantics provable; the delta decoder's strict UTF-8 already rejects malformed text upstream. - Tier 1.5 skip-only: shapes whose results cannot be maintained (DESC without tiebreak, OFFSET, DISTINCT, unprojected keys) but whose WHERE is an evaluable conjunction get proven-miss elision with no cached state at all; any hit or unprovable cell re-queries. - Tier 3 aggregates: COUNT(*)/COUNT/SUM/MIN/MAX/AVG(col) AS alias over an evaluable (possibly empty) predicate, seeded exactly by a one-time snapshot query at admission and maintained per delta (NULL cells follow SQL semantics; integer sums exact; AVG derived as sum/count; a departing MIN/MAX extremum bails and re-seeds asynchronously). Engine: sealed IvmState dispatch, per-table meta cache (table_info + CREATE sql), windowed emissions compare by row identity so invisible tail changes don't emit, aggregate re-seed scheduling after bails. New counters: ivm_admitted_skip/agg_total, ivm_hit_fallback_total. 38 unit tests across classifier modes and all three state machines; full suite 312/312. Co-Authored-By: Claude Fable 5 --- lib/src/profile_counters.dart | 12 + lib/src/stream_engine.dart | 213 +++++-- lib/src/stream_ivm.dart | 1057 +++++++++++++++++++++++++------ test/profile_counters_test.dart | 6 + test/stream_ivm_test.dart | 690 +++++++++++++++----- 5 files changed, 1594 insertions(+), 384 deletions(-) diff --git a/lib/src/profile_counters.dart b/lib/src/profile_counters.dart index a5b73c2d..7efdff9a 100644 --- a/lib/src/profile_counters.dart +++ b/lib/src/profile_counters.dart @@ -183,8 +183,14 @@ class ProfileCounters { /// accepted the query; rejected = outside the grammar (including /// multi-table queries that never reach the classifier). static int ivmAdmittedTotal = 0; + static int ivmAdmittedSkipTotal = 0; + static int ivmAdmittedAggTotal = 0; static int ivmRejectedTotal = 0; + /// Skip-only entries whose write cycle contained a predicate hit (or an + /// unprovable cell) and fell back to a re-query. + static int ivmHitFallbackTotal = 0; + /// Take a named snapshot of all counter values. static Map snapshot() => { 'rows_decoded': rowsDecoded, @@ -206,7 +212,10 @@ class ProfileCounters { 'ivm_applied_total': ivmAppliedTotal, 'ivm_bail_total': ivmBailTotal, 'ivm_admitted_total': ivmAdmittedTotal, + 'ivm_admitted_skip_total': ivmAdmittedSkipTotal, + 'ivm_admitted_agg_total': ivmAdmittedAggTotal, 'ivm_rejected_total': ivmRejectedTotal, + 'ivm_hit_fallback_total': ivmHitFallbackTotal, }; /// Compute `after - before` for every key present in both snapshots. @@ -245,6 +254,9 @@ class ProfileCounters { ivmAppliedTotal = 0; ivmBailTotal = 0; ivmAdmittedTotal = 0; + ivmAdmittedSkipTotal = 0; + ivmAdmittedAggTotal = 0; ivmRejectedTotal = 0; + ivmHitFallbackTotal = 0; } } diff --git a/lib/src/stream_engine.dart b/lib/src/stream_engine.dart index 5c0241b1..218ea33f 100644 --- a/lib/src/stream_engine.dart +++ b/lib/src/stream_engine.dart @@ -243,8 +243,26 @@ final class StreamEngine { } } - /// `PRAGMA table_info` results per table, shared across admissions. - final Map>>> _tableInfoCache = {}; + /// Per-table admission metadata (`PRAGMA table_info` + the CREATE + /// statement from sqlite_master, which gates TEXT-equality admission on + /// the absence of COLLATE clauses), shared across admissions. + final Map< + String, + Future<(List>, String?)> + > _tableMetaCache = {}; + + Future<(List>, String?)> _tableMeta(String table) { + return _tableMetaCache[table] ??= () async { + final escaped = table.replaceAll('"', '""'); + final info = await _pool.select('PRAGMA table_info("$escaped")'); + final master = await _pool.select( + "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?", + [table], + ); + final createSql = master.isEmpty ? null : master.first['sql'] as String?; + return (info, createSql); + }(); + } /// Attempt to maintain [entry]'s cached result from the write cycle's /// [batch] instead of re-querying. Returns true when the entry is fully @@ -261,32 +279,90 @@ final class StreamEngine { return false; } - if (ivm.rows == null) { - final last = entry.lastResult; - if (last == null || !ivm.rebuild(last)) { - // The cached result cannot be keyed (non-int pk, unexpected - // ordering). This is structural, not transient — demote. - entry.ivm = null; + switch (ivm) { + case IvmSkipState(): + // Tier 1.5: prove all deltas miss, or fall back. There is no + // cache to drop on a hit. + if (ivm.apply(tableDeltas) == IvmOutcome.unchanged) { + if (kProfileMode) ProfileCounters.ivmSkippedTotal++; + return true; + } + if (kProfileMode) ProfileCounters.ivmHitFallbackTotal++; return false; - } + + case IvmFullState(): + if (ivm.rows == null) { + final last = entry.lastResult; + if (last == null || !ivm.rebuild(last)) { + // The cached result cannot be keyed (non-int pk/order key, + // unexpected ordering). Structural — demote. + entry.ivm = null; + return false; + } + } + switch (ivm.apply(tableDeltas)) { + case IvmOutcome.unchanged: + if (kProfileMode) ProfileCounters.ivmSkippedTotal++; + return true; + case IvmOutcome.applied: + final visible = ivm.visibleRows(); + // Patches confined to a complete window's invisible tail + // change the cache but not the emission. + final previous = entry.lastResult; + if (previous != null && _sameRowList(previous, visible)) { + entry.lastResult = visible; + entry.lastResultHash = _ivmStaleHash; + entry.lastRowCount = visible.length; + if (kProfileMode) ProfileCounters.ivmSkippedTotal++; + return true; + } + entry.lastResult = visible; + entry.lastResultHash = _ivmStaleHash; + entry.lastRowCount = visible.length; + entry.emit(visible); + if (kProfileMode) ProfileCounters.ivmAppliedTotal++; + return true; + case IvmOutcome.bail: + if (kProfileMode) ProfileCounters.ivmBailTotal++; + return false; + } + + case IvmAggregateState(): + if (!ivm.seeded) { + _scheduleAggregateReseed(entry, ivm); + return false; + } + switch (ivm.apply(tableDeltas)) { + case IvmOutcome.unchanged: + if (kProfileMode) ProfileCounters.ivmSkippedTotal++; + return true; + case IvmOutcome.applied: + final visible = [ivm.visibleRow()]; + entry.lastResult = visible; + entry.lastResultHash = _ivmStaleHash; + entry.lastRowCount = 1; + entry.emit(visible); + if (kProfileMode) ProfileCounters.ivmAppliedTotal++; + return true; + case IvmOutcome.bail: + _scheduleAggregateReseed(entry, ivm); + if (kProfileMode) ProfileCounters.ivmBailTotal++; + return false; + } } + } - switch (ivm.apply(tableDeltas)) { - case IvmOutcome.unchanged: - if (kProfileMode) ProfileCounters.ivmSkippedTotal++; - return true; - case IvmOutcome.applied: - final rows = ivm.rows!; - entry.lastResult = rows; - entry.lastResultHash = _ivmStaleHash; - entry.lastRowCount = rows.length; - entry.emit(rows); - if (kProfileMode) ProfileCounters.ivmAppliedTotal++; - return true; - case IvmOutcome.bail: - if (kProfileMode) ProfileCounters.ivmBailTotal++; - return false; + /// Rows-identical check by element identity (apply() clones any row it + /// changes, so identity captures "visibly unchanged" exactly). + bool _sameRowList( + List> a, + List> b, + ) { + if (a.length != b.length) return false; + for (var i = 0; i < a.length; i++) { + if (!identical(a[i], b[i])) return false; } + return true; } /// Try to admit [entry] for incremental maintenance. Best-effort and @@ -294,24 +370,78 @@ final class StreamEngine { /// plain re-query path. Future _admitIvm(StreamEntry entry, String table) async { try { - final escaped = table.replaceAll('"', '""'); - final info = await (_tableInfoCache[table] ??= _pool.select( - 'PRAGMA table_info("$escaped")', - )); + final (info, createSql) = await _tableMeta(table); if (entry.subscribers.isEmpty || entry.ivm != null) return; - final shape = classifyIvmQuery(entry.sql, entry.params, table, info); - if (shape == null) { - if (kProfileMode) ProfileCounters.ivmRejectedTotal++; - return; + final shape = classifyIvmQuery( + entry.sql, + entry.params, + table, + info, + createSql: createSql, + ); + switch (shape) { + case null: + if (kProfileMode) ProfileCounters.ivmRejectedTotal++; + case IvmFullShape(): + entry.ivm = IvmFullState(shape); + if (kProfileMode) ProfileCounters.ivmAdmittedTotal++; + case IvmSkipShape(): + entry.ivm = IvmSkipState(shape); + if (kProfileMode) { + ProfileCounters.ivmAdmittedTotal++; + ProfileCounters.ivmAdmittedSkipTotal++; + } + case IvmAggregateShape(): + final state = IvmAggregateState(shape); + await _seedAggregate(state, info); + if (entry.subscribers.isEmpty || entry.ivm != null) return; + if (state.seeded) { + entry.ivm = state; + if (kProfileMode) { + ProfileCounters.ivmAdmittedTotal++; + ProfileCounters.ivmAdmittedAggTotal++; + } + } else if (kProfileMode) { + ProfileCounters.ivmRejectedTotal++; + } } - entry.ivm = IvmState(shape); - if (kProfileMode) ProfileCounters.ivmAdmittedTotal++; } catch (_) { // Classification is best-effort; the entry stays on re-query. if (kProfileMode) ProfileCounters.ivmRejectedTotal++; } } + /// Seed (or re-seed) an aggregate state with an exact snapshot. + Future _seedAggregate( + IvmAggregateState state, + List> tableInfo, + ) async { + final snapshot = await _pool.select( + buildAggregateSnapshotSql(state.shape, tableInfo), + ); + if (snapshot.length == 1) { + state.seedFromSnapshot(snapshot.first); + } + } + + /// After an aggregate bail, re-seed asynchronously so a later write can + /// resume maintenance; re-queries cover the interim. + void _scheduleAggregateReseed(StreamEntry entry, IvmAggregateState state) { + if (state.reseedInFlight) return; + state.reseedInFlight = true; + unawaited(() async { + try { + final (info, _) = await _tableMeta(state.shape.table); + if (entry.subscribers.isEmpty) return; + await _seedAggregate(state, info); + } catch (_) { + // Stay unseeded; the entry keeps re-querying. + } finally { + state.reseedInFlight = false; + } + }()); + } + void _flushQueue() { if (_requeryQueue.isEmpty) { return; @@ -341,7 +471,7 @@ final class StreamEngine { _tableIndex.clear(); _unknownDepsEntries.clear(); _requeryQueue.clear(); - _tableInfoCache.clear(); + _tableMetaCache.clear(); } /// Create a new stream entry and return a subscriber stream. @@ -461,8 +591,17 @@ final class StreamEngine { entry.lastResult = rows; // A fresh query result invalidates the maintained cache; it is // rebuilt lazily from lastResult on the next applicable delta. - entry.ivm?.rows = null; - entry.ivm?.keys = null; + // A fresh query result invalidates maintained state; full caches + // rebuild lazily from lastResult, aggregates re-seed by snapshot. + switch (entry.ivm) { + case IvmFullState ivm: + ivm.rows = null; + ivm.keys = null; + case IvmAggregateState ivm: + ivm.seeded = false; + case IvmSkipState() || null: + break; + } entry.emit(rows); } catch (e, st) { diff --git a/lib/src/stream_ivm.dart b/lib/src/stream_ivm.dart index d6b06f7e..c0d9c04d 100644 --- a/lib/src/stream_ivm.dart +++ b/lib/src/stream_ivm.dart @@ -1,34 +1,41 @@ -/// Tier-1 incremental view maintenance for streams (exp 160). +/// Tiered incremental view maintenance for streams (exp 160). /// -/// A stream is *admitted* when its query falls inside a deliberately tiny -/// grammar whose semantics this module can mirror exactly: +/// A stream query is classified once, at registration, into one of three +/// admission modes — each strictly fail-closed (a classifier miss or an +/// unprovable condition at apply time costs a re-query, never +/// correctness): /// -/// SELECT FROM
-/// [WHERE (AND ...)*] -/// [ORDER BY [ASC]] +/// **Full maintenance** ([IvmFullShape]): single-table SELECT of bare +/// columns whose result order is fully determined (`ORDER BY pk`, +/// `ORDER BY intCol [DESC], pk [DESC]`, or a pk-equality pin), optionally +/// windowed by `LIMIT K`. The engine maintains the materialized result +/// from row deltas: proven misses skip the reader pool entirely, +/// in-window changes patch clone-on-write and emit. /// -/// with every comparison on INTEGER-typed values only, the table having a -/// single-column `INTEGER PRIMARY KEY` (a rowid alias), the key column in -/// the projection, and either an `ORDER BY ` or a `pk = ?` equality -/// predicate (so result order is fully determined). Everything outside the -/// grammar simply stays on the existing re-query path — a classifier miss -/// costs performance, never correctness. +/// **Skip-only** ([IvmSkipShape], tier 1.5): shapes whose *results* cannot +/// be maintained (DESC without a pk tiebreak, OFFSET, DISTINCT, free +/// ORDER BY, aggregate mixes) but whose WHERE clause is still a +/// conjunction of evaluable comparisons. Deltas that fail the predicate +/// before and after the write are proven misses — no re-query; any hit or +/// unprovable cell falls back to the normal re-query path. /// -/// For admitted streams, the engine maintains the materialized result by -/// applying the writer's row deltas: +/// **Aggregates** ([IvmAggregateShape], tier 3): projections consisting +/// solely of `COUNT(*) / COUNT(col) / SUM(col) / MIN(col) / MAX(col) / +/// AVG(col) AS alias` over an evaluable (possibly empty) predicate. State +/// is seeded exactly by a one-time snapshot query at admission and then +/// maintained per delta; a departing MIN/MAX extremum falls back. /// -/// * a delta row failing the predicate before AND after the write is a -/// *proven miss* — no reader dispatch, no SQLite, no hash; -/// * in-window updates / entries / departures patch the cached rows -/// locally and emit; -/// * anything unprovable (type mismatch, cache inconsistency, schema -/// drift) bails to the normal re-query path. +/// Comparison semantics are mirrored only where provably exact: INTEGER +/// cells against integer constants, and TEXT equality against string +/// constants when the table's CREATE statement contains no COLLATE clause +/// (BINARY collation; the delta decoder's strict UTF-8 decode rejects +/// malformed text upstream). Everything else stays on re-query. library; import 'row_deltas.dart'; // --------------------------------------------------------------------------- -// Query shape +// Predicates // --------------------------------------------------------------------------- final class IvmPredicate { @@ -37,69 +44,190 @@ final class IvmPredicate { /// Table column index (cid from `PRAGMA table_info`). final int columnIndex; - /// One of `=`, `<`, `<=`, `>`, `>=` (== normalizes to =). + /// One of `=`, `<`, `<=`, `>`, `>=` (`==` normalizes to `=`; only `=` + /// is admitted for TEXT values). final String op; - /// Comparison constant, resolved from a literal or the stream's fixed - /// bind parameters at classification time. - final int value; - - bool evaluate(int cell) => switch (op) { - '=' => cell == value, - '<' => cell < value, - '<=' => cell <= value, - '>' => cell > value, - '>=' => cell >= value, - _ => false, - }; + /// Comparison constant: an [int], or a [String] (TEXT equality under + /// verified BINARY collation). Resolved from a literal or the stream's + /// fixed bind parameters at classification time. + final Object value; + + /// Evaluates the predicate against a delta cell. Returns null when the + /// comparison is unprovable (type mismatch with the admitted constant). + /// A NULL cell never matches, mirroring SQL comparison semantics. + bool? evaluate(Object? cell) { + if (cell == null) return false; + final value = this.value; + if (value is int) { + if (cell is! int) return null; + return switch (op) { + '=' => cell == value, + '<' => cell < value, + '<=' => cell <= value, + '>' => cell > value, + '>=' => cell >= value, + _ => null, + }; + } + // TEXT equality (admission guarantees op == '='). + if (cell is! String) return null; + return cell == value as String; + } } -final class IvmShape { - const IvmShape({ +/// Evaluates a predicate conjunction against table-indexed [values]. +/// Returns null when any term is unprovable. +bool? evaluatePredicates(List predicates, List values) { + for (final pred in predicates) { + final match = pred.evaluate(values[pred.columnIndex]); + if (match == null) return null; + if (!match) return false; + } + return true; +} + +// --------------------------------------------------------------------------- +// Admission shapes +// --------------------------------------------------------------------------- + +sealed class IvmAdmission { + const IvmAdmission({ required this.table, required this.predicates, - required this.projection, - required this.pkColumnIndex, - required this.pkOutputName, required this.tableColumnCount, }); final String table; final List predicates; + /// `PRAGMA table_info` row count at classification time. A delta whose + /// column count differs means the schema changed under us — demote. + final int tableColumnCount; +} + +/// Fully-maintained result, optionally windowed by `LIMIT K`. +final class IvmFullShape extends IvmAdmission { + const IvmFullShape({ + required super.table, + required super.predicates, + required super.tableColumnCount, + required this.projection, + required this.pkColumnIndex, + required this.pkOutputName, + required this.orderColumnIndex, + required this.orderOutputName, + required this.orderDesc, + required this.pkDesc, + required this.limit, + }); + /// Output column name → table column index, in projection order. final List<(String, int)> projection; - /// Table column index of the INTEGER PRIMARY KEY (rowid alias). + /// Table column index of the INTEGER PRIMARY KEY (rowid alias) and its + /// name in emitted row maps. final int pkColumnIndex; - - /// The pk's name as it appears in emitted row maps. final String pkOutputName; - /// `PRAGMA table_info` row count at classification time. A delta whose - /// column count differs means the schema changed under us — demote. - final int tableColumnCount; + /// Primary order key (table cid + output name); equals the pk for + /// `ORDER BY pk` and pk-equality shapes. + final int orderColumnIndex; + final String orderOutputName; + final bool orderDesc; + + /// Tiebreak direction for the pk term. + final bool pkDesc; + + /// Window size, or null for an unwindowed (complete) result. + final int? limit; + + /// Compares composite sort keys per the admitted ORDER BY. + int compareKeys((int, int) a, (int, int) b) { + var c = a.$1.compareTo(b.$1); + if (orderDesc) c = -c; + if (c != 0) return c; + c = a.$2.compareTo(b.$2); + return pkDesc ? -c : c; + } +} + +/// Tier 1.5: predicates are evaluable but the result is not maintainable. +final class IvmSkipShape extends IvmAdmission { + const IvmSkipShape({ + required super.table, + required super.predicates, + required super.tableColumnCount, + }); +} + +enum IvmAggregateKind { countStar, count, sum, min, max, avg } + +final class IvmAggregate { + const IvmAggregate(this.kind, this.columnIndex, this.outputName); + + final IvmAggregateKind kind; + + /// Table column index; -1 for `COUNT(*)`. + final int columnIndex; + + /// The `AS` alias the value is emitted under. + final String outputName; +} + +/// Tier 3: an aggregates-only projection maintained from deltas. +final class IvmAggregateShape extends IvmAdmission { + const IvmAggregateShape({ + required super.table, + required super.predicates, + required super.tableColumnCount, + required this.aggregates, + }); + + final List aggregates; + + /// Whether exact seeding needs a snapshot query (anything beyond + /// COUNT(*) tracks per-column non-null counts and extrema). + bool get needsSnapshot => + aggregates.any((a) => a.kind != IvmAggregateKind.countStar); + + /// Distinct non-`*` aggregate column indexes, in first-seen order. + List get aggregateColumns { + final seen = {}; + final out = []; + for (final agg in aggregates) { + if (agg.columnIndex >= 0 && seen.add(agg.columnIndex)) { + out.add(agg.columnIndex); + } + } + return out; + } } // --------------------------------------------------------------------------- // Classifier // --------------------------------------------------------------------------- -/// Classify a stream query against the tier-1 grammar. +const _aggregateNames = {'count', 'sum', 'min', 'max', 'avg'}; + +/// Classify a stream query. /// -/// [tableInfo] is the result of `PRAGMA table_info(table)`. Returns `null` -/// when the query is not admissible. -IvmShape? classifyIvmQuery( +/// [tableInfo] is the result of `PRAGMA table_info(table)`. [createSql] +/// is the table's CREATE statement from `sqlite_master`; TEXT equality +/// predicates are admitted only when it is present and contains no +/// COLLATE clause (so every column provably uses BINARY collation). +/// Returns null when the query is not admissible in any mode. +IvmAdmission? classifyIvmQuery( String sql, List params, String table, - List> tableInfo, -) { - // Table metadata: name (declared case) → cid, and the single INTEGER - // PRIMARY KEY column. + List> tableInfo, { + String? createSql, +}) { if (tableInfo.isEmpty || tableInfo.length > 64) return null; final cidByLowerName = {}; final declaredNameByCid = {}; + final typeByCid = {}; var pkCid = -1; var pkCount = 0; for (final col in tableInfo) { @@ -110,25 +238,38 @@ IvmShape? classifyIvmQuery( if (cid is! int || name is! String || pk is! int) return null; cidByLowerName[name.toLowerCase()] = cid; declaredNameByCid[cid] = name; + typeByCid[cid] = type is String ? type.toUpperCase() : ''; if (pk > 0) { pkCount++; - if (pk == 1 && type is String && type.toUpperCase() == 'INTEGER') { + if (pk == 1 && typeByCid[cid] == 'INTEGER') { pkCid = cid; } } } - // Exactly one pk column, and it must be the INTEGER rowid alias. - if (pkCount != 1 || pkCid < 0) return null; + // The pk requirements only gate full maintenance; skip/aggregate modes + // work without a usable rowid alias. + final hasIntPk = pkCount == 1 && pkCid >= 0; + + final allowText = + createSql != null && !createSql.toLowerCase().contains('collate'); final tokens = _tokenize(sql); if (tokens == null) return null; final cursor = _Cursor(tokens); if (!cursor.takeKeyword('select')) return null; + final distinct = cursor.takeKeyword('distinct'); - // Projection. + // ---- Projection -------------------------------------------------------- + // Parsed into bare columns and/or aggregate calls. `fullEligible` decays + // as soon as anything beyond bare projected columns appears. final projection = <(String, int)>[]; + final aggregates = []; + var sawBareColumn = false; + var sawStar = false; + if (cursor.takeSymbol('*')) { + sawStar = true; for (var cid = 0; cid < tableInfo.length; cid++) { final name = declaredNameByCid[cid]; if (name == null) return null; @@ -138,10 +279,57 @@ IvmShape? classifyIvmQuery( while (true) { final ident = cursor.takeIdent(); if (ident == null) return null; - final cid = cidByLowerName[ident.toLowerCase()]; - if (cid == null) return null; - // Bare column reference: SQLite names the result column as written. - projection.add((ident, cid)); + if (cursor.takeSymbol('(')) { + // Aggregate call. + if (!_aggregateNames.contains(ident.toLowerCase())) return null; + final IvmAggregateKind kind; + int columnIndex; + if (cursor.takeSymbol('*')) { + if (ident.toLowerCase() != 'count') return null; + kind = IvmAggregateKind.countStar; + columnIndex = -1; + } else { + final argIdent = cursor.takeIdent(); + if (argIdent == null) return null; + final cid = cidByLowerName[argIdent.toLowerCase()]; + if (cid == null) return null; + columnIndex = cid; + kind = switch (ident.toLowerCase()) { + 'count' => IvmAggregateKind.count, + 'sum' => IvmAggregateKind.sum, + 'min' => IvmAggregateKind.min, + 'max' => IvmAggregateKind.max, + 'avg' => IvmAggregateKind.avg, + _ => IvmAggregateKind.count, // unreachable + }; + // Numeric aggregates only over INTEGER-declared columns; the + // apply path additionally bails on any non-int runtime cell. + if (kind != IvmAggregateKind.count && + typeByCid[cid] != 'INTEGER') { + return null; + } + } + if (!cursor.takeSymbol(')')) return null; + // Require an explicit alias so emitted names never depend on + // SQLite's expression-naming rules. + if (!cursor.takeKeyword('as')) return null; + final alias = cursor.takeIdent(); + if (alias == null) return null; + aggregates.add(IvmAggregate(kind, columnIndex, alias)); + } else { + final cid = cidByLowerName[ident.toLowerCase()]; + if (cid == null) return null; + sawBareColumn = true; + if (cursor.takeKeyword('as')) { + final alias = cursor.takeIdent(); + if (alias == null) return null; + projection.add((alias, cid)); + } else { + // Bare column reference: SQLite names the result column as + // written. + projection.add((ident, cid)); + } + } if (!cursor.takeSymbol(',')) break; } } @@ -152,7 +340,7 @@ IvmShape? classifyIvmQuery( return null; } - // WHERE conjunction. + // ---- WHERE ------------------------------------------------------------- final predicates = []; var paramIndex = 0; var hasPkEquality = false; @@ -164,63 +352,185 @@ IvmShape? classifyIvmQuery( if (cid == null) return null; final op = cursor.takeOperator(); if (op == null) return null; - final int value; + final Object value; if (cursor.takeSymbol('?')) { if (paramIndex >= params.length) return null; final param = params[paramIndex++]; - if (param is! int) return null; - value = param; + if (param is int) { + value = param; + } else if (param is String) { + if (!allowText || op != '=') return null; + value = param; + } else { + return null; + } } else { final literal = cursor.takeIntLiteral(); - if (literal == null) return null; - value = literal; + if (literal != null) { + value = literal; + } else { + final text = cursor.takeStringLiteral(); + if (text == null) return null; + if (!allowText || op != '=') return null; + value = text; + } } predicates.add(IvmPredicate(cid, op, value)); - if (op == '=' && cid == pkCid) hasPkEquality = true; + if (op == '=' && cid == pkCid && value is int) hasPkEquality = true; if (!cursor.takeKeyword('and')) break; } } - // Every bind parameter must be consumed by the WHERE clause — a `?` - // anywhere else was already rejected by the grammar, but a surplus - // parameter means the SQL used it somewhere we didn't model. - if (paramIndex != params.length) return null; - // ORDER BY — only the pk, only ascending. - var orderedByPk = false; + // ---- ORDER BY ---------------------------------------------------------- + final orderColumns = <(int, bool)>[]; // (cid, desc) if (cursor.takeKeyword('order')) { if (!cursor.takeKeyword('by')) return null; - final ident = cursor.takeIdent(); - if (ident == null) return null; - if (cidByLowerName[ident.toLowerCase()] != pkCid) return null; - cursor.takeKeyword('asc'); // optional - orderedByPk = true; + while (true) { + final ident = cursor.takeIdent(); + if (ident == null) return null; + final cid = cidByLowerName[ident.toLowerCase()]; + if (cid == null) return null; + var desc = false; + if (cursor.takeKeyword('desc')) { + desc = true; + } else { + cursor.takeKeyword('asc'); + } + orderColumns.add((cid, desc)); + if (!cursor.takeSymbol(',')) break; + } + } + + // ---- LIMIT / OFFSET ---------------------------------------------------- + int? limit; + var sawLimit = false; + var sawOffset = false; + if (cursor.takeKeyword('limit')) { + sawLimit = true; + if (cursor.takeSymbol('?')) { + if (paramIndex >= params.length) return null; + final param = params[paramIndex++]; + if (param is! int) return null; + limit = param; + } else { + limit = cursor.takeIntLiteral(); + if (limit == null) return null; + } + if (cursor.takeKeyword('offset')) { + sawOffset = true; + if (cursor.takeSymbol('?')) { + if (paramIndex >= params.length) return null; + paramIndex++; + } else if (cursor.takeIntLiteral() == null) { + return null; + } + } } cursor.takeSymbol(';'); if (!cursor.atEnd) return null; + // Every bind parameter must have been consumed by a position we model. + if (paramIndex != params.length) return null; + + IvmSkipShape? skipShape() => predicates.isEmpty + ? null + : IvmSkipShape( + table: table, + predicates: predicates, + tableColumnCount: tableInfo.length, + ); + + // ---- Aggregate admission ---------------------------------------------- + if (aggregates.isNotEmpty) { + final aggregateAdmissible = + !sawBareColumn && + !sawStar && + !distinct && + orderColumns.isEmpty && + !sawLimit; + if (aggregateAdmissible) { + return IvmAggregateShape( + table: table, + predicates: predicates, + tableColumnCount: tableInfo.length, + aggregates: aggregates, + ); + } + return skipShape(); + } - // Result order must be fully determined: either ordered by pk, or pinned - // to at most one row by a pk equality predicate. - if (!orderedByPk && !hasPkEquality) return null; + // ---- Full-maintenance admission ---------------------------------------- + var fullEligible = hasIntPk && !distinct && !sawOffset; + // Every predicate value must be provable; TEXT values were already + // gated, so nothing further here. - // The pk must be projected so cached rows can be keyed. + int orderCid = pkCid; + var orderDesc = false; + var pkDesc = false; + if (fullEligible) { + switch (orderColumns) { + case []: + // No ORDER BY: result order is only determined when pinned to at + // most one row. + if (!hasPkEquality) fullEligible = false; + case [(final cid, final desc)] when cid == pkCid: + orderCid = pkCid; + orderDesc = desc; + pkDesc = desc; + case [(final cid, final desc), (final cid2, final desc2)] + when cid2 == pkCid && cid != pkCid: + if (typeByCid[cid] != 'INTEGER') { + fullEligible = false; + } else { + orderCid = cid; + orderDesc = desc; + pkDesc = desc2; + } + default: + fullEligible = false; + } + } + + // The pk (and a non-pk order column) must be projected so cached rows + // can be keyed and re-sorted. String? pkOutputName; - for (final (name, cid) in projection) { - if (cid == pkCid) { - pkOutputName = name; - break; - } - } - if (pkOutputName == null) return null; - - return IvmShape( - table: table, - predicates: predicates, - projection: projection, - pkColumnIndex: pkCid, - pkOutputName: pkOutputName, - tableColumnCount: tableInfo.length, - ); + String? orderOutputName; + if (fullEligible) { + for (final (name, cid) in projection) { + if (cid == pkCid) pkOutputName ??= name; + if (cid == orderCid) orderOutputName ??= name; + } + if (pkOutputName == null || orderOutputName == null) { + fullEligible = false; + } + } + + if (fullEligible && sawLimit) { + final k = limit!; + if (hasPkEquality) { + // LIMIT over a ≤1-row pin is a no-op; treat as unwindowed. + limit = null; + } else if (k <= 0 || k > 1024 || orderColumns.isEmpty) { + fullEligible = false; + } + } + + if (fullEligible) { + return IvmFullShape( + table: table, + predicates: predicates, + tableColumnCount: tableInfo.length, + projection: projection, + pkColumnIndex: pkCid, + pkOutputName: pkOutputName!, + orderColumnIndex: orderCid, + orderOutputName: orderOutputName!, + orderDesc: orderDesc, + pkDesc: pkDesc, + limit: sawLimit ? limit : null, + ); + } + return skipShape(); } // --------------------------------------------------------------------------- @@ -229,7 +539,7 @@ IvmShape? classifyIvmQuery( final class _Token { const _Token(this.kind, this.text, [this.intValue = 0]); - final int kind; // 0 ident/keyword, 1 int literal, 2 symbol/operator + final int kind; // 0 ident/keyword, 1 int literal, 2 symbol/op, 3 string final String text; final int intValue; } @@ -269,6 +579,30 @@ List<_Token>? _tokenize(String sql) { tokens.add(_Token(1, text, value)); continue; } + // Single-quoted string literal with '' escaping. + if (c == 0x27) { + i++; + final buf = StringBuffer(); + var closed = false; + while (i < sql.length) { + final ch = sql.codeUnitAt(i); + if (ch == 0x27) { + if (i + 1 < sql.length && sql.codeUnitAt(i + 1) == 0x27) { + buf.writeCharCode(0x27); + i += 2; + continue; + } + i++; + closed = true; + break; + } + buf.writeCharCode(ch); + i++; + } + if (!closed) return null; + tokens.add(_Token(3, buf.toString())); + continue; + } // Operators / symbols. switch (c) { case 0x2a: // * @@ -283,6 +617,12 @@ List<_Token>? _tokenize(String sql) { case 0x3b: // ; tokens.add(const _Token(2, ';')); i++; + case 0x28: // ( + tokens.add(const _Token(2, '(')); + i++; + case 0x29: // ) + tokens.add(const _Token(2, ')')); + i++; case 0x3d: // = or == i++; if (i < sql.length && sql.codeUnitAt(i) == 0x3d) i++; @@ -304,7 +644,7 @@ List<_Token>? _tokenize(String sql) { tokens.add(const _Token(2, '>')); } default: - // Anything else (quotes, dots, parens, arithmetic, ...) is outside + // Anything else (double quotes, dots, arithmetic, ...) is outside // the grammar. return null; } @@ -354,6 +694,7 @@ const _reservedAsIdent = { 'collate', 'glob', 'exists', + 'over', }; final class _Cursor { @@ -414,62 +755,113 @@ final class _Cursor { pos++; return t.intValue; } + + String? takeStringLiteral() { + if (atEnd) return null; + final t = tokens[pos]; + if (t.kind != 3) return null; + pos++; + return t.text; + } } // --------------------------------------------------------------------------- -// Maintained state + delta application +// Maintained state // --------------------------------------------------------------------------- enum IvmOutcome { /// All deltas were proven irrelevant — nothing to emit, no re-query. unchanged, - /// The cached result was patched; the caller should emit it. + /// The cached state was patched; the caller should emit. applied, /// Something was unprovable or inconsistent — fall back to re-query. bail, } -final class IvmState { - IvmState(this.shape); +sealed class IvmState { + IvmAdmission get shape; +} + +/// Tier 1.5 state: nothing cached; only proves misses. +final class IvmSkipState extends IvmState { + IvmSkipState(this.shape); - final IvmShape shape; + @override + final IvmSkipShape shape; - /// Materialized rows, ascending by pk. `null` until built from the - /// entry's last result (and after any bail/fallback re-query). + /// [IvmOutcome.unchanged] when every delta row fails the predicate both + /// before and after the write; [IvmOutcome.bail] otherwise (a hit or an + /// unprovable cell — the caller re-queries; there is no cache to drop). + IvmOutcome apply(List deltas) { + for (final delta in deltas) { + final values = delta.newValues ?? delta.oldValues; + if (values == null || values.length != shape.tableColumnCount) { + return IvmOutcome.bail; + } + final oldMatch = delta.oldValues == null + ? false + : evaluatePredicates(shape.predicates, delta.oldValues!); + final newMatch = delta.newValues == null + ? false + : evaluatePredicates(shape.predicates, delta.newValues!); + if (oldMatch != false || newMatch != false) return IvmOutcome.bail; + } + return IvmOutcome.unchanged; + } +} + +/// Fully-maintained rows (optionally a top-K window). +final class IvmFullState extends IvmState { + IvmFullState(this.shape); + + @override + final IvmFullShape shape; + + /// Maintained rows in admitted order. For windowed shapes this is the + /// top-K (or the complete filtered set when it is smaller). `null` + /// until built from the entry's last result, and after any bail. List>? rows; - /// Sorted pk values parallel to [rows]. - List? keys; + /// Composite (orderKey, pk) sort keys parallel to [rows]. + List<(int, int)>? keys; + + /// Windowed only: whether [rows] holds the *entire* filtered set (so + /// departures never need a re-query). Unwindowed shapes are always + /// complete by construction. + bool complete = true; /// Build the maintained cache from the last emitted result. Returns - /// false (leaving the cache unset) when the rows cannot be keyed. + /// false (leaving the cache unset) when the rows cannot be keyed or + /// are not in admitted order. bool rebuild(List> lastResult) { final newRows = List>.generate( lastResult.length, (i) => Map.of(lastResult[i]), growable: true, ); - final newKeys = []; + final newKeys = <(int, int)>[]; for (final row in newRows) { - final key = row[shape.pkOutputName]; - if (key is! int) return false; - if (newKeys.isNotEmpty && key <= newKeys.last) { - // Result was not strictly ascending by pk — never patch it. + final pk = row[shape.pkOutputName]; + final orderKey = row[shape.orderOutputName]; + if (pk is! int || orderKey is! int) return false; + final key = (orderKey, pk); + if (newKeys.isNotEmpty && shape.compareKeys(newKeys.last, key) >= 0) { + // Not strictly ascending in admitted order — never patch it. return false; } newKeys.add(key); } + final limit = shape.limit; + if (limit != null && newRows.length > limit) return false; rows = newRows; keys = newKeys; + complete = limit == null || newRows.length < limit; return true; } /// Apply [deltas] (already filtered to this shape's table). - /// - /// On [IvmOutcome.applied], [rows] is a *fresh* list (previously emitted - /// lists are never mutated). On [IvmOutcome.bail], the cache is cleared. IvmOutcome apply(List deltas) { var currentRows = rows; var currentKeys = keys; @@ -477,84 +869,147 @@ final class IvmState { var mutated = false; List> workRows = currentRows; - List workKeys = currentKeys; + List<(int, int)> workKeys = currentKeys; + final limit = shape.limit; void ensureMutable() { if (!mutated) { workRows = List>.of(workRows); - workKeys = List.of(workKeys); + workKeys = List<(int, int)>.of(workKeys); mutated = true; } } - bool? evalPredicates(List values) { - for (final pred in shape.predicates) { - final cell = values[pred.columnIndex]; - if (cell == null) return false; // SQL comparison with NULL → no match - if (cell is! int) return null; // unprovable type - if (!pred.evaluate(cell)) return false; - } - return true; - } - bool applyOne( bool hasOld, bool hasNew, - int oldKey, - int newKey, + int oldPk, + int newPk, List? oldValues, List? newValues, ) { - final pOldOrNull = hasOld ? evalPredicates(oldValues!) : false; - final pNewOrNull = hasNew ? evalPredicates(newValues!) : false; + final pOldOrNull = hasOld + ? evaluatePredicates(shape.predicates, oldValues!) + : false; + final pNewOrNull = hasNew + ? evaluatePredicates(shape.predicates, newValues!) + : false; if (pOldOrNull == null || pNewOrNull == null) return false; final pOld = pOldOrNull; final pNew = pNewOrNull; - final oldIdx = _binarySearch(workKeys, oldKey); + (int, int)? oldKey; + if (hasOld && pOld) { + final cell = oldValues[shape.orderColumnIndex]; + if (cell is! int) return false; + oldKey = (cell, oldPk); + } + (int, int)? newKey; + if (hasNew && pNew) { + final cell = newValues[shape.orderColumnIndex]; + if (cell is! int) return false; + newKey = (cell, newPk); + } + + final oldIdx = oldKey == null ? null : _search(workKeys, oldKey); + // For diagnostics on misses, locate the row by pk regardless of the + // (possibly changed) order key. if (!pOld && !pNew) { - // Proven miss — but if the cache claims the row is present, it is + // Proven miss — but a cached row with this pk means the cache is // out of sync with reality. - if (hasOld && oldIdx >= 0) return false; + if (hasOld && _pkPresent(workRows, oldPk)) return false; return true; } if (pOld && pNew) { - // Row stays in the result; keys are equal here (rowid changes were - // split by the caller). - if (oldIdx < 0) return false; + // The row stays in the filtered set; it may move or patch. + final foundIdx = oldIdx != null && oldIdx >= 0 + ? oldIdx + : _pkIndex(workRows, oldPk); + if (foundIdx < 0) { + // Below an incomplete window before the write. + if (limit == null || complete) return false; + // It may enter the window now. + final lastKey = workKeys.isEmpty ? null : workKeys.last; + if (workKeys.length >= limit && + lastKey != null && + shape.compareKeys(newKey!, lastKey) >= 0) { + return true; // still below the window + } + return _insertRow( + ensureMutable, + () => workRows, + () => workKeys, + newKey!, + newValues, + limit, + ); + } final patched = _projectRow(newValues); if (patched == null) return false; - if (_projectedEquals(workRows[oldIdx], patched)) return true; + final keyChanged = workKeys[foundIdx] != newKey; + if (!keyChanged && _projectedEquals(workRows[foundIdx], patched)) { + return true; + } ensureMutable(); - workRows[oldIdx] = patched; + workRows.removeAt(foundIdx); + workKeys.removeAt(foundIdx); + if (limit != null && + !complete && + workKeys.isNotEmpty && + shape.compareKeys(newKey!, workKeys.last) >= 0 && + workKeys.length + 1 >= limit) { + // Moving to/past the boundary of an incomplete window — the true + // occupant of the freed slot is unknown. + return false; + } + final insertAt = _insertionPoint(workKeys, newKey!); + workRows.insert(insertAt, patched); + workKeys.insert(insertAt, newKey); return true; } if (pOld) { - // Row leaves the result. - if (oldIdx < 0) return false; + // Row leaves the filtered set. + final foundIdx = oldIdx != null && oldIdx >= 0 + ? oldIdx + : _pkIndex(workRows, oldPk); + if (foundIdx < 0) { + // Below the window of an incomplete windowed cache: invisible. + if (limit != null && !complete) return true; + return false; + } ensureMutable(); - workRows.removeAt(oldIdx); - workKeys.removeAt(oldIdx); + workRows.removeAt(foundIdx); + workKeys.removeAt(foundIdx); + if (limit != null && !complete && workRows.length < limit) { + // The window lost a member and the replacement is unknown. + return false; + } return true; } - // Row enters the result. - final newIdx = _binarySearch(workKeys, newKey); - if (newIdx >= 0) return false; - final inserted = _projectRow(newValues!); - if (inserted == null) return false; - ensureMutable(); - final insertAt = -(newIdx + 1); - workRows.insert(insertAt, inserted); - workKeys.insert(insertAt, newKey); - return true; + // Row enters the filtered set. + if (_pkPresent(workRows, newPk)) return false; + if (limit != null && !complete) { + final lastKey = workKeys.isEmpty ? null : workKeys.last; + if (workKeys.length >= limit && + lastKey != null && + shape.compareKeys(newKey!, lastKey) >= 0) { + return true; // enters below the window — invisible + } + } + return _insertRow( + ensureMutable, + () => workRows, + () => workKeys, + newKey!, + newValues!, + limit, + ); } for (final delta in deltas) { - // Schema drift guard: the capture's column count must match the - // shape's view of the table. final values = delta.newValues ?? delta.oldValues; if (values == null || values.length != shape.tableColumnCount) { return _bail(); @@ -598,21 +1053,60 @@ final class IvmState { } if (!mutated) return IvmOutcome.unchanged; + + // Complete windowed caches may grow past the window; keep them bounded. + if (limit != null && complete && workRows.length > limit + 64) { + workRows = workRows.sublist(0, limit); + workKeys = workKeys.sublist(0, limit); + complete = false; + } + rows = workRows; keys = workKeys; return IvmOutcome.applied; } + /// The result to emit: the visible window of the maintained rows. + List> visibleRows() { + final limit = shape.limit; + final all = rows!; + if (limit == null || all.length <= limit) return all; + return all.sublist(0, limit); + } + + bool _insertRow( + void Function() ensureMutable, + List> Function() getRows, + List<(int, int)> Function() getKeys, + (int, int) key, + List values, + int? limit, + ) { + final inserted = _projectRow(values); + if (inserted == null) return false; + ensureMutable(); + final rows = getRows(); + final keys = getKeys(); + final insertAt = _insertionPoint(keys, key); + rows.insert(insertAt, inserted); + keys.insert(insertAt, key); + if (limit != null && !complete && rows.length > limit) { + // The displaced row is no longer the window's business. + rows.removeLast(); + keys.removeLast(); + } + return true; + } + IvmOutcome _bail() { rows = null; keys = null; return IvmOutcome.bail; } - /// Build an emitted row from table-indexed [values] via the projection. - /// Returns null when the pk cell is not an int (unprovable). Map? _projectRow(List values) { if (values[shape.pkColumnIndex] is! int) return null; + if (values[shape.orderColumnIndex] is! int) return null; final row = {}; for (final (name, cid) in shape.projection) { row[name] = values[cid]; @@ -620,32 +1114,237 @@ final class IvmState { return row; } - /// Whether two projected rows are provably identical. Non-primitive - /// values (blobs) compare by identity, which can only produce a false - /// "changed" — never a false "unchanged". bool _projectedEquals(Map a, Map b) { for (final (name, _) in shape.projection) { if (a[name] != b[name]) return false; } return true; } + + int _search(List<(int, int)> keys, (int, int) key) { + var lo = 0; + var hi = keys.length - 1; + while (lo <= hi) { + final mid = (lo + hi) >> 1; + final c = shape.compareKeys(keys[mid], key); + if (c < 0) { + lo = mid + 1; + } else if (c > 0) { + hi = mid - 1; + } else { + return mid; + } + } + return -(lo + 1); + } + + int _insertionPoint(List<(int, int)> keys, (int, int) key) { + final idx = _search(keys, key); + return idx >= 0 ? idx : -(idx + 1); + } + + bool _pkPresent(List> rows, int pk) => + _pkIndex(rows, pk) >= 0; + + int _pkIndex(List> rows, int pk) { + for (var i = 0; i < rows.length; i++) { + if (rows[i][shape.pkOutputName] == pk) return i; + } + return -1; + } } -/// Standard binary search: index when found, `-(insertion point) - 1` -/// when absent. -int _binarySearch(List keys, int key) { - var lo = 0; - var hi = keys.length - 1; - while (lo <= hi) { - final mid = (lo + hi) >> 1; - final v = keys[mid]; - if (v < key) { - lo = mid + 1; - } else if (v > key) { - hi = mid - 1; - } else { - return mid; +/// Tier 3 state: exact aggregate values maintained from deltas. +final class IvmAggregateState extends IvmState { + IvmAggregateState(this.shape); + + @override + final IvmAggregateShape shape; + + /// Whether the state has been seeded (from the entry's first result for + /// COUNT(*)-only shapes, otherwise from the admission snapshot query). + bool seeded = false; + + /// Guards concurrent re-seed scheduling after a bail. + bool reseedInFlight = false; + + /// Total rows matching the predicate. + int rowCount = 0; + + /// Per aggregate column (by cid): non-null count, exact integer sum, + /// and current extrema. + final Map nonNullCount = {}; + final Map sum = {}; + final Map minValue = {}; + final Map maxValue = {}; + + /// Seed from the admission snapshot row (see + /// `buildAggregateSnapshotSql`). Returns false when the snapshot has an + /// unusable shape. + bool seedFromSnapshot(Map snapshot) { + final count = snapshot['__rows']; + if (count is! int) return false; + rowCount = count; + for (final cid in shape.aggregateColumns) { + final n = snapshot['__n$cid']; + if (n is! int) return false; + nonNullCount[cid] = n; + final s = snapshot['__s$cid']; + if (s is int) { + sum[cid] = s; + } else if (s == null) { + sum[cid] = 0; + } else { + return false; // non-integer sum (REAL cells present) + } + final lo = snapshot['__lo$cid']; + final hi = snapshot['__hi$cid']; + if ((lo is! int && lo != null) || (hi is! int && hi != null)) { + return false; + } + minValue[cid] = lo as int?; + maxValue[cid] = hi as int?; + } + seeded = true; + return true; + } + + IvmOutcome apply(List deltas) { + if (!seeded) return IvmOutcome.bail; + + // Work on copies so a mid-batch bail leaves the state unseeded + // rather than torn. + var newRowCount = rowCount; + final newNonNull = Map.of(nonNullCount); + final newSum = Map.of(sum); + final newMin = Map.of(minValue); + final newMax = Map.of(maxValue); + var mutated = false; + + bool contribute(List values, int sign) { + newRowCount += sign; + mutated = true; + for (final cid in shape.aggregateColumns) { + final cell = values[cid]; + if (cell == null) continue; + if (cell is! int) return false; + newNonNull[cid] = newNonNull[cid]! + sign; + newSum[cid] = newSum[cid]! + sign * cell; + final lo = newMin[cid]; + final hi = newMax[cid]; + if (sign > 0) { + if (lo == null || cell < lo) newMin[cid] = cell; + if (hi == null || cell > hi) newMax[cid] = cell; + } else { + if (newNonNull[cid] == 0) { + newMin[cid] = null; + newMax[cid] = null; + } else if (cell == lo || cell == hi) { + // The departing cell was an extremum; the next one is unknown. + return false; + } + } + } + return true; + } + + for (final delta in deltas) { + final values = delta.newValues ?? delta.oldValues; + if (values == null || values.length != shape.tableColumnCount) { + return _bail(); + } + if (delta.oldValues != null && + delta.oldValues!.length != shape.tableColumnCount) { + return _bail(); + } + final oldMatch = delta.oldValues == null + ? false + : evaluatePredicates(shape.predicates, delta.oldValues!); + final newMatch = delta.newValues == null + ? false + : evaluatePredicates(shape.predicates, delta.newValues!); + if (oldMatch == null || newMatch == null) return _bail(); + if (oldMatch && !contribute(delta.oldValues!, -1)) return _bail(); + if (newMatch && !contribute(delta.newValues!, 1)) return _bail(); + } + + if (!mutated) return IvmOutcome.unchanged; + rowCount = newRowCount; + nonNullCount + ..clear() + ..addAll(newNonNull); + sum + ..clear() + ..addAll(newSum); + minValue + ..clear() + ..addAll(newMin); + maxValue + ..clear() + ..addAll(newMax); + return IvmOutcome.applied; + } + + /// The single-row result to emit. + Map visibleRow() { + final row = {}; + for (final agg in shape.aggregates) { + final cid = agg.columnIndex; + row[agg.outputName] = switch (agg.kind) { + IvmAggregateKind.countStar => rowCount, + IvmAggregateKind.count => nonNullCount[cid], + IvmAggregateKind.sum => + (nonNullCount[cid] ?? 0) == 0 ? null : sum[cid], + IvmAggregateKind.min => minValue[cid], + IvmAggregateKind.max => maxValue[cid], + IvmAggregateKind.avg => + (nonNullCount[cid] ?? 0) == 0 + ? null + : sum[cid]! / nonNullCount[cid]!, + }; } + return row; + } + + IvmOutcome _bail() { + seeded = false; + return IvmOutcome.bail; + } +} + +/// Build the internal snapshot query that seeds an aggregate state +/// exactly. All identifiers come from `PRAGMA table_info` and all values +/// from the validated predicate constants, quoted defensively. +String buildAggregateSnapshotSql( + IvmAggregateShape shape, + List> tableInfo, +) { + String quoteIdent(String name) => '"${name.replaceAll('"', '""')}"'; + final nameByCid = { + for (final col in tableInfo) + if (col['cid'] is int && col['name'] is String) + col['cid'] as int: col['name'] as String, + }; + + final selects = ['COUNT(*) AS __rows']; + for (final cid in shape.aggregateColumns) { + final col = quoteIdent(nameByCid[cid]!); + selects.add('COUNT($col) AS __n$cid'); + selects.add('SUM($col) AS __s$cid'); + selects.add('MIN($col) AS __lo$cid'); + selects.add('MAX($col) AS __hi$cid'); } - return -(lo + 1); + + final where = shape.predicates.isEmpty + ? '' + : ' WHERE ${shape.predicates.map((p) { + final col = quoteIdent(nameByCid[p.columnIndex]!); + final value = p.value; + final lit = value is int + ? '$value' + : "'${(value as String).replaceAll("'", "''")}'"; + return '$col ${p.op} $lit'; + }).join(' AND ')}'; + + return 'SELECT ${selects.join(', ')} FROM ${quoteIdent(shape.table)}$where'; } diff --git a/test/profile_counters_test.dart b/test/profile_counters_test.dart index de4fb44f..9d0ec673 100644 --- a/test/profile_counters_test.dart +++ b/test/profile_counters_test.dart @@ -33,7 +33,10 @@ void main() { 'ivm_applied_total': 0, 'ivm_bail_total': 0, 'ivm_admitted_total': 0, + 'ivm_admitted_skip_total': 0, + 'ivm_admitted_agg_total': 0, 'ivm_rejected_total': 0, + 'ivm_hit_fallback_total': 0, }); ProfileCounters.reset(); @@ -58,7 +61,10 @@ void main() { 'ivm_applied_total': 0, 'ivm_bail_total': 0, 'ivm_admitted_total': 0, + 'ivm_admitted_skip_total': 0, + 'ivm_admitted_agg_total': 0, 'ivm_rejected_total': 0, + 'ivm_hit_fallback_total': 0, }); }); diff --git a/test/stream_ivm_test.dart b/test/stream_ivm_test.dart index bef08b8a..6edb53d0 100644 --- a/test/stream_ivm_test.dart +++ b/test/stream_ivm_test.dart @@ -14,117 +14,203 @@ final _itemsTableInfo = >[ {'cid': 3, 'name': 'name', 'type': 'TEXT', 'pk': 0}, ]; +const _itemsCreateSql = + 'CREATE TABLE items(id INTEGER PRIMARY KEY, flag INTEGER NOT NULL, ' + 'score INTEGER NOT NULL, name TEXT NOT NULL)'; + void main() { - group('classifyIvmQuery', () { - IvmShape? classify(String sql, [List params = const []]) => - classifyIvmQuery(sql, params, 'items', _itemsTableInfo); + group('classifyIvmQuery (modes)', () { + IvmAdmission? classify( + String sql, [ + List params = const [], + String? createSql = _itemsCreateSql, + ]) => + classifyIvmQuery( + sql, + params, + 'items', + _itemsTableInfo, + createSql: createSql, + ); - test('admits range + ORDER BY pk', () { + test('full: range + ORDER BY pk', () { final shape = classify( 'SELECT id, score, name FROM items WHERE id >= ? AND id < ? ORDER BY id', [10, 20], ); - expect(shape, isNotNull); - expect(shape!.predicates, hasLength(2)); - expect(shape.pkOutputName, 'id'); - expect(shape.projection.map((p) => p.$1), ['id', 'score', 'name']); + expect(shape, isA()); + final full = shape! as IvmFullShape; + expect(full.predicates, hasLength(2)); + expect(full.pkOutputName, 'id'); + expect(full.limit, isNull); + expect(full.projection.map((c) => c.$1), ['id', 'score', 'name']); }); - test('admits pk equality without ORDER BY', () { - expect(classify('SELECT * FROM items WHERE id = ?', [5]), isNotNull); + test('full: pk equality without ORDER BY, * projection', () { + expect( + classify('SELECT * FROM items WHERE id = ?', [5]), + isA(), + ); }); - test('admits SELECT * with table-order projection', () { - final shape = classify('SELECT * FROM items WHERE id = 3'); - expect(shape, isNotNull); - expect(shape!.projection.map((p) => p.$1), [ - 'id', - 'flag', - 'score', - 'name', - ]); + test('full: DESC on pk alone is deterministic', () { + final shape = classify('SELECT id, score FROM items ORDER BY id DESC'); + expect(shape, isA()); + expect((shape! as IvmFullShape).orderDesc, isTrue); }); - test('admits integer equality on non-pk column with ORDER BY pk', () { - expect( - classify('SELECT id, name FROM items WHERE flag = 1 ORDER BY id'), - isNotNull, + test('full windowed: composite ORDER BY with pk tiebreak + LIMIT', () { + final shape = classify( + 'SELECT id, score FROM items WHERE flag = 1 ' + 'ORDER BY score DESC, id DESC LIMIT 20', + ); + expect(shape, isA()); + final full = shape! as IvmFullShape; + expect(full.limit, 20); + expect(full.orderDesc, isTrue); + expect(full.pkDesc, isTrue); + expect(full.orderOutputName, 'score'); + }); + + test('full windowed: LIMIT as a bind parameter', () { + final shape = classify( + 'SELECT id, score FROM items ORDER BY score, id LIMIT ?', + [30], ); + expect(shape, isA()); + expect((shape! as IvmFullShape).limit, 30); }); - test('rejects everything outside the grammar', () { - final rejected = [ - // No deterministic order and no pk pin. + test('full: aliased pk keeps keying through the alias', () { + final shape = classify('SELECT id AS x FROM items WHERE id = 1'); + expect(shape, isA()); + expect((shape! as IvmFullShape).pkOutputName, 'x'); + }); + + test('skip: evaluable predicate with unmaintainable result', () { + final cases = [ + // No deterministic order, no pk pin. 'SELECT id FROM items WHERE flag = 1', - // Order by non-pk. + // Order by non-pk without tiebreak. 'SELECT id, score FROM items WHERE id > 0 ORDER BY score', - 'SELECT id FROM items ORDER BY id DESC', - 'SELECT id FROM items ORDER BY id LIMIT 10', - 'SELECT id FROM items WHERE id > 0 OR flag = 1 ORDER BY id', - 'SELECT id FROM items WHERE name = ? ORDER BY id', // text param - 'SELECT count(*) FROM items WHERE id = 1', - 'SELECT i.id FROM items i WHERE id = 1', - 'SELECT id FROM items JOIN other ON 1 WHERE id = 1', + // DESC window without pk tiebreak. + 'SELECT id, score FROM items WHERE flag = 1 ORDER BY score DESC LIMIT 20', + // OFFSET demotes. + 'SELECT id FROM items WHERE flag = 1 ORDER BY id LIMIT 10 OFFSET 5', + // DISTINCT demotes. 'SELECT DISTINCT id FROM items WHERE id = 1', - 'SELECT id AS x FROM items WHERE id = 1', - 'SELECT id FROM other WHERE id = 1', // table mismatch - 'SELECT flag FROM items WHERE id = 1', // pk not projected - 'SELECT id FROM items WHERE missing = 1 ORDER BY id', - "SELECT id FROM items WHERE name = 'x' ORDER BY id", - 'SELECT id FROM items WHERE id IN (1, 2) ORDER BY id', + // pk not projected. + 'SELECT flag FROM items WHERE id = 1', + // Order column not projected. + 'SELECT id FROM items WHERE flag = 1 ORDER BY score, id LIMIT 5', ]; - for (final sql in rejected) { - expect(classify(sql), isNull, reason: sql); + for (final sql in cases) { + expect(classify(sql), isA(), reason: sql); } }); - test('rejects text bind parameter values', () { + test('skip: TEXT equality admitted only without COLLATE', () { + const sql = "SELECT id FROM items WHERE name = 'x'"; + expect(classify(sql), isA()); + expect( + classify(sql, const [], null), + isNull, + reason: 'unknown CREATE sql must reject text predicates', + ); expect( - classify('SELECT id FROM items WHERE id = ? ORDER BY id', ['5']), + classify( + sql, + const [], + 'CREATE TABLE items(id INTEGER PRIMARY KEY, ' + 'name TEXT COLLATE NOCASE)', + ), isNull, + reason: 'COLLATE anywhere in the table rejects text predicates', ); }); - test('rejects tables without an INTEGER pk rowid alias', () { - final noPk = [ - {'cid': 0, 'name': 'a', 'type': 'INTEGER', 'pk': 0}, - ]; + test('full: TEXT equality predicate with ORDER BY pk', () { + final shape = classify( + 'SELECT id, name FROM items WHERE name = ? ORDER BY id', + ['row_7'], + ); + expect(shape, isA()); + }); + + test('skip: TEXT inequality never admitted', () { expect( - classifyIvmQuery('SELECT a FROM t WHERE a = 1', const [], 't', noPk), + classify("SELECT id FROM items WHERE name > 'x' ORDER BY id"), isNull, ); - final textPk = [ - {'cid': 0, 'name': 'k', 'type': 'TEXT', 'pk': 1}, - ]; + }); + + test('aggregate: aliased aggregates over evaluable predicate', () { + final shape = classify( + 'SELECT COUNT(*) AS n, SUM(score) AS total, MIN(score) AS lo, ' + 'MAX(score) AS hi, AVG(score) AS mean ' + 'FROM items WHERE flag = ?', + [1], + ); + expect(shape, isA()); + final agg = shape! as IvmAggregateShape; + expect(agg.aggregates, hasLength(5)); + expect(agg.aggregateColumns, [2]); + }); + + test('aggregate: COUNT(*) without WHERE', () { expect( - classifyIvmQuery('SELECT k FROM t WHERE k = 1', const [], 't', textPk), + classify('SELECT COUNT(*) AS n FROM items'), + isA(), + ); + }); + + test('aggregate: missing AS alias rejects', () { + expect(classify('SELECT COUNT(*) FROM items WHERE flag = 1'), isNull); + }); + + test('aggregate: non-INTEGER column for SUM rejects to skip', () { + expect( + classify('SELECT SUM(name) AS s FROM items WHERE flag = 1'), isNull, ); - final compositePk = [ - {'cid': 0, 'name': 'a', 'type': 'INTEGER', 'pk': 1}, - {'cid': 1, 'name': 'b', 'type': 'INTEGER', 'pk': 2}, + }); + + test('rejected outright', () { + final cases = [ + 'SELECT id FROM items WHERE id > 0 OR flag = 1 ORDER BY id', + 'SELECT i.id FROM items i WHERE id = 1', + 'SELECT id FROM items JOIN other ON 1 WHERE id = 1', + 'SELECT id FROM other WHERE id = 1', + 'SELECT id FROM items WHERE missing = 1 ORDER BY id', + 'SELECT id FROM items WHERE id IN (1, 2) ORDER BY id', + 'SELECT id FROM items ORDER BY id LIMIT 10', // window without preds is full though ]; + for (final sql in cases.sublist(0, cases.length - 1)) { + expect(classify(sql), isNull, reason: sql); + } + // Windowed full without predicates is admissible. + expect(classify(cases.last), isA()); + }); + + test('rejects text bind parameter for int column ops', () { expect( - classifyIvmQuery( - 'SELECT a, b FROM t WHERE a = 1', - const [], - 't', - compositePk, - ), + classify('SELECT id FROM items WHERE id < ? ORDER BY id', ['5']), isNull, ); }); }); - group('IvmState.apply', () { - IvmState freshState() { + group('IvmFullState (unwindowed)', () { + IvmFullState freshState() { final shape = classifyIvmQuery( - 'SELECT id, score, name FROM items WHERE id >= 10 AND id < 20 ORDER BY id', + 'SELECT id, score, name FROM items ' + 'WHERE id >= 10 AND id < 20 ORDER BY id', const [], 'items', _itemsTableInfo, - )!; - final state = IvmState(shape); + createSql: _itemsCreateSql, + )! as IvmFullShape; + final state = IvmFullState(shape); expect( state.rebuild([ {'id': 11, 'score': 5, 'name': 'a'}, @@ -148,145 +234,413 @@ void main() { test('proven miss leaves the cache untouched', () { final state = freshState(); final before = state.rows; - final outcome = state.apply([ - update(50, [50, 0, 1, 'x'], [50, 0, 2, 'x']), - ]); - expect(outcome, IvmOutcome.unchanged); + expect( + state.apply([ + update(50, [50, 0, 1, 'x'], [50, 0, 2, 'x']), + ]), + IvmOutcome.unchanged, + ); expect(identical(state.rows, before), isTrue); }); - test('in-window patch emits a fresh list and preserves order', () { + test('patch clones and preserves order', () { final state = freshState(); final before = state.rows; - final outcome = state.apply([ - update(15, [15, 0, 7, 'b'], [15, 0, 9, 'b2']), - ]); - expect(outcome, IvmOutcome.applied); + expect( + state.apply([ + update(15, [15, 0, 7, 'b'], [15, 0, 9, 'b2']), + ]), + IvmOutcome.applied, + ); expect(identical(state.rows, before), isFalse); expect(state.rows, [ {'id': 11, 'score': 5, 'name': 'a'}, {'id': 15, 'score': 9, 'name': 'b2'}, ]); - // The pre-patch list (held by earlier subscribers) is unchanged. expect(before!.last['score'], 7); }); - test('update touching only unprojected columns is unchanged', () { - final state = freshState(); - final outcome = state.apply([ - update(15, [15, 0, 7, 'b'], [15, 1, 7, 'b']), // flag not projected - ]); - expect(outcome, IvmOutcome.unchanged); - }); - - test('insert enters at the sorted position', () { + test('insert, delete, rowid change', () { final state = freshState(); - final outcome = state.apply([ - RowDelta( - op: deltaOpInsert, - table: 'items', - oldRowid: 13, - newRowid: 13, - oldValues: null, - newValues: [13, 0, 1, 'c'], - ), - ]); - expect(outcome, IvmOutcome.applied); + expect( + state.apply([ + RowDelta( + op: deltaOpInsert, + table: 'items', + oldRowid: 13, + newRowid: 13, + oldValues: null, + newValues: [13, 0, 1, 'c'], + ), + ]), + IvmOutcome.applied, + ); expect(state.rows!.map((r) => r['id']), [11, 13, 15]); + + expect( + state.apply([ + RowDelta( + op: deltaOpDelete, + table: 'items', + oldRowid: 11, + newRowid: 11, + oldValues: [11, 0, 5, 'a'], + newValues: null, + ), + ]), + IvmOutcome.applied, + ); + expect(state.rows!.map((r) => r['id']), [13, 15]); + + expect( + state.apply([ + RowDelta( + op: deltaOpUpdate, + table: 'items', + oldRowid: 15, + newRowid: 12, + oldValues: [15, 0, 7, 'b'], + newValues: [12, 0, 7, 'b'], + ), + ]), + IvmOutcome.applied, + ); + expect(state.rows!.map((r) => r['id']), [12, 13]); }); - test('delete departs exactly', () { + test('unprovable cell and cache inconsistency bail', () { final state = freshState(); - final outcome = state.apply([ - RowDelta( - op: deltaOpDelete, - table: 'items', - oldRowid: 11, - newRowid: 11, - oldValues: [11, 0, 5, 'a'], - newValues: null, - ), - ]); - expect(outcome, IvmOutcome.applied); - expect(state.rows!.map((r) => r['id']), [15]); + expect( + state.apply([ + update(50, ['oops', 0, 1, 'x'], ['oops', 0, 2, 'x']), + ]), + IvmOutcome.bail, + ); + expect(state.rows, isNull); + + final state2 = freshState(); + expect( + state2.apply([ + RowDelta( + op: deltaOpInsert, + table: 'items', + oldRowid: 15, + newRowid: 15, + oldValues: null, + newValues: [15, 0, 1, 'dup'], + ), + ]), + IvmOutcome.bail, + ); }); + }); - test('rowid change splits into departure + entry', () { - final state = freshState(); - final outcome = state.apply([ - RowDelta( - op: deltaOpUpdate, - table: 'items', - oldRowid: 15, - newRowid: 12, - oldValues: [15, 0, 7, 'b'], - newValues: [12, 0, 7, 'b'], - ), - ]); - expect(outcome, IvmOutcome.applied); - expect(state.rows!.map((r) => r['id']), [11, 12]); + group('IvmFullState (windowed)', () { + // Window: top 3 by (score DESC, id DESC) of flag = 1 rows. + IvmFullShape windowShape() => + classifyIvmQuery( + 'SELECT id, score FROM items WHERE flag = 1 ' + 'ORDER BY score DESC, id DESC LIMIT 3', + const [], + 'items', + _itemsTableInfo, + createSql: _itemsCreateSql, + )! + as IvmFullShape; + + List row(int id, int flag, int score) => [id, flag, score, 'r$id']; + + RowDelta insert(int id, int flag, int score) => RowDelta( + op: deltaOpInsert, + table: 'items', + oldRowid: id, + newRowid: id, + oldValues: null, + newValues: row(id, flag, score), + ); + + RowDelta delete(int id, int flag, int score) => RowDelta( + op: deltaOpDelete, + table: 'items', + oldRowid: id, + newRowid: id, + oldValues: row(id, flag, score), + newValues: null, + ); + + test('full window: entries inside/below, departures fall back', () { + final state = IvmFullState(windowShape()); + expect( + state.rebuild([ + {'id': 9, 'score': 90}, + {'id': 7, 'score': 70}, + {'id': 5, 'score': 50}, + ]), + isTrue, + ); + expect(state.complete, isFalse); // len == K, more may exist + + // Below-window entry is invisible and ignored. + expect(state.apply([insert(2, 1, 10)]), IvmOutcome.unchanged); + + // In-window entry displaces the tail. + expect(state.apply([insert(8, 1, 80)]), IvmOutcome.applied); + expect(state.visibleRows().map((r) => r['id']), [9, 8, 7]); + + // Miss: flag = 0 rows never matter. + expect(state.apply([insert(99, 0, 999)]), IvmOutcome.unchanged); + + // Departure from a full, incomplete window cannot be patched. + expect(state.apply([delete(9, 1, 90)]), IvmOutcome.bail); + expect(state.rows, isNull); }); - test('NULL predicate cell means the row does not match', () { - final state = freshState(); - final outcome = state.apply([ - update(50, [null, 0, 1, 'x'], [null, 0, 2, 'x']), - ]); - expect(outcome, IvmOutcome.unchanged); + test('complete window absorbs departures', () { + final state = IvmFullState(windowShape()); + expect( + state.rebuild([ + {'id': 9, 'score': 90}, + {'id': 7, 'score': 70}, + ]), + isTrue, + ); + expect(state.complete, isTrue); // fewer rows than K exist + + expect(state.apply([delete(9, 1, 90)]), IvmOutcome.applied); + expect(state.visibleRows().map((r) => r['id']), [7]); + + expect(state.apply([insert(8, 1, 80)]), IvmOutcome.applied); + expect(state.apply([insert(6, 1, 60)]), IvmOutcome.applied); + expect(state.apply([insert(4, 1, 40)]), IvmOutcome.applied); + // Window shows top 3; the 4th row is retained (complete set). + expect(state.visibleRows().map((r) => r['id']), [8, 7, 6]); + expect(state.rows!.length, 4); + + // Now a departure from the window promotes the retained row. + expect(state.apply([delete(7, 1, 70)]), IvmOutcome.applied); + expect(state.visibleRows().map((r) => r['id']), [8, 6, 4]); }); - test('non-int predicate cell bails', () { - final state = freshState(); - final outcome = state.apply([ - update(50, ['oops', 0, 1, 'x'], ['oops', 0, 2, 'x']), - ]); - expect(outcome, IvmOutcome.bail); - expect(state.rows, isNull); + test('tie order follows the explicit pk tiebreak', () { + final state = IvmFullState(windowShape()); + expect( + state.rebuild([ + {'id': 9, 'score': 50}, + {'id': 4, 'score': 50}, + ]), + isTrue, + ); + expect(state.apply([insert(6, 1, 50)]), IvmOutcome.applied); + expect(state.visibleRows().map((r) => r['id']), [9, 6, 4]); }); + }); - test('cache inconsistency bails (entry already present)', () { - final state = freshState(); - final outcome = state.apply([ + group('IvmSkipState', () { + IvmSkipState skipState() => + IvmSkipState( + classifyIvmQuery( + 'SELECT id, score FROM items WHERE flag = 1 ' + 'ORDER BY score DESC LIMIT 20', + const [], + 'items', + _itemsTableInfo, + createSql: _itemsCreateSql, + )! + as IvmSkipShape, + ); + + RowDelta update(int rowid, List oldV, List newV) => RowDelta( - op: deltaOpInsert, + op: deltaOpUpdate, table: 'items', - oldRowid: 15, - newRowid: 15, - oldValues: null, - newValues: [15, 0, 1, 'dup'], - ), - ]); - expect(outcome, IvmOutcome.bail); + oldRowid: rowid, + newRowid: rowid, + oldValues: oldV, + newValues: newV, + ); + + test('all-miss batches are proven unchanged', () { + final state = skipState(); + expect( + state.apply([ + update(1, [1, 0, 5, 'a'], [1, 0, 9, 'a']), + update(2, [2, 0, 5, 'b'], [2, 0, 9, 'b']), + ]), + IvmOutcome.unchanged, + ); }); - test('schema drift (column count mismatch) bails', () { - final state = freshState(); - final outcome = state.apply([ - update(15, [15, 0, 7], [15, 0, 9]), - ]); - expect(outcome, IvmOutcome.bail); + test('any hit falls back', () { + final state = skipState(); + expect( + state.apply([ + update(1, [1, 0, 5, 'a'], [1, 1, 5, 'a']), // enters flag = 1 + ]), + IvmOutcome.bail, + ); }); - test('rebuild rejects unkeyable or unordered results', () { - final shape = classifyIvmQuery( - 'SELECT id, score, name FROM items WHERE id >= 0 ORDER BY id', - const [], - 'items', - _itemsTableInfo, - )!; + test('unprovable cells fall back', () { + final state = skipState(); expect( - IvmState(shape).rebuild([ - {'id': 'nope', 'score': 1, 'name': 'a'}, + state.apply([ + update(1, [1, 'x', 5, 'a'], [1, 'x', 9, 'a']), ]), - isFalse, + IvmOutcome.bail, ); + }); + }); + + group('IvmAggregateState', () { + IvmAggregateShape aggShape() => + classifyIvmQuery( + 'SELECT COUNT(*) AS n, SUM(score) AS total, MIN(score) AS lo, ' + 'MAX(score) AS hi, AVG(score) AS mean ' + 'FROM items WHERE flag = 1', + const [], + 'items', + _itemsTableInfo, + createSql: _itemsCreateSql, + )! + as IvmAggregateShape; + + IvmAggregateState seeded() { + final state = IvmAggregateState(aggShape()); + expect( + state.seedFromSnapshot({ + '__rows': 2, + '__n2': 2, + '__s2': 30, + '__lo2': 10, + '__hi2': 20, + }), + isTrue, + ); + return state; + } + + List row(int id, int flag, int? score) => [id, flag, score, 'r']; + + test('entries, departures, and patches maintain exact values', () { + final state = seeded(); + + // Entry of (score 5): count 3, sum 35, min 5, max 20. expect( - IvmState(shape).rebuild([ - {'id': 5, 'score': 1, 'name': 'a'}, - {'id': 3, 'score': 1, 'name': 'b'}, + state.apply([ + RowDelta( + op: deltaOpInsert, + table: 'items', + oldRowid: 7, + newRowid: 7, + oldValues: null, + newValues: row(7, 1, 5), + ), ]), - isFalse, + IvmOutcome.applied, ); + expect(state.visibleRow(), { + 'n': 3, + 'total': 35, + 'lo': 5, + 'hi': 20, + 'mean': closeTo(35 / 3, 1e-9), + }); + + // NULL cells count for COUNT(*) but not the column aggregates. + expect( + state.apply([ + RowDelta( + op: deltaOpInsert, + table: 'items', + oldRowid: 8, + newRowid: 8, + oldValues: null, + newValues: row(8, 1, null), + ), + ]), + IvmOutcome.applied, + ); + expect(state.visibleRow()['n'], 4); + expect(state.visibleRow()['total'], 35); + + // Patch a non-extremum row: score 10 -> 12 (extremum departures + // are covered below and legitimately bail). + expect( + state.apply([ + RowDelta( + op: deltaOpUpdate, + table: 'items', + oldRowid: 1, + newRowid: 1, + oldValues: row(1, 1, 10), + newValues: row(1, 1, 12), + ), + ]), + IvmOutcome.applied, + ); + expect(state.visibleRow()['total'], 37); + expect(state.visibleRow()['lo'], 5); + + // Misses never touch state. + expect( + state.apply([ + RowDelta( + op: deltaOpUpdate, + table: 'items', + oldRowid: 50, + newRowid: 50, + oldValues: row(50, 0, 1), + newValues: row(50, 0, 2), + ), + ]), + IvmOutcome.unchanged, + ); + }); + + test('departing extremum bails and unseeds', () { + final state = seeded(); + expect( + state.apply([ + RowDelta( + op: deltaOpDelete, + table: 'items', + oldRowid: 3, + newRowid: 3, + oldValues: row(3, 1, 20), // current max departs + newValues: null, + ), + ]), + IvmOutcome.bail, + ); + expect(state.seeded, isFalse); + }); + + test('empty set reports SQL aggregate semantics', () { + final state = IvmAggregateState(aggShape()); + expect( + state.seedFromSnapshot({ + '__rows': 0, + '__n2': 0, + '__s2': null, + '__lo2': null, + '__hi2': null, + }), + isTrue, + ); + expect(state.visibleRow(), { + 'n': 0, + 'total': null, + 'lo': null, + 'hi': null, + 'mean': null, + }); + }); + + test('snapshot SQL is well-formed', () { + final sql = buildAggregateSnapshotSql(aggShape(), _itemsTableInfo); + expect(sql, contains('COUNT(*) AS __rows')); + expect(sql, contains('SUM("score") AS __s2')); + expect(sql, contains('FROM "items" WHERE "flag" = 1')); }); }); From 6ab458dfe579bb8360c0cf6bf03d02cdf792c063 Mon Sep 17 00:00:00 2001 From: Dan Reynolds Date: Wed, 10 Jun 2026 19:09:29 -0400 Subject: [PATCH 08/10] exp 160 tier hardening: ordering correctness + hot-path optimization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two defects found by the randomized equivalence harness under dart test load, both invisible to deterministic runs: 1. Reader-built IVM baselines race late writer replies. An aggregate snapshot (or full-cache build) executed on a reader can observe a commit whose delta is then applied on top of it — cross-port event ordering gives no guarantee a reader result is processed before the writer reply that produced what it read. All IVM state is now built through writer-ordered reads (Database wires a writer.locked selectInTransaction hook into the engine): the writer port is FIFO with the writes themselves, so a snapshot's position totally orders it against every delta. 2. A maintained state only survives an unbroken chain of processed cycles. A delta-bearing write routed to the re-query fallback (dirty/in-flight guard, malformed or absent deltas, capture overflow) leaves the state's baseline permanently stale — and a hash-suppressed re-query validates emissions without re-syncing it (ledger-captured: seed at 2784, insert swallowed by an unchanged-hash re-query, every later apply walking the -1 forward). The engine now drops maintained state whenever a cycle bypasses it; the writer-ordered rebuild restores an exact baseline. Known trade: churn cycles (e.g. overflow batches) trigger rebuild storms — steady state is untouched. Hot path: predicate conjunctions compile to flattened primitive arrays (IvmPredicateProgram — no string switches or Object re-checks per delta), and full states keep a pk set for O(1) presence checks on the proven-miss path, which is the single hottest IVM operation. Equivalence harness: 20/20 clean full-loop runs post-fix (~1-in-5 diverged before); full suite green. Co-Authored-By: Claude Fable 5 --- benchmark/profile/agg_bisect.dart | 108 ++++++++ benchmark/profile/agg_repro.dart | 136 +++++++++ benchmark/profile/ivm_admission_audit.dart | 44 ++- .../exp-147-writer-sqlite-wall-aggregate.md | 16 +- lib/src/database.dart | 10 + lib/src/stream_engine.dart | 112 +++++++- lib/src/stream_ivm.dart | 181 ++++++++---- test/stream_ivm_equivalence_test.dart | 261 ++++++++++++++++++ 8 files changed, 792 insertions(+), 76 deletions(-) create mode 100644 benchmark/profile/agg_bisect.dart create mode 100644 benchmark/profile/agg_repro.dart create mode 100644 test/stream_ivm_equivalence_test.dart diff --git a/benchmark/profile/agg_bisect.dart b/benchmark/profile/agg_bisect.dart new file mode 100644 index 00000000..9d3a4a10 --- /dev/null +++ b/benchmark/profile/agg_bisect.dart @@ -0,0 +1,108 @@ +import 'dart:io'; +import 'dart:math' as math; +import 'package:resqlite/resqlite.dart'; + +Future main(List args) async { + final mode = args.first; + for (var trial = 0; trial < 6; trial++) { + final dir = await Directory.systemTemp.createTemp('agg_bisect_'); + final db = await Database.open('${dir.path}/t.db'); + await db.execute( + 'CREATE TABLE msgs(id INTEGER PRIMARY KEY, v INTEGER NOT NULL)'); + await db.executeBatch('INSERT INTO msgs(id, v) VALUES (?, ?)', + [for (var i = 1; i <= 50; i++) [i, i]]); + var nextId = 51; + final live = List.generate(50, (i) => i + 1); + final prng = math.Random(trial * 31 + 7); + final emissions = []; + final sub = db + .stream('SELECT COUNT(*) AS n FROM msgs') + .listen((r) => emissions.add(r[0]['n'] as int)); + await Future.delayed(const Duration(milliseconds: 120)); + Future check(int op) async { + var last = -1; + while (emissions.isNotEmpty && emissions.last != last) { + last = emissions.last; + await Future.delayed(const Duration(milliseconds: 100)); + } + final truth = (await db.select('SELECT COUNT(*) AS n FROM msgs'))[0]['n']; + if (emissions.isNotEmpty && emissions.last != truth) { + print('$mode trial $trial: DIVERGED at op $op ' + 'emitted=${emissions.last} truth=$truth'); + return false; + } + return true; + } + var diverged = false; + for (var op = 0; op < 400; op++) { + if (op % 25 == 24 && !await check(op)) { diverged = true; break; } + var effMode = mode; + if (mode == 'storm' || mode.startsWith('storm-')) { + final drop = mode.startsWith('storm-') ? mode.substring(6) : ''; + final classes = ['insert', 'update', 'mix', 'rowid', 'tx', 'overflow'] + .where((c) => c != drop) + .toList(); + effMode = classes[prng.nextInt(classes.length)]; + } + switch (effMode) { + case 'insert': + await db.execute('INSERT INTO msgs(id, v) VALUES (?, 1)', [nextId++]); + live.add(nextId - 1); + case 'mix': + if (prng.nextBool() || live.isEmpty) { + await db.execute('INSERT INTO msgs(id, v) VALUES (?, 1)', [nextId++]); + live.add(nextId - 1); + } else { + final id = live.removeAt(prng.nextInt(live.length)); + await db.execute('DELETE FROM msgs WHERE id = ?', [id]); + } + case 'rowid': + if (live.isNotEmpty && prng.nextBool()) { + final i = prng.nextInt(live.length); + final oldId = live[i]; + live[i] = nextId++; + await db.execute('UPDATE msgs SET id = ? WHERE id = ?', [live[i], oldId]); + } else { + await db.execute('INSERT INTO msgs(id, v) VALUES (?, 1)', [nextId++]); + live.add(nextId - 1); + } + case 'tx': + await db.transaction((tx) async { + await tx.execute('INSERT INTO msgs(id, v) VALUES (?, 1)', [nextId++]); + live.add(nextId - 1); + if (prng.nextBool()) { + try { + await tx.transaction((tx2) async { + await tx2.execute('UPDATE msgs SET v = 9 WHERE id = ?', + [live[prng.nextInt(live.length)]]); + throw StateError('undo'); + }); + } on StateError {/**/} + } + }); + case 'update': + if (live.isNotEmpty) { + await db.execute('UPDATE msgs SET v = ? WHERE id = ?', + [prng.nextInt(100), live[prng.nextInt(live.length)]]); + } + case 'overflow': + if (op % 40 == 13) { + final base = nextId; nextId += 300; + live.addAll([for (var i = base; i < base + 300; i++) i]); + await db.executeBatch('INSERT INTO msgs(id, v) VALUES (?, ?)', + [for (var i = base; i < base + 300; i++) [i, 1]]); + } else { + await db.execute('INSERT INTO msgs(id, v) VALUES (?, 1)', [nextId++]); + live.add(nextId - 1); + } + } + } + final ok = diverged ? false : await check(400); + if (ok) print('$mode trial $trial: OK'); + await sub.cancel(); + await db.close(); + await dir.delete(recursive: true); + if (!ok) exit(1); + } + exit(0); +} diff --git a/benchmark/profile/agg_repro.dart b/benchmark/profile/agg_repro.dart new file mode 100644 index 00000000..535450a8 --- /dev/null +++ b/benchmark/profile/agg_repro.dart @@ -0,0 +1,136 @@ +// Replicates the equivalence harness write generator exactly (seed-driven) +// with a configurable subset of its streams, to isolate which stream +// interaction lets the global COUNT(*) drift. +import 'dart:async'; +import 'dart:io'; +import 'dart:math' as math; +import 'package:resqlite/resqlite.dart'; + +Future main(List args) async { + final seed = int.parse(args[0]); + final streamSet = args[1]; // 'count' | 'aggpair' | 'all' + final dir = await Directory.systemTemp.createTemp('agg_repro_'); + final db = await Database.open('${dir.path}/t.db'); + await db.execute( + 'CREATE TABLE msgs(id INTEGER PRIMARY KEY, conv INTEGER NOT NULL, ' + 'score INTEGER, body TEXT NOT NULL, kind TEXT NOT NULL)'); + final prng = math.Random(seed); + var nextId = 1; + await db.executeBatch( + 'INSERT INTO msgs(id, conv, score, body, kind) VALUES (?, ?, ?, ?, ?)', + [ + for (; nextId <= 60; nextId++) + [nextId, nextId % 6, prng.nextBool() ? null : prng.nextInt(1000), + 'b$nextId', nextId % 3 == 0 ? 'pin' : 'note'] + ]); + + final sqls = <(String, String, List)>[ + ('global count', 'SELECT COUNT(*) AS n FROM msgs', const []), + if (streamSet != 'count') + ('aggregates', + 'SELECT COUNT(*) AS n, SUM(score) AS total, MIN(score) AS lo, ' + 'MAX(score) AS hi, AVG(score) AS mean FROM msgs WHERE conv = ?', [2]), + if (streamSet == 'all') ...[ + ('full range', + 'SELECT id, conv, score FROM msgs WHERE id >= ? AND id < ? ORDER BY id', + [10, 40]), + ('windowed feed', + 'SELECT id, score FROM msgs WHERE conv = ? ORDER BY score DESC, id DESC LIMIT 5', + [2]), + ('windowed asc', 'SELECT id, conv FROM msgs ORDER BY conv, id LIMIT 7', + const []), + ('skip pane', + 'SELECT id, body FROM msgs WHERE conv = ? ORDER BY score DESC LIMIT 4', + [3]), + ('text eq', "SELECT id, kind FROM msgs WHERE kind = 'pin' ORDER BY id", + const []), + ('control', + 'SELECT conv, COUNT(*) AS n FROM msgs WHERE id > 0 GROUP BY conv ORDER BY conv', + const []), + ], + ]; + final emissions = {for (final s in sqls) s.$1: >>[]}; + final subs = [ + for (final s in sqls) + db.stream(s.$2, s.$3).listen((r) => emissions[s.$1]!.add(r)), + ]; + await Future.delayed(const Duration(milliseconds: 200)); + + final liveIds = List.generate(60, (i) => i + 1); + Future oneWrite() async { + final roll = prng.nextInt(100); + if (roll < 35 || liveIds.isEmpty) { + final id = nextId++; + liveIds.add(id); + await db.execute( + 'INSERT INTO msgs(id, conv, score, body, kind) VALUES (?, ?, ?, ?, ?)', + [id, prng.nextInt(6), prng.nextBool() ? null : prng.nextInt(1000), + 'b$id', prng.nextBool() ? 'pin' : 'note']); + } else if (roll < 65) { + final id = liveIds[prng.nextInt(liveIds.length)]; + await db.execute( + 'UPDATE msgs SET conv = ?, score = ?, kind = ? WHERE id = ?', + [prng.nextInt(6), prng.nextBool() ? null : prng.nextInt(1000), + prng.nextBool() ? 'pin' : 'note', id]); + } else if (roll < 80) { + final id = liveIds.removeAt(prng.nextInt(liveIds.length)); + await db.execute('DELETE FROM msgs WHERE id = ?', [id]); + } else if (roll < 90) { + final idx = prng.nextInt(liveIds.length); + final oldId = liveIds[idx]; + final newId = nextId++; + liveIds[idx] = newId; + await db.execute('UPDATE msgs SET id = ? WHERE id = ?', [newId, oldId]); + } else if (roll < 96) { + await db.transaction((tx) async { + final id = nextId++; + liveIds.add(id); + await tx.execute( + "INSERT INTO msgs(id, conv, score, body, kind) VALUES (?, ?, ?, ?, 'note')", + [id, prng.nextInt(6), prng.nextInt(1000), 'tx$id']); + if (prng.nextBool()) { + try { + await tx.transaction((tx2) async { + await tx2.execute('UPDATE msgs SET score = 777777 WHERE id = ?', + [liveIds[prng.nextInt(liveIds.length)]]); + throw StateError('undo'); + }); + } on StateError {/**/} + } + }); + } else { + final base = nextId; + nextId += 300; + liveIds.addAll([for (var i = base; i < base + 300; i++) i]); + await db.executeBatch( + "INSERT INTO msgs(id, conv, score, body, kind) VALUES (?, ?, ?, ?, 'note')", + [for (var i = base; i < base + 300; i++) [i, i % 6, i % 7 == 0 ? null : i, 'bulk$i']]); + } + } + + Future settle() async { + int total() => emissions.values.fold(0, (a, l) => a + l.length); + var last = total(); var quiet = 0; + while (quiet < 2) { + await Future.delayed(const Duration(milliseconds: 40)); + final now = total(); + if (now == last) { quiet++; } else { quiet = 0; last = now; } + } + } + + for (var round = 0; round < 12; round++) { + for (var w = 0; w < 18; w++) { await oneWrite(); } + await settle(); + final truth = (await db.select('SELECT COUNT(*) AS n FROM msgs'))[0]['n']; + final got = emissions['global count']!.last[0]['n']; + if (got != truth) { + print('seed $seed [$streamSet] DIVERGED round $round: got=$got truth=$truth'); + exit(1); + } + } + print('seed $seed [$streamSet] OK'); + for (final s in subs) { await s.cancel(); } + await db.close(); + await dir.delete(recursive: true); + exit(0); +} diff --git a/benchmark/profile/ivm_admission_audit.dart b/benchmark/profile/ivm_admission_audit.dart index e9158634..ac9f56ee 100644 --- a/benchmark/profile/ivm_admission_audit.dart +++ b/benchmark/profile/ivm_admission_audit.dart @@ -64,12 +64,19 @@ final _specs = <_StreamSpec>[ _convParam, ), _StreamSpec( - 'conversation list (DESC + LIMIT)', + 'conversation list (DESC + LIMIT, no tiebreak)', 'conversations', 'SELECT id, last_msg_at FROM conversations ' 'ORDER BY last_msg_at DESC LIMIT 30', _noParams, ), + _StreamSpec( + 'conversation list (DESC + LIMIT, pk tiebreak)', + 'conversations', + 'SELECT id, last_msg_at FROM conversations ' + 'ORDER BY last_msg_at DESC, id DESC LIMIT 30', + _noParams, + ), _StreamSpec( 'unread badge (aggregate)', 'messages', @@ -114,12 +121,27 @@ Future main() async { await _setupSchema(db); - // Direct classifier verdict per spec, against the real table_info. - final verdicts = {}; + // Direct classifier verdict per spec, against the real table metadata. + final verdicts = {}; for (final spec in _specs) { final info = await db.select('PRAGMA table_info("${spec.table}")'); - final shape = classifyIvmQuery(spec.sql, spec.paramsFor(0), spec.table, info); - verdicts[spec.label] = shape != null; + final master = await db.select( + "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?", + [spec.table], + ); + final shape = classifyIvmQuery( + spec.sql, + spec.paramsFor(0), + spec.table, + info, + createSql: master.isEmpty ? null : master.first['sql'] as String?, + ); + verdicts[spec.label] = switch (shape) { + null => 'no', + IvmFullShape(limit: final l) => l == null ? 'full' : 'windowed', + IvmSkipShape() => 'skip-only', + IvmAggregateShape() => 'aggregate', + }; } // Install the stream mix and wait for initial emissions. @@ -216,17 +238,22 @@ Future main() async { for (final spec in _specs) { print( '| ${spec.label} | $_instancesPerSpec ' - '| ${verdicts[spec.label]! ? 'yes' : 'no'} ' + '| ${verdicts[spec.label]} ' '| ${emissions[spec.label]} |', ); } - final admittedSpecs = verdicts.values.where((v) => v).length; + final admittedSpecs = verdicts.values.where((v) => v != 'no').length; print('\nSpecs admitted: $admittedSpecs/${_specs.length} ' '(${admittedSpecs * _instancesPerSpec}/${_specs.length * _instancesPerSpec} stream instances)'); print('Burst wall: ${(sw.elapsedMicroseconds / 1000).toStringAsFixed(2)} ms ' 'for $_writeCount write ops (writes issue 1-2 statements each)'); print('\nEngine admission counters (at registration):\n'); - for (final key in ['ivm_admitted_total', 'ivm_rejected_total']) { + for (final key in [ + 'ivm_admitted_total', + 'ivm_admitted_skip_total', + 'ivm_admitted_agg_total', + 'ivm_rejected_total', + ]) { print('- `$key`: ${admissionSnap[key]}'); } print('\nEngine counters across the burst:\n'); @@ -234,6 +261,7 @@ Future main() async { 'ivm_skipped_total', 'ivm_applied_total', 'ivm_bail_total', + 'ivm_hit_fallback_total', 'invalidate_count', 'completion_handler_count', ]) { diff --git a/benchmark/profile/results/exp-147-writer-sqlite-wall-aggregate.md b/benchmark/profile/results/exp-147-writer-sqlite-wall-aggregate.md index b0623b1a..6cc9f631 100644 --- a/benchmark/profile/results/exp-147-writer-sqlite-wall-aggregate.md +++ b/benchmark/profile/results/exp-147-writer-sqlite-wall-aggregate.md @@ -16,19 +16,19 @@ dart run -DRESQLITE_PROFILE=true benchmark/profile/writer_sqlite_wall_audit.dart | workload | shape | wall_ms | writer_sqlite_us | writer_sqlite_count | invalidate_us | invalidate_count | residual_us | parked_total | max_parked | emissions | |---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:| -| A11c baseline | 0 streams x 500 writes | 86.88 | 23071 | 500 | 0 | 0 | 63810 | 0 | 0 | 0 | -| A11c disjoint | 50 streams x 500 writes | 92.78 | 17738 | 500 | 23833 | 500 | 51205 | 0 | 0 | 0 | -| A11c overlap | 50 streams x 500 writes | 166.75 | 15678 | 500 | 31389 | 500 | 119679 | 0 | 0 | 35 | -| keyed PK subscriptions | 50 streams x 200 random writes | 36.95 | 6679 | 200 | 6897 | 200 | 23370 | 0 | 0 | 3 | +| A11c baseline | 0 streams x 500 writes | 72.65 | 22637 | 500 | 0 | 0 | 50012 | 0 | 0 | 0 | +| A11c disjoint | 50 streams x 500 writes | 114.21 | 20228 | 500 | 26671 | 500 | 67315 | 0 | 0 | 0 | +| A11c overlap | 50 streams x 500 writes | 206.36 | 19671 | 500 | 137389 | 500 | 49298 | 0 | 0 | 500 | +| keyed PK subscriptions | 50 streams x 200 random writes | 48.06 | 5918 | 200 | 21862 | 200 | 20285 | 0 | 0 | 3 | ## Derived fractions | workload | SQLite / wall | invalidation / wall | residual / wall | SQLite us/write | invalidation us/write | ns/intersection entry | |---|---:|---:|---:|---:|---:|---:| -| A11c baseline | 26.55% | 0.00% | 73.45% | 46.14 | 0.00 | 0 | -| A11c disjoint | 19.12% | 25.69% | 55.19% | 35.48 | 47.67 | 328 | -| A11c overlap | 9.40% | 18.82% | 71.77% | 31.36 | 62.78 | 231 | -| keyed PK subscriptions | 18.08% | 18.67% | 63.25% | 33.40 | 34.48 | 246 | +| A11c baseline | 31.16% | 0.00% | 68.84% | 45.27 | 0.00 | 0 | +| A11c disjoint | 17.71% | 23.35% | 58.94% | 40.46 | 53.34 | 339 | +| A11c overlap | 9.53% | 66.58% | 23.89% | 39.34 | 274.78 | 233 | +| keyed PK subscriptions | 12.31% | 45.48% | 42.20% | 29.59 | 109.31 | 356 | ## Reading the table diff --git a/lib/src/database.dart b/lib/src/database.dart index e10d8ac0..cb194481 100644 --- a/lib/src/database.dart +++ b/lib/src/database.dart @@ -56,6 +56,16 @@ final class Database { // Spawn the single writer isolate. final writer = await Writer.spawn(streamEngine, _handle); + // IVM state builders read through the writer so their snapshots are + // FIFO-ordered against the write replies that carry row deltas — + // a reader-side snapshot can observe a commit whose delta is then + // applied on top of it (double-counting). Exp 160 tiers. + streamEngine.attachWriterRead( + (sql, params) => writer.locked( + () async => (await writer.selectInTransaction(sql, params)).rows, + ), + ); + return ( readerPool: readerPool, streamEngine: streamEngine, diff --git a/lib/src/stream_engine.dart b/lib/src/stream_engine.dart index 218ea33f..4100f9bc 100644 --- a/lib/src/stream_engine.dart +++ b/lib/src/stream_engine.dart @@ -197,6 +197,14 @@ final class StreamEngine { if (deltaBatch != null && _tryIncrementalMaintain(entry, deltaBatch)) { continue; } + // This write cycle is bypassing the maintained state (re-query + // fallback). The state's baseline is now stale relative to the + // deltas it never saw — and a hash-suppressed re-query would + // validate emissions without re-syncing it — so drop it; the + // writer-ordered rebuild/reseed path restores an exact baseline. + // A maintained state only survives an unbroken chain of cycles + // it fully processed. + _dropMaintainedState(entry.ivm); entry.dirty = true; if (traceCorrelationId != null) { entry.pendingTraceCorrelationId = traceCorrelationId; @@ -243,6 +251,33 @@ final class StreamEngine { } } + /// Writer-ordered read used to build IVM caches and aggregate + /// snapshots. Reads through the writer port are FIFO-ordered against + /// the write replies that carry row deltas, so a snapshot's position + /// totally orders it against every delta: writes whose replies were + /// processed before the snapshot reply are included in it, and all + /// later deltas apply cleanly on top. (A reader-side snapshot has no + /// such ordering — it can observe a commit whose delta then lands + /// after the build, double-applying the write.) + Future>> Function( + String sql, + List params, + )? + _writerRead; + + /// Wired by [Database] once the writer isolate has spawned. Until (and + /// unless) attached, full/aggregate admissions are not made and streams + /// stay on the plain re-query path. + void attachWriterRead( + Future>> Function( + String sql, + List params, + ) + read, + ) { + _writerRead = read; + } + /// Per-table admission metadata (`PRAGMA table_info` + the CREATE /// statement from sqlite_master, which gates TEXT-equality admission on /// the absence of COLLATE clauses), shared across admissions. @@ -270,7 +305,9 @@ final class StreamEngine { /// and emitted). bool _tryIncrementalMaintain(StreamEntry entry, RowDeltaBatch batch) { final ivm = entry.ivm; - if (ivm == null || entry.inFlight || entry.dirty) return false; + if (ivm == null || entry.inFlight || entry.dirty) { + return false; + } final tableDeltas = batch.forTable(ivm.shape.table); if (tableDeltas == null || tableDeltas.isEmpty) { @@ -292,13 +329,11 @@ final class StreamEngine { case IvmFullState(): if (ivm.rows == null) { - final last = entry.lastResult; - if (last == null || !ivm.rebuild(last)) { - // The cached result cannot be keyed (non-int pk/order key, - // unexpected ordering). Structural — demote. - entry.ivm = null; - return false; - } + // Build (or rebuild after a bail/re-query) through the writer + // so the cache is FIFO-ordered against deltas; until it lands, + // writes keep falling back to re-query. + _scheduleFullBuild(entry, ivm); + return false; } switch (ivm.apply(tableDeltas)) { case IvmOutcome.unchanged: @@ -352,6 +387,20 @@ final class StreamEngine { } } + /// Drop maintained state whose delta chain has been broken. Skip-only + /// states carry nothing to drop. + void _dropMaintainedState(IvmState? state) { + switch (state) { + case IvmFullState ivm: + ivm.rows = null; + ivm.keys = null; + case IvmAggregateState ivm: + ivm.seeded = false; + case IvmSkipState() || null: + break; + } + } + /// Rows-identical check by element identity (apply() clones any row it /// changes, so identity captures "visibly unchanged" exactly). bool _sameRowList( @@ -383,7 +432,14 @@ final class StreamEngine { case null: if (kProfileMode) ProfileCounters.ivmRejectedTotal++; case IvmFullShape(): - entry.ivm = IvmFullState(shape); + if (_writerRead == null) { + if (kProfileMode) ProfileCounters.ivmRejectedTotal++; + return; + } + final state = IvmFullState(shape); + await _buildFullCache(entry, state); + if (entry.subscribers.isEmpty || entry.ivm != null) return; + entry.ivm = state; if (kProfileMode) ProfileCounters.ivmAdmittedTotal++; case IvmSkipShape(): entry.ivm = IvmSkipState(shape); @@ -392,6 +448,10 @@ final class StreamEngine { ProfileCounters.ivmAdmittedSkipTotal++; } case IvmAggregateShape(): + if (_writerRead == null) { + if (kProfileMode) ProfileCounters.ivmRejectedTotal++; + return; + } final state = IvmAggregateState(shape); await _seedAggregate(state, info); if (entry.subscribers.isEmpty || entry.ivm != null) return; @@ -411,19 +471,49 @@ final class StreamEngine { } } - /// Seed (or re-seed) an aggregate state with an exact snapshot. + /// Seed (or re-seed) an aggregate state with an exact, writer-ordered + /// snapshot. Future _seedAggregate( IvmAggregateState state, List> tableInfo, ) async { - final snapshot = await _pool.select( + final read = _writerRead; + if (read == null) return; + final snapshot = await read( buildAggregateSnapshotSql(state.shape, tableInfo), + const [], ); if (snapshot.length == 1) { state.seedFromSnapshot(snapshot.first); } } + /// Build a full-maintenance cache with a writer-ordered read of the + /// entry's own SQL. The result feeds only the cache — emissions remain + /// reader-driven until deltas start applying. + Future _buildFullCache(StreamEntry entry, IvmFullState state) async { + final read = _writerRead; + if (read == null) return; + final rows = await read(entry.sql, entry.params); + state.rebuild(rows); + } + + /// After a bail or re-query invalidated a full cache, rebuild it + /// asynchronously off the writer; deltas keep falling back meanwhile. + void _scheduleFullBuild(StreamEntry entry, IvmFullState state) { + if (state.buildInFlight) return; + state.buildInFlight = true; + unawaited(() async { + try { + await _buildFullCache(entry, state); + } catch (_) { + // Stay cacheless; the entry keeps re-querying. + } finally { + state.buildInFlight = false; + } + }()); + } + /// After an aggregate bail, re-seed asynchronously so a later write can /// resume maintenance; re-queries cover the interim. void _scheduleAggregateReseed(StreamEntry entry, IvmAggregateState state) { diff --git a/lib/src/stream_ivm.dart b/lib/src/stream_ivm.dart index c0d9c04d..511993a0 100644 --- a/lib/src/stream_ivm.dart +++ b/lib/src/stream_ivm.dart @@ -38,53 +38,116 @@ import 'row_deltas.dart'; // Predicates // --------------------------------------------------------------------------- +/// Predicate comparison op codes, pre-resolved at classification so the +/// per-delta hot path never switches on strings. +abstract final class IvmOp { + static const int eq = 0; + static const int lt = 1; + static const int le = 2; + static const int gt = 3; + static const int ge = 4; +} + final class IvmPredicate { - const IvmPredicate(this.columnIndex, this.op, this.value); + IvmPredicate(this.columnIndex, String op, this.value) + : opCode = switch (op) { + '=' => IvmOp.eq, + '<' => IvmOp.lt, + '<=' => IvmOp.le, + '>' => IvmOp.gt, + '>=' => IvmOp.ge, + _ => throw ArgumentError.value(op, 'op'), + }; /// Table column index (cid from `PRAGMA table_info`). final int columnIndex; - /// One of `=`, `<`, `<=`, `>`, `>=` (`==` normalizes to `=`; only `=` - /// is admitted for TEXT values). - final String op; + /// [IvmOp] code; only [IvmOp.eq] is admitted for TEXT values. + final int opCode; /// Comparison constant: an [int], or a [String] (TEXT equality under /// verified BINARY collation). Resolved from a literal or the stream's /// fixed bind parameters at classification time. final Object value; - /// Evaluates the predicate against a delta cell. Returns null when the - /// comparison is unprovable (type mismatch with the admitted constant). - /// A NULL cell never matches, mirroring SQL comparison semantics. - bool? evaluate(Object? cell) { - if (cell == null) return false; - final value = this.value; - if (value is int) { + String get op => switch (opCode) { + IvmOp.eq => '=', + IvmOp.lt => '<', + IvmOp.le => '<=', + IvmOp.gt => '>', + _ => '>=', + }; +} + +/// Flattened predicate conjunction: the per-delta hot path reads parallel +/// primitive arrays instead of walking object fields and switching on +/// strings (per-miss evaluation is the single hottest IVM operation — +/// every write x every admitted stream on the table runs it twice). +final class IvmPredicateProgram { + IvmPredicateProgram(List predicates) + : _columns = List.generate( + predicates.length, + (i) => predicates[i].columnIndex, + growable: false, + ), + _ops = List.generate( + predicates.length, + (i) => predicates[i].opCode, + growable: false, + ), + _intValues = List.generate( + predicates.length, + (i) => predicates[i].value is int ? predicates[i].value as int : 0, + growable: false, + ), + _textValues = List.generate( + predicates.length, + (i) => + predicates[i].value is String + ? predicates[i].value as String + : null, + growable: false, + ); + + final List _columns; + final List _ops; + final List _intValues; + final List _textValues; + + /// Evaluates the conjunction against table-indexed [values]. Returns + /// null when any term is unprovable (type mismatch). NULL cells never + /// match, mirroring SQL comparison semantics. + bool? evaluate(List values) { + for (var i = 0; i < _columns.length; i++) { + final cell = values[_columns[i]]; + final text = _textValues[i]; + if (text != null) { + if (cell == null) return false; + if (cell is! String) return null; + if (cell != text) return false; + continue; + } + if (cell == null) return false; if (cell is! int) return null; - return switch (op) { - '=' => cell == value, - '<' => cell < value, - '<=' => cell <= value, - '>' => cell > value, - '>=' => cell >= value, - _ => null, + final v = _intValues[i]; + final ok = switch (_ops[i]) { + IvmOp.eq => cell == v, + IvmOp.lt => cell < v, + IvmOp.le => cell <= v, + IvmOp.gt => cell > v, + _ => cell >= v, }; + if (!ok) return false; } - // TEXT equality (admission guarantees op == '='). - if (cell is! String) return null; - return cell == value as String; + return true; } } /// Evaluates a predicate conjunction against table-indexed [values]. -/// Returns null when any term is unprovable. +/// Returns null when any term is unprovable. (Test/diagnostic entry +/// point; states use a pre-built [IvmPredicateProgram].) bool? evaluatePredicates(List predicates, List values) { - for (final pred in predicates) { - final match = pred.evaluate(values[pred.columnIndex]); - if (match == null) return null; - if (!match) return false; - } - return true; + return IvmPredicateProgram(predicates).evaluate(values); } // --------------------------------------------------------------------------- @@ -786,11 +849,13 @@ sealed class IvmState { /// Tier 1.5 state: nothing cached; only proves misses. final class IvmSkipState extends IvmState { - IvmSkipState(this.shape); + IvmSkipState(this.shape) : _program = IvmPredicateProgram(shape.predicates); @override final IvmSkipShape shape; + final IvmPredicateProgram _program; + /// [IvmOutcome.unchanged] when every delta row fails the predicate both /// before and after the write; [IvmOutcome.bail] otherwise (a hit or an /// unprovable cell — the caller re-queries; there is no cache to drop). @@ -802,10 +867,10 @@ final class IvmSkipState extends IvmState { } final oldMatch = delta.oldValues == null ? false - : evaluatePredicates(shape.predicates, delta.oldValues!); + : _program.evaluate(delta.oldValues!); final newMatch = delta.newValues == null ? false - : evaluatePredicates(shape.predicates, delta.newValues!); + : _program.evaluate(delta.newValues!); if (oldMatch != false || newMatch != false) return IvmOutcome.bail; } return IvmOutcome.unchanged; @@ -814,11 +879,19 @@ final class IvmSkipState extends IvmState { /// Fully-maintained rows (optionally a top-K window). final class IvmFullState extends IvmState { - IvmFullState(this.shape); + IvmFullState(this.shape) : _program = IvmPredicateProgram(shape.predicates); @override final IvmFullShape shape; + final IvmPredicateProgram _program; + + /// Cached pks for O(1) presence checks on the (hot) proven-miss path. + Set? _pks; + + /// Guards concurrent writer-ordered cache builds after a bail. + bool buildInFlight = false; + /// Maintained rows in admitted order. For windowed shapes this is the /// top-K (or the complete filtered set when it is smaller). `null` /// until built from the entry's last result, and after any bail. @@ -857,6 +930,7 @@ final class IvmFullState extends IvmState { if (limit != null && newRows.length > limit) return false; rows = newRows; keys = newKeys; + _pks = {for (final (_, pk) in newKeys) pk}; complete = limit == null || newRows.length < limit; return true; } @@ -870,12 +944,14 @@ final class IvmFullState extends IvmState { var mutated = false; List> workRows = currentRows; List<(int, int)> workKeys = currentKeys; + Set workPks = _pks!; final limit = shape.limit; void ensureMutable() { if (!mutated) { workRows = List>.of(workRows); workKeys = List<(int, int)>.of(workKeys); + workPks = Set.of(workPks); mutated = true; } } @@ -888,12 +964,8 @@ final class IvmFullState extends IvmState { List? oldValues, List? newValues, ) { - final pOldOrNull = hasOld - ? evaluatePredicates(shape.predicates, oldValues!) - : false; - final pNewOrNull = hasNew - ? evaluatePredicates(shape.predicates, newValues!) - : false; + final pOldOrNull = hasOld ? _program.evaluate(oldValues!) : false; + final pNewOrNull = hasNew ? _program.evaluate(newValues!) : false; if (pOldOrNull == null || pNewOrNull == null) return false; final pOld = pOldOrNull; final pNew = pNewOrNull; @@ -917,7 +989,7 @@ final class IvmFullState extends IvmState { if (!pOld && !pNew) { // Proven miss — but a cached row with this pk means the cache is // out of sync with reality. - if (hasOld && _pkPresent(workRows, oldPk)) return false; + if (hasOld && workPks.contains(oldPk)) return false; return true; } @@ -925,7 +997,7 @@ final class IvmFullState extends IvmState { // The row stays in the filtered set; it may move or patch. final foundIdx = oldIdx != null && oldIdx >= 0 ? oldIdx - : _pkIndex(workRows, oldPk); + : (workPks.contains(oldPk) ? _pkIndex(workRows, oldPk) : -1); if (foundIdx < 0) { // Below an incomplete window before the write. if (limit == null || complete) return false; @@ -940,6 +1012,7 @@ final class IvmFullState extends IvmState { ensureMutable, () => workRows, () => workKeys, + () => workPks, newKey!, newValues, limit, @@ -954,6 +1027,7 @@ final class IvmFullState extends IvmState { ensureMutable(); workRows.removeAt(foundIdx); workKeys.removeAt(foundIdx); + workPks.remove(oldPk); if (limit != null && !complete && workKeys.isNotEmpty && @@ -966,6 +1040,7 @@ final class IvmFullState extends IvmState { final insertAt = _insertionPoint(workKeys, newKey!); workRows.insert(insertAt, patched); workKeys.insert(insertAt, newKey); + workPks.add(newPk); return true; } @@ -973,7 +1048,7 @@ final class IvmFullState extends IvmState { // Row leaves the filtered set. final foundIdx = oldIdx != null && oldIdx >= 0 ? oldIdx - : _pkIndex(workRows, oldPk); + : (workPks.contains(oldPk) ? _pkIndex(workRows, oldPk) : -1); if (foundIdx < 0) { // Below the window of an incomplete windowed cache: invisible. if (limit != null && !complete) return true; @@ -982,6 +1057,7 @@ final class IvmFullState extends IvmState { ensureMutable(); workRows.removeAt(foundIdx); workKeys.removeAt(foundIdx); + workPks.remove(oldPk); if (limit != null && !complete && workRows.length < limit) { // The window lost a member and the replacement is unknown. return false; @@ -990,7 +1066,7 @@ final class IvmFullState extends IvmState { } // Row enters the filtered set. - if (_pkPresent(workRows, newPk)) return false; + if (workPks.contains(newPk)) return false; if (limit != null && !complete) { final lastKey = workKeys.isEmpty ? null : workKeys.last; if (workKeys.length >= limit && @@ -1003,6 +1079,7 @@ final class IvmFullState extends IvmState { ensureMutable, () => workRows, () => workKeys, + () => workPks, newKey!, newValues!, limit, @@ -1058,11 +1135,13 @@ final class IvmFullState extends IvmState { if (limit != null && complete && workRows.length > limit + 64) { workRows = workRows.sublist(0, limit); workKeys = workKeys.sublist(0, limit); + workPks = {for (final (_, pk) in workKeys) pk}; complete = false; } rows = workRows; keys = workKeys; + _pks = workPks; return IvmOutcome.applied; } @@ -1078,6 +1157,7 @@ final class IvmFullState extends IvmState { void Function() ensureMutable, List> Function() getRows, List<(int, int)> Function() getKeys, + Set Function() workPksOf, (int, int) key, List values, int? limit, @@ -1090,10 +1170,12 @@ final class IvmFullState extends IvmState { final insertAt = _insertionPoint(keys, key); rows.insert(insertAt, inserted); keys.insert(insertAt, key); + workPksOf().add(key.$2); if (limit != null && !complete && rows.length > limit) { // The displaced row is no longer the window's business. + final removedKey = keys.removeLast(); rows.removeLast(); - keys.removeLast(); + workPksOf().remove(removedKey.$2); } return true; } @@ -1101,6 +1183,7 @@ final class IvmFullState extends IvmState { IvmOutcome _bail() { rows = null; keys = null; + _pks = null; return IvmOutcome.bail; } @@ -1143,9 +1226,6 @@ final class IvmFullState extends IvmState { return idx >= 0 ? idx : -(idx + 1); } - bool _pkPresent(List> rows, int pk) => - _pkIndex(rows, pk) >= 0; - int _pkIndex(List> rows, int pk) { for (var i = 0; i < rows.length; i++) { if (rows[i][shape.pkOutputName] == pk) return i; @@ -1156,11 +1236,14 @@ final class IvmFullState extends IvmState { /// Tier 3 state: exact aggregate values maintained from deltas. final class IvmAggregateState extends IvmState { - IvmAggregateState(this.shape); + IvmAggregateState(this.shape) + : _program = IvmPredicateProgram(shape.predicates); @override final IvmAggregateShape shape; + final IvmPredicateProgram _program; + /// Whether the state has been seeded (from the entry's first result for /// COUNT(*)-only shapes, otherwise from the admission snapshot query). bool seeded = false; @@ -1259,10 +1342,10 @@ final class IvmAggregateState extends IvmState { } final oldMatch = delta.oldValues == null ? false - : evaluatePredicates(shape.predicates, delta.oldValues!); + : _program.evaluate(delta.oldValues!); final newMatch = delta.newValues == null ? false - : evaluatePredicates(shape.predicates, delta.newValues!); + : _program.evaluate(delta.newValues!); if (oldMatch == null || newMatch == null) return _bail(); if (oldMatch && !contribute(delta.oldValues!, -1)) return _bail(); if (newMatch && !contribute(delta.newValues!, 1)) return _bail(); diff --git a/test/stream_ivm_equivalence_test.dart b/test/stream_ivm_equivalence_test.dart new file mode 100644 index 00000000..701e52a8 --- /dev/null +++ b/test/stream_ivm_equivalence_test.dart @@ -0,0 +1,261 @@ +/// Randomized equivalence harness for tiered incremental maintenance. +/// +/// Registers one stream per admission mode (full, windowed full, +/// skip-only, aggregate, text-equality) plus an inadmissible control, +/// then runs seeded random write storms — inserts, updates, deletes, +/// rowid changes, NULL cells, transactions, savepoint rollbacks, and +/// capture-overflow batches. After every storm settles, each stream's +/// latest emission must equal a fresh `select()` of the same SQL. +/// +/// This is the load-bearing safety net for the IVM tiers: any divergence +/// between maintained state and re-query semantics fails here, whichever +/// path produced the emission. +import 'dart:async'; +import 'dart:io'; +import 'dart:math' as math; + +import 'package:resqlite/resqlite.dart'; +import 'package:test/test.dart'; + +const _seeds = [7, 160, 4242]; +const _roundsPerSeed = 12; +const _writesPerRound = 18; + +final class _Watched { + _Watched(this.label, this.sql, this.params); + + final String label; + final String sql; + final List params; + final List>> emissions = []; + StreamSubscription>>? sub; +} + +void main() { + for (final seed in _seeds) { + test('emissions equal fresh selects under write storm (seed $seed)', + () async { + final tempDir = await Directory.systemTemp.createTemp( + 'resqlite_ivm_equiv_', + ); + final db = await Database.open('${tempDir.path}/t.db'); + addTearDown(() async { + await db.close(); + try { + await tempDir.delete(recursive: true); + } on PathNotFoundException { + // ignore + } + }); + + await db.execute( + 'CREATE TABLE msgs(id INTEGER PRIMARY KEY, conv INTEGER NOT NULL, ' + 'score INTEGER, body TEXT NOT NULL, kind TEXT NOT NULL)', + ); + final prng = math.Random(seed); + var nextId = 1; + await db.executeBatch( + 'INSERT INTO msgs(id, conv, score, body, kind) VALUES (?, ?, ?, ?, ?)', + [ + for (; nextId <= 60; nextId++) + [ + nextId, + nextId % 6, + prng.nextBool() ? null : prng.nextInt(1000), + 'b$nextId', + nextId % 3 == 0 ? 'pin' : 'note', + ], + ], + ); + + final watched = <_Watched>[ + _Watched( + 'full range', + 'SELECT id, conv, score FROM msgs ' + 'WHERE id >= ? AND id < ? ORDER BY id', + [10, 40], + ), + _Watched( + 'full keyed', + 'SELECT * FROM msgs WHERE id = ?', + [25], + ), + _Watched( + 'windowed feed', + 'SELECT id, score FROM msgs WHERE conv = ? ' + 'ORDER BY score DESC, id DESC LIMIT 5', + [2], + ), + _Watched( + 'windowed asc', + 'SELECT id, conv FROM msgs ORDER BY conv, id LIMIT 7', + const [], + ), + _Watched( + 'skip-only pane', + 'SELECT id, body FROM msgs WHERE conv = ? ' + 'ORDER BY score DESC LIMIT 4', + [3], + ), + _Watched( + 'text equality', + "SELECT id, kind FROM msgs WHERE kind = 'pin' ORDER BY id", + const [], + ), + _Watched( + 'aggregates', + 'SELECT COUNT(*) AS n, SUM(score) AS total, MIN(score) AS lo, ' + 'MAX(score) AS hi, AVG(score) AS mean ' + 'FROM msgs WHERE conv = ?', + [2], + ), + _Watched( + 'global count', + 'SELECT COUNT(*) AS n FROM msgs', + const [], + ), + _Watched( + 'inadmissible control', + 'SELECT conv, COUNT(*) AS n FROM msgs ' + 'WHERE id > 0 GROUP BY conv ORDER BY conv', + const [], + ), + ]; + + for (final w in watched) { + w.sub = db.stream(w.sql, w.params).listen(w.emissions.add); + addTearDown(() => w.sub!.cancel()); + } + + Future settle() async { + // Quiet-window drain: counts stable across consecutive windows. + int total() => + watched.fold(0, (a, w) => a + w.emissions.length); + var last = total(); + var quiet = 0; + final deadline = DateTime.now().add(const Duration(seconds: 20)); + while (quiet < 2 && DateTime.now().isBefore(deadline)) { + await Future.delayed(const Duration(milliseconds: 40)); + final now = total(); + if (now == last) { + quiet++; + } else { + quiet = 0; + last = now; + } + } + } + + await settle(); + for (final w in watched) { + expect(w.emissions, isNotEmpty, reason: '${w.label}: no initial'); + } + // Let async admissions land before the storm so maintained paths + // actually engage. + await Future.delayed(const Duration(milliseconds: 100)); + + final liveIds = List.generate(60, (i) => i + 1); + + Future oneWrite(math.Random prng) async { + final roll = prng.nextInt(100); + if (roll < 35 || liveIds.isEmpty) { + // Insert. + final id = nextId++; + liveIds.add(id); + await db.execute( + 'INSERT INTO msgs(id, conv, score, body, kind) ' + 'VALUES (?, ?, ?, ?, ?)', + [ + id, + prng.nextInt(6), + prng.nextBool() ? null : prng.nextInt(1000), + 'b$id', + prng.nextBool() ? 'pin' : 'note', + ], + ); + } else if (roll < 65) { + // Update (sometimes to NULL score, sometimes kind flips). + final id = liveIds[prng.nextInt(liveIds.length)]; + await db.execute( + 'UPDATE msgs SET conv = ?, score = ?, kind = ? WHERE id = ?', + [ + prng.nextInt(6), + prng.nextBool() ? null : prng.nextInt(1000), + prng.nextBool() ? 'pin' : 'note', + id, + ], + ); + } else if (roll < 80) { + // Delete. + final idx = prng.nextInt(liveIds.length); + final id = liveIds.removeAt(idx); + await db.execute('DELETE FROM msgs WHERE id = ?', [id]); + } else if (roll < 90) { + // Rowid change. + final idx = prng.nextInt(liveIds.length); + final oldId = liveIds[idx]; + final newId = nextId++; + liveIds[idx] = newId; + await db.execute('UPDATE msgs SET id = ? WHERE id = ?', [ + newId, + oldId, + ]); + } else if (roll < 96) { + // Transaction, occasionally with a rolled-back savepoint. + await db.transaction((tx) async { + final id = nextId++; + liveIds.add(id); + await tx.execute( + 'INSERT INTO msgs(id, conv, score, body, kind) ' + "VALUES (?, ?, ?, ?, 'note')", + [id, prng.nextInt(6), prng.nextInt(1000), 'tx$id'], + ); + if (prng.nextBool()) { + try { + await tx.transaction((tx2) async { + await tx2.execute( + 'UPDATE msgs SET score = 777777 WHERE id = ?', + [liveIds[prng.nextInt(liveIds.length)]], + ); + throw StateError('undo'); + }); + } on StateError { + // expected + } + } + }); + } else { + // Capture-overflow batch (> 256 delta rows poisons the cycle). + final base = nextId; + nextId += 300; + liveIds.addAll([for (var i = base; i < base + 300; i++) i]); + await db.executeBatch( + 'INSERT INTO msgs(id, conv, score, body, kind) ' + "VALUES (?, ?, ?, ?, 'note')", + [ + for (var i = base; i < base + 300; i++) + [i, i % 6, i % 7 == 0 ? null : i, 'bulk$i'], + ], + ); + } + } + + for (var round = 0; round < _roundsPerSeed; round++) { + for (var w = 0; w < _writesPerRound; w++) { + await oneWrite(prng); + } + await settle(); + for (final w in watched) { + final fresh = await db.select(w.sql, w.params); + expect( + w.emissions.last, + fresh, + reason: + '${w.label} diverged at seed $seed round $round ' + '(emissions: ${w.emissions.length})', + ); + } + } + }); + } +} From e10d9f2730afa27bc438d488c153db9e651f26f5 Mon Sep 17 00:00:00 2001 From: Dan Reynolds Date: Wed, 10 Jun 2026 19:12:04 -0400 Subject: [PATCH 09/10] exp 160: v2 tier record + cross-port ordering journal lesson Co-Authored-By: Claude Fable 5 --- experiments/160-stream-delta-ivm.md | 84 ++++++++++++++++++++++++++++- experiments/JOURNAL.md | 24 +++++++++ 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/experiments/160-stream-delta-ivm.md b/experiments/160-stream-delta-ivm.md index b2ba6932..0f3f3adc 100644 --- a/experiments/160-stream-delta-ivm.md +++ b/experiments/160-stream-delta-ivm.md @@ -1,4 +1,4 @@ -# Experiment 160: Tier-1 incremental stream maintenance (row deltas) +# Experiment 160: Tiered incremental stream maintenance (row deltas) **Date:** 2026-06-10 **Status:** In Review @@ -208,6 +208,88 @@ the existing busy guard real; the sacrifice path clears the bracket before `Isolate.exit`. Validation: crash reproduced locally pre-fix; 100/100 clean stress iterations post-fix. +## Tier expansion (v2 stream engine) + +Building on tier 1's foundation, the classifier generalized into three +fail-closed admission modes sharing one strict grammar (string literals, +aggregate calls, AS aliases, DESC, LIMIT/OFFSET, DISTINCT): + +- **Full maintenance** gained composite ordering + (`ORDER BY intCol [DESC], pk [DESC]` — explicit pk tiebreak required so + tie order is exact) and `LIMIT K` windows: a top-K cache with + complete-set tracking. Entries and in-window patches are O(delta); + departures from a full incomplete window and boundary-crossing moves + fall back (the replacement row is unknown). TEXT equality predicates + admit when the table's CREATE statement (sqlite_master, cached per + table) contains no COLLATE clause, making BINARY semantics provable. +- **Tier 1.5 skip-only**: shapes whose results cannot be maintained + (DESC without tiebreak, OFFSET, DISTINCT, unprojected keys, aggregate + mixes) but whose WHERE is an evaluable conjunction get proven-miss + elision with no cached state; any hit or unprovable cell re-queries. +- **Tier 3 aggregates**: `COUNT(*)/COUNT/SUM/MIN/MAX/AVG(col) AS alias` + over an evaluable (possibly empty) predicate, seeded exactly by a + writer-ordered snapshot and maintained per delta (SQL NULL semantics; + exact integer sums; AVG = sum/count; a departing MIN/MAX extremum + bails and re-seeds). + +Hot path: predicate conjunctions compile to flattened primitive arrays +(`IvmPredicateProgram`), and full states keep a pk set for O(1) presence +checks on the proven-miss path. + +### Two ordering defects the equivalence harness caught + +Both surfaced only under `dart test` scheduling load (~1-in-5 runs) and +were invisible to deterministic same-seed replays: + +1. **Reader-built baselines race late writer replies.** An aggregate + snapshot (or cache build) executed on a reader can observe a commit + whose delta is then applied on top of it — cross-port event delivery + gives no happens-before between a reader result and the writer reply + for a write it already saw. Fix: all IVM state is built through + **writer-ordered reads** (a `writer.locked` select hook wired by + `Database`): the writer port is FIFO with the writes themselves, so a + snapshot's port position totally orders it against every delta. +2. **A maintained state only survives an unbroken chain of processed + cycles.** A delta-bearing write routed to the re-query fallback + (dirty/in-flight guard, absent or malformed deltas, capture overflow) + leaves the state's baseline permanently stale — and a hash-suppressed + re-query validates *emissions* without re-syncing *state* (captured + in an event ledger: seed lands at 2784, the next insert is swallowed + by an unchanged-hash re-query, every later apply walks the −1 + forward). Fix: the engine drops maintained state whenever a cycle + bypasses it; the writer-ordered rebuild restores an exact baseline. + Known trade: churn cycles (overflow batches, unknown-deps fallbacks) + trigger rebuild storms; steady state is untouched. + +The randomized equivalence harness +(`test/stream_ivm_equivalence_test.dart`: 3 seeds × 12 rounds × 9 +streams across every admission mode, with rowid changes, NULLs, +savepoint rollbacks, and overflow batches; every emission compared to a +fresh select) is the load-bearing safety net — 20/20 clean full-loop +runs post-fix. + +### Admission audit, v2 (app-shaped stream mix) + +| stream shape | mode | burst emissions | +|---|---|---:| +| message pane (JOIN + DESC + LIMIT) | no | 57 | +| message pane, denormalized (DESC + LIMIT) | **skip-only** | 47 | +| conversation list (no tiebreak) | no | ~1,900 | +| conversation list (pk tiebreak) | **windowed** | ~2,160 | +| unread badge | **aggregate** | 47 | +| user card | **full** | 1 | +| full transcript | **full** | 48 | +| feed page (`created_at DESC, id DESC LIMIT 50`) | **windowed** | 30 | +| author drafts | **full** | 3 | + +7/9 shapes admitted (was 3/8 at tier 1); 52/63 distinct entries. +Burst: `ivm_skipped=7,325 / applied=318 / bail=0 / hit_fallback=48` — +**7,643 invalidation decisions resolved without a reader re-query** +(tier 1: 3,000) at a burst wall of 132.9 ms (main's equivalent ~200 ms +while delivering a fraction of the emissions). The two remaining +re-query shapes have documented upgrade paths: the JOIN pane (tier 4) +and the tiebreak-less conversation list (add `, id DESC`). + ## Future Notes - **Emission cadence**: IVM emits per write where the re-query path diff --git a/experiments/JOURNAL.md b/experiments/JOURNAL.md index f1e04bae..aa729293 100644 --- a/experiments/JOURNAL.md +++ b/experiments/JOURNAL.md @@ -229,6 +229,30 @@ the signal real. A guard that silently stopped being maintained is strictly worse than no guard, because readers of the consuming code assume protection.* +### Cross-port replies have no happens-before — build derived state on the port that defines its baseline + +[Exp 160](160-stream-delta-ivm.md)'s tier expansion seeded aggregate +state from a reader-pool snapshot. Under load, the snapshot could +observe a commit whose writer reply — the event that carries the row +delta — was still queued: reader results and writer replies arrive on +different ports, and the event loop guarantees nothing about their +relative order. The delta then applied on top of a baseline that already +included it. The fix was structural, not a sleep: route every IVM state +build through the writer port, whose FIFO totally orders a snapshot +against the deltas it must compose with. A second instance of the same +class: maintained state that misses even one delta-bearing cycle (a +dirty-routed write whose covering re-query was hash-suppressed) drifts +permanently — state must be dropped the moment its chain breaks, because +emission-level checks cannot re-synchronize it. Both defects fired in +~1-in-5 `dart test` runs and in zero deterministic same-seed replays; +the randomized equivalence harness (every emission vs a fresh select) +was the only net that caught them. + +*Reapplies whenever derived state composes a snapshot with an event +stream. Ask which port/queue defines the baseline's position and build +the snapshot there; treat any bypassed event as poison, not as "covered +elsewhere".* + ## How to add to this file Add an entry when an experiment surfaces a transferable lesson — something a From 984d24b61200023cbac02211e2c393e1c157dfb0 Mon Sep 17 00:00:00 2001 From: Dan Reynolds Date: Wed, 10 Jun 2026 19:25:19 -0400 Subject: [PATCH 10/10] exp 160 v2: gate results, signals, and tier record MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit many-streams-writer-throughput clears the Tracelite primary gate with a formal `improved` verdict (-18.2%, 546 -> 447 ms, p < 0.001) — the first cleared gate in the stream-rerun-dispatch direction's history — and reproduces order-flipped (+19.5% main-slower, CI +83..+94 ms). High-cardinality carries a real ~4% cost in both orders: per-write main-isolate predicate evaluation is O(admitted streams on the table), replacing reader-distributed hash suppression; the identified v3 fix is a per-table equality-predicate index for O(1) delta dispatch. Keyed-PK is honestly neutral for the v2 stack (sign flips across passes). Co-Authored-By: Claude Fable 5 --- .../exp-147-writer-sqlite-wall-aggregate.md | 16 ++++----- experiments/160-stream-delta-ivm.md | 34 +++++++++++++++++++ experiments/signals.json | 11 ++++-- 3 files changed, 50 insertions(+), 11 deletions(-) diff --git a/benchmark/profile/results/exp-147-writer-sqlite-wall-aggregate.md b/benchmark/profile/results/exp-147-writer-sqlite-wall-aggregate.md index 6cc9f631..3a1475f6 100644 --- a/benchmark/profile/results/exp-147-writer-sqlite-wall-aggregate.md +++ b/benchmark/profile/results/exp-147-writer-sqlite-wall-aggregate.md @@ -16,19 +16,19 @@ dart run -DRESQLITE_PROFILE=true benchmark/profile/writer_sqlite_wall_audit.dart | workload | shape | wall_ms | writer_sqlite_us | writer_sqlite_count | invalidate_us | invalidate_count | residual_us | parked_total | max_parked | emissions | |---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:| -| A11c baseline | 0 streams x 500 writes | 72.65 | 22637 | 500 | 0 | 0 | 50012 | 0 | 0 | 0 | -| A11c disjoint | 50 streams x 500 writes | 114.21 | 20228 | 500 | 26671 | 500 | 67315 | 0 | 0 | 0 | -| A11c overlap | 50 streams x 500 writes | 206.36 | 19671 | 500 | 137389 | 500 | 49298 | 0 | 0 | 500 | -| keyed PK subscriptions | 50 streams x 200 random writes | 48.06 | 5918 | 200 | 21862 | 200 | 20285 | 0 | 0 | 3 | +| A11c baseline | 0 streams x 500 writes | 55.32 | 18370 | 500 | 0 | 0 | 36953 | 0 | 0 | 0 | +| A11c disjoint | 50 streams x 500 writes | 86.32 | 14234 | 500 | 11393 | 500 | 60695 | 0 | 0 | 0 | +| A11c overlap | 50 streams x 500 writes | 93.86 | 16032 | 500 | 35049 | 500 | 42780 | 0 | 0 | 500 | +| keyed PK subscriptions | 50 streams x 200 random writes | 23.93 | 3989 | 200 | 10130 | 200 | 9806 | 0 | 0 | 3 | ## Derived fractions | workload | SQLite / wall | invalidation / wall | residual / wall | SQLite us/write | invalidation us/write | ns/intersection entry | |---|---:|---:|---:|---:|---:|---:| -| A11c baseline | 31.16% | 0.00% | 68.84% | 45.27 | 0.00 | 0 | -| A11c disjoint | 17.71% | 23.35% | 58.94% | 40.46 | 53.34 | 339 | -| A11c overlap | 9.53% | 66.58% | 23.89% | 39.34 | 274.78 | 233 | -| keyed PK subscriptions | 12.31% | 45.48% | 42.20% | 29.59 | 109.31 | 356 | +| A11c baseline | 33.20% | 0.00% | 66.80% | 36.74 | 0.00 | 0 | +| A11c disjoint | 16.49% | 13.20% | 70.31% | 28.47 | 22.79 | 166 | +| A11c overlap | 17.08% | 37.34% | 45.58% | 32.06 | 70.10 | 136 | +| keyed PK subscriptions | 16.67% | 42.34% | 40.99% | 19.95 | 50.65 | 56 | ## Reading the table diff --git a/experiments/160-stream-delta-ivm.md b/experiments/160-stream-delta-ivm.md index 0f3f3adc..af2fcc3b 100644 --- a/experiments/160-stream-delta-ivm.md +++ b/experiments/160-stream-delta-ivm.md @@ -290,6 +290,40 @@ while delivering a fraction of the emissions). The two remaining re-query shapes have documented upgrade paths: the JOIN pane (tier 4) and the tiebreak-less conversation list (add `, id DESC`). +### Tracelite gate, v2 stack + +Standard-order pass (3 runs per side): + +| scenario | delta | evidence | verdict | +|---|---:|---|---| +| many-streams-writer-throughput | **−18.2%** (546 → 447 ms) | CV 1.5%, p < 0.001 | **`improved` — primary gate cleared** | +| keyed-pk-subscriptions | −5.5% | CI −22.5..−4.4 ms (excludes zero); baseline CV 0.12 vs candidate 0.01 | too_noisy label, win signal | +| high-cardinality-fanout | +3.55% | CI +7.3..+16.4 ms | real small cost (see below) | + +The many-streams `improved` verdict is the first formally-cleared +primary gate in this direction's history — every prior stream +experiment topped out at "neutral". + +The high-cardinality cost has a clear mechanism: 100 admitted streams +on one table mean every write runs O(streams) predicate evaluations on +the main isolate, replacing the baseline's reader-distributed hash +suppression. At that density they trade ~3.5% wall for per-write patch +delivery. The identified fix (next step, not in this branch): a +per-table **equality-predicate index** grouping admitted streams by +(column, value) so a delta dispatches to its interested streams in +O(1) instead of a linear scan — the same idea column elision applies at +table granularity, pushed down to predicate granularity. Order-flipped +confirmation of this pass recorded below. + +**Order-flipped confirmation** (v2 collected first, main second; deltas +as main-vs-v2): many-streams **+19.5% main-slower** (CI +83.3..+94.1 ms, +CV 1.9%) — the headline win reproduces in both collection orders. +High-cardinality: main −4.79% (CI −28.7..−4.8 ms) — the ~4% v2 cost is +real in both orders, as the mechanism predicts. Keyed-PK flipped sign +across passes (−5.5% / +3.9%) — honestly neutral for the v2 stack +(tier 1 alone showed a clear keyed-PK win; the v2 delta capture and +admission machinery absorb part of it on this scenario's shape). + ## Future Notes - **Emission cadence**: IVM emits per write where the re-query path diff --git a/experiments/signals.json b/experiments/signals.json index c71a2821..a124168d 100644 --- a/experiments/signals.json +++ b/experiments/signals.json @@ -654,11 +654,16 @@ "Row-level dirty precision is mergeable when admission is a registration-time, fail-closed classifier instead of exp 134's write-path SQL recognizer: tier-1 (single table, integer predicates, INTEGER-PK ordering) covers the canonical stream workloads and turns 98% of A11c-overlap invalidation decisions into proven misses with zero reader dispatches and zero bails", "The preupdate hook's old/new row values are capturable within bounded cost (256 rows / 32 cols / 256 KB per cycle, poisoned by savepoint rollback) and ship as raw bytes decoded lazily on the main isolate", "Maintained streams change emission cadence: each write's patch is delivered synchronously instead of coalescing behind re-query latency (A11c overlap 44 -> 500 emissions during the same burst) — the semantics an infinitely-fast re-query would produce", - "Tracelite stream gate, both collection orders: many-streams -18.5% / +22.2% main-slower (CIs -122..-96 ms and +100..+113 ms), keyed-PK -14.1% / +9.4% (CIs -60..-27 ms and +13..+37 ms) with keyed-PK within-run CV dropping 0.13-0.15 -> 0.02-0.05 under IVM in both passes; the standard-order pass's high-cardinality +2.11% flag did not reproduce order-flipped (CI -3.6..+9.6 ms) — drift, the exp 159 journal pattern" + "Tracelite stream gate, both collection orders: many-streams -18.5% / +22.2% main-slower (CIs -122..-96 ms and +100..+113 ms), keyed-PK -14.1% / +9.4% (CIs -60..-27 ms and +13..+37 ms) with keyed-PK within-run CV dropping 0.13-0.15 -> 0.02-0.05 under IVM in both passes; the standard-order pass's high-cardinality +2.11% flag did not reproduce order-flipped (CI -3.6..+9.6 ms) — drift, the exp 159 journal pattern", + "Tier expansion (v2): one fail-closed grammar classifies into full maintenance (composite ORDER BY with explicit pk tiebreak + LIMIT-K windows), skip-only proven-miss elision, and exactly-maintained aggregates seeded by writer-ordered snapshots; TEXT equality admits when the table CREATE sql has no COLLATE clause", + "Reader-built IVM baselines race late writer replies (no cross-port happens-before); all IVM state builds through writer-ordered reads, and maintained state drops whenever a delta-bearing cycle bypasses it — both defects found only by the randomized equivalence harness (~1-in-5 dart test runs, never in deterministic replays)", + "v2 gate: many-streams -18.2% with a formal `improved` primary-gate verdict (first in this direction's history); keyed-PK -5.5% (CI excludes zero); high-cardinality +3.55% real cost from O(admitted-streams) main-isolate predicate eval at 100-streams-one-table density", + "v2 admission on the app-shaped mix: 7/9 shapes, 7,643 invalidation decisions resolved without reader re-query, 0 bails, burst wall 132.9 ms vs main's ~200 ms while delivering per-write patches" ], "nextSignals": [ - "delta capture currently runs unconditionally (bounded); if any write-path benchmark ever flags it, gate capture on a writer-side admitted-streams flag", - "tier-2 candidates in evidence order: TEXT equality under introspected BINARY collation, LIMIT windows with K+buffer caches, SQLite-side predicate evaluation against bound delta rows", + "per-table equality-predicate index: group admitted streams by (column, value) so a delta dispatches to interested streams in O(1) — removes the v2 high-cardinality +3.55% linear-eval cost and is the natural v3 dispatch structure", + "tier 4 one-hop joins: maintain the base stream and patch joined columns via keyed point lookups; demand property-based equivalence testing first", + "rebuild storms after churn cycles (overflow/unknown-deps drop all maintained states, then N writer-ordered rebuilds): batch or coalesce rebuilds if a workload shows it", "if subscribers prefer coalesced emissions over per-write patches, apply the exp 045 microtask-batch pattern to IVM emissions and measure" ] },