Patch Doltlite mergeScan to skip tree entries covered by a mutmap DELETE

When prollyBtCursorIndexMoveto lands a cursor directly at a mutmap
INSERT entry via setCursorToMutMapEntryPhys, mmIdx can jump past
sort-earlier DELETE entries. The subsequent scan's mergeScan emits
those tree entries as live because the "cmp*dir < 0" branch doesn't
consult the mutmap for the current tree key. SQLite then follows the
emitted rowid into TableMoveto and trips "database disk image is
malformed" in sqlite3VdbeFinishMoveto.

Fix reproduced at the CLI level: inside BEGIN IMMEDIATE on a composite-PK
table, the sequence INSERT + DELETE + UPDATE-with-prefix-where produces
the malformed-disk-image error without the patch and returns the
expected single updated row with it.

Drop the diagnostic step; verify each patch's effect in-line instead.

Author: JanJakes

.github/doltlite-patches/mergeScan-check-tree-delete.patch

@@ -0,0 +1,61 @@
+Check mutmap for tree-key DELETE when emitting a tree entry in mergeScan
+
+prollyBtCursorIndexMoveto can position the merge cursor directly at a
+mutmap INSERT entry via setCursorToMutMapEntryPhys, which jumps mmIdx
+past any sort-earlier mutmap entries. The subsequent mergeScan
+iteration, on seeing `cmp*dir < 0` (tree entry ahead of mutmap[mmIdx]),
+emits the tree entry without noticing that the mutmap contains a
+DELETE for that tree key at an earlier order index. The tree key was
+logically deleted but stays visible to the scan — SQLite then follows
+its rowid and calls TableMoveto to a rowid that mutmap says is
+deleted, which hits the "database disk image is malformed" check in
+sqlite3VdbeFinishMoveto.
+
+Fix: when about to emit a tree entry, look up the current tree key in
+the mutmap; if a DELETE record exists, skip the tree entry and advance
+the tree cursor. No behavior change in the common case (no matching
+mutmap entry → emit tree as before), constant-factor overhead
+otherwise.
+
+Reproduces deterministically on:
+
+  CREATE TABLE c (ka TEXT, kb TEXT, kc TEXT, v TEXT,
+                  PRIMARY KEY (ka, kb, kc));
+  INSERT INTO c VALUES ('db','t','a','');
+  BEGIN IMMEDIATE;
+  INSERT INTO c VALUES ('db','t','b','');
+  DELETE FROM c WHERE ka='db' AND kb='t' AND kc='a';
+  UPDATE c SET v = 'x' WHERE ka='db' AND kb='t';  -- malformed w/o fix
+
+--- a/src/prolly_btree.c
++++ b/src/prolly_btree.c
+@@ -3631,6 +3631,29 @@
+     }
+     cmp = mergeCompare(pCur, e);
+     if( cmp*dir < 0 ){
++      /* Tree entry is ahead of mutmap[mmIdx] in scan direction.
++      ** Check whether the mutmap has a DELETE entry for the tree
++      ** key at an order index the iteration has already walked
++      ** past (e.g. after setCursorToMutMapEntryPhys jumped mmIdx
++      ** directly to a later INSERT). Without this check the scan
++      ** would emit a logically-deleted tree row and SQLite would
++      ** later TableMoveto a rowid that mutmap says is gone. */
++      ProllyMutMapEntry *delE = 0;
++      int delRc;
++      if( pCur->curIntKey ){
++        delRc = prollyMutMapFindRc(pCur->pMutMap, 0, 0,
++                                   prollyCursorIntKey(&pCur->pCur), &delE);
++      }else{
++        const u8 *pK; int nK;
++        prollyCursorKey(&pCur->pCur, &pK, &nK);
++        delRc = prollyMutMapFindRc(pCur->pMutMap, pK, nK, 0, &delE);
++      }
++      if( delRc!=SQLITE_OK ) return delRc;
++      if( delE && delE->op==PROLLY_EDIT_DELETE ){
++        int advRc = advanceTreeCursor(pCur, dir);
++        if( advRc!=SQLITE_OK ) return advRc;
++        continue;
++      }
+       pCur->mergeSrc = MERGE_SRC_TREE;
+       if( pRes ) *pRes = 0;
+       return SQLITE_OK;

.github/workflows/phpunit-tests-doltlite.yml

@@ -65,8 +65,12 @@ jobs:
           patch -d doltlite-src -p1 --no-backup-if-mismatch \
             < .github/doltlite-patches/eqSeen-preservation.patch
 
+          # --- Patch 3: check mutmap for tree-key DELETE in mergeScan ---
+          patch -d doltlite-src -p1 --no-backup-if-mismatch \
+            < .github/doltlite-patches/mergeScan-check-tree-delete.patch
+
           echo "--- Patched lines ---"
-          grep -n 'patched off in CI\|savedEqSeen' "$BUILD" "$MOVETO"
+          grep -n 'patched off in CI\|savedEqSeen\|walked past' "$BUILD" "$MOVETO"
 
       - name: Build Doltlite shared library
         run: |
@@ -155,6 +159,25 @@ jobs:
             echo "::error::eqSeen-preservation patch did not take effect: expected only 'id', got '${REMAINING}'"
             exit 1
           fi
+          # 3) mergeScan patch: UPDATE inside a transaction after an
+          # INSERT + DELETE no longer trips "database disk image is
+          # malformed".
+          VAL=$(php -r '
+            $db = new PDO("sqlite::memory:");
+            $db->exec("CREATE TABLE c (ka TEXT, kb TEXT, kc TEXT, v TEXT, PRIMARY KEY (ka, kb, kc))");
+            $db->exec("INSERT INTO c VALUES (\"db\", \"t\", \"a\", \"\")");
+            $db->exec("BEGIN IMMEDIATE");
+            $db->exec("INSERT INTO c VALUES (\"db\", \"t\", \"b\", \"\")");
+            $db->exec("DELETE FROM c WHERE ka = \"db\" AND kb = \"t\" AND kc = \"a\"");
+            $db->exec("UPDATE c SET v = \"x\" WHERE ka = \"db\" AND kb = \"t\"");
+            $db->exec("COMMIT");
+            echo $db->query("SELECT v FROM c")->fetch()[0];
+          ' 2>&1)
+          echo "post-UPDATE value = ${VAL}"
+          if [ "${VAL}" != "x" ]; then
+            echo "::error::mergeScan patch did not take effect: expected 'x', got '${VAL}'"
+            exit 1
+          fi
 
       - name: Install Composer dependencies (root)
         uses: ramsey/composer-install@v3
@@ -169,42 +192,6 @@ jobs:
           ignore-cache: "yes"
           composer-options: "--optimize-autoloader"
 
-      - name: Diagnose testAlterTableAddAndDropColumns
-        run: |
-          php -r '
-            require __DIR__ . "/packages/mysql-on-sqlite/tests/wp-sqlite-schema.php";
-            require __DIR__ . "/packages/mysql-on-sqlite/src/load.php";
-            $GLOBALS["table_prefix"] = "wptests_";
-            $GLOBALS["wpdb"] = new class() { public function set_prefix(string $p): void {} };
-            if (!function_exists("do_action")) { function do_action() {} }
-            if (!function_exists("apply_filters")) { function apply_filters($t, $v, ...$a) { return $v; } }
-            define("FQDB", ":memory:");
-            define("FQDBDIR", __DIR__);
-
-            $pdo_class = PHP_VERSION_ID >= 80400 ? PDO\SQLite::class : PDO::class;
-            $pdo = new $pdo_class("sqlite::memory:");
-            $conn = new WP_SQLite_Connection(["pdo" => $pdo]);
-            $driver = new WP_SQLite_Driver($conn, "wp");
-            // The driver creates its own internal connection; attach the
-            // trace logger to THAT connection so we see every query.
-            $driver->get_connection()->set_query_logger(function (string $sql, array $params) {
-              $short = substr(str_replace("\n", " ", preg_replace("/\s+/", " ", trim($sql))), 0, 250);
-              fwrite(STDOUT, sprintf("[SQL] %s\n", $short));
-            });
-
-            $driver->query("CREATE TABLE t (a INT)");
-            echo "== running ALTER TABLE t ADD b INT, DROP a ==\n";
-            try {
-              $driver->query("ALTER TABLE t ADD b INT, DROP a");
-              echo "ALTER succeeded\n";
-            } catch (Throwable $e) {
-              echo "Exception: " . $e->getMessage() . "\n";
-              if ($e->getPrevious()) echo "Cause: " . $e->getPrevious()->getMessage() . "\n";
-              echo "At: " . $e->getFile() . ":" . $e->getLine() . "\n";
-            }
-          '
-        continue-on-error: true
-
       - name: Run PHPUnit tests against Doltlite
         run: php ./vendor/bin/phpunit -c ./phpunit.xml.dist
         working-directory: packages/mysql-on-sqlite