vfs: fix Windows WAL corruption by mapping the wal-index into wasm memory

.github/workflows/wal-repro.yml

@@ -0,0 +1,27 @@
+name: WAL Repro
+
+# On-demand regression run for the Windows WAL-index corruption. The test is
+# opt-in (gated behind SQLITE3_TEST_WAL_STRESS) so it never taxes the normal
+# test matrix. On an unfixed Windows build it fails in the first round; on the
+# fixed build, and on every direct-mapped VFS (unix, native), it stays green.
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  wal:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [windows-latest, ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    env:
+      SQLITE3_TEST_WAL_STRESS: "1"
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-go@v6
+        with: { go-version: stable }
+      - name: WAL concurrent-writers regression
+        run: go test ./vfs/tests/ -run '^TestWALConcurrentWriters$' -v -count=1 -timeout 25m

internal/sqlite3_wrap/mem_windows.go

@@ -2,16 +2,19 @@ package sqlite3_wrap
 
 import (
 	"math"
+	"os"
 	"unsafe"
 
 	"golang.org/x/sys/windows"
 )
 
 type Memory struct {
-	Buf []byte
-	Max int64
-	com int
-	ptr uintptr
+	Buf    []byte
+	Max    int64
+	com    int
+	ptr    uintptr
+	placeh bool      // reserved with placeholders (Win10 1803+)
+	views  []uintptr // offsets of live file views, for Close
 }
 
 func (m *Memory) Slice() *[]byte {
@@ -44,18 +47,30 @@ func (m *Memory) allocate(max uint64) {
 
 	if res > math.MaxInt {
 		// This ensures uintptr(res) overflows to a large value,
-		// and windows.VirtualAlloc returns an error.
+		// and the reservation fails.
 		res = math.MaxUint64
 	}
 
 	// Reserve res bytes of address space, to ensure we won't need to move it.
-	r, err := windows.VirtualAlloc(0, uintptr(res), windows.MEM_RESERVE, windows.PAGE_READWRITE)
-	if err != nil {
-		panic(err)
+	// Prefer a placeholder reservation (Windows 10 1803+): it can later have
+	// file views mapped into it, which the WAL-index shared memory uses.
+	m.placeh = PlaceholdersSupported()
+	if m.placeh {
+		r, err := virtualAlloc2(0, uintptr(res),
+			_MEM_RESERVE|_MEM_RESERVE_PLACEHOLDERS, _PAGE_NOACCESS)
+		if err != nil {
+			panic(err)
+		}
+		m.ptr = r
+	} else {
+		r, err := windows.VirtualAlloc(0, uintptr(res), windows.MEM_RESERVE, windows.PAGE_READWRITE)
+		if err != nil {
+			panic(err)
+		}
+		m.ptr = r
 	}
-	m.ptr = r
 
-	ptr := *(*unsafe.Pointer)(unsafe.Pointer(&r))
+	ptr := *(*unsafe.Pointer)(unsafe.Pointer(&m.ptr))
 	m.Buf = unsafe.Slice((*byte)(ptr), res)[:0]
 }
 
@@ -69,20 +84,144 @@ func (m *Memory) reallocate(size uint64) {
 		new = min(max(size, new), res)
 		new = (new + rnd) &^ rnd
 
-		// Commit additional memory up to new bytes.
-		_, err := windows.VirtualAlloc(m.ptr, uintptr(new), windows.MEM_COMMIT, windows.PAGE_READWRITE)
-		if err != nil {
-			panic(err)
+		if m.placeh {
+			// Round to the 64K allocation granularity so every committed
+			// chunk boundary is 64K-aligned. A wal-index view (64K-aligned,
+			// 64K) can then never straddle two allocations, which a
+			// placeholder split cannot cross.
+			const gran = 64 * 1024
+			new = (new + gran - 1) &^ (gran - 1)
+			new = min(new, res)
+			// Split the trailing placeholder to [com, new) and replace it
+			// with committed memory. Placeholder-lineage allocations can be
+			// carved back into placeholders later, which file-view mapping
+			// relies on.
+			if new < res {
+				if err := splitPlaceholder(m.ptr+uintptr(com), uintptr(new-com)); err != nil {
+					panic(err)
+				}
+			}
+			if _, err := virtualAlloc2(m.ptr+uintptr(com), uintptr(new-com),
+				_MEM_RESERVE|_MEM_COMMIT|_MEM_REPLACE_PLACEHOLDER, _PAGE_READWRITE); err != nil {
+				panic(err)
+			}
+		} else {
+			// Commit additional memory up to new bytes.
+			if _, err := windows.VirtualAlloc(m.ptr, uintptr(new), windows.MEM_COMMIT, windows.PAGE_READWRITE); err != nil {
+				panic(err)
+			}
 		}
 		m.com = int(new)
 	}
 	m.Buf = m.Buf[:size]
 }
 
+// CanMapFiles reports whether file views can be mapped into this memory.
+func (m *Memory) CanMapFiles() bool { return m.placeh }
+
+// MapFileRegion maps size bytes of f at offset fileOff into the linear
+// memory at offset addrOff, replacing previously committed private memory.
+// addrOff must be allocation-granularity aligned and [addrOff, addrOff+size)
+// must lie within a single committed chunk (e.g. one sqlite3_malloc block
+// well inside the heap). The caller owns making the range unused by Go/wasm
+// code for the lifetime of the view.
+func (m *Memory) MapFileRegion(f *os.File, fileOff int64, addrOff, size uintptr) error {
+	addr := m.ptr + addrOff
+	if err := splitPlaceholder(addr, size); err != nil {
+		return err
+	}
+	maxSize := uint64(fileOff) + uint64(size)
+	h, err := windows.CreateFileMapping(windows.Handle(f.Fd()), nil,
+		windows.PAGE_READWRITE, uint32(maxSize>>32), uint32(maxSize), nil)
+	if h == 0 {
+		m.restoreCommitted(addr, size)
+		return err
+	}
+	if _, err := mapViewOfFile3(h, addr, uint64(fileOff), size,
+		_MEM_REPLACE_PLACEHOLDER, _PAGE_READWRITE); err != nil {
+		windows.CloseHandle(h)
+		m.restoreCommitted(addr, size)
+		return err
+	}
+	// The view keeps the section alive; the handle is no longer needed.
+	windows.CloseHandle(h)
+	m.views = append(m.views, addrOff)
+	return nil
+}
+
+// restoreCommitted turns a placeholder back into committed private memory.
+// It must not fail: a range of the linear memory would otherwise be left
+// PAGE_NOACCESS, and whatever wasm code owns it would fault on next touch.
+// Callers rely on the invariant that MapFileRegion/UnmapFileRegion return
+// with the address space intact.
+func (m *Memory) restoreCommitted(addr, size uintptr) {
+	if _, err := virtualAlloc2(addr, size,
+		_MEM_RESERVE|_MEM_COMMIT|_MEM_REPLACE_PLACEHOLDER, _PAGE_READWRITE); err != nil {
+		panic(err)
+	}
+}
+
+// UnmapFileRegion releases a view created by MapFileRegion and restores
+// committed private (zeroed) memory in its place.
+func (m *Memory) UnmapFileRegion(addrOff, size uintptr) error {
+	addr := m.ptr + addrOff
+	if err := unmapViewOfFile2(addr, _MEM_PRESERVE_PLACEHOLDER); err != nil {
+		// The view is still mapped; the address space is intact.
+		return err
+	}
+	// The view is gone: record that before anything else,
+	// so Close never unmaps this offset a second time.
+	for i, off := range m.views {
+		if off == addrOff {
+			m.views = append(m.views[:i], m.views[i+1:]...)
+			break
+		}
+	}
+	m.restoreCommitted(addr, size)
+	return nil
+}
+
 func (m *Memory) Close() error {
-	err := windows.VirtualFree(m.ptr, 0, windows.MEM_RELEASE)
+	if m.ptr == 0 {
+		m.Buf = nil
+		return nil
+	}
+	var err error
+	if m.placeh {
+		// Unmap views first, then release every allocation and the
+		// remaining placeholders piecewise.
+		for _, off := range m.views {
+			unmapViewOfFile2(m.ptr+off, 0)
+		}
+		// Walk the region, releasing each allocation.
+		addr := m.ptr
+		end := m.ptr + uintptr(cap(m.Buf))
+		for addr < end {
+			var info windows.MemoryBasicInformation
+			if e := windows.VirtualQuery(addr, &info, unsafe.Sizeof(info)); e != nil {
+				// Cannot advance without RegionSize; the rest leaks.
+				if err == nil {
+					err = e
+				}
+				break
+			}
+			if info.State != _MEM_FREE {
+				if e := windows.VirtualFree(info.AllocationBase, 0, windows.MEM_RELEASE); e != nil && err == nil {
+					err = e
+				}
+			}
+			next := info.BaseAddress + info.RegionSize
+			if next <= addr {
+				break
+			}
+			addr = next
+		}
+	} else {
+		err = windows.VirtualFree(m.ptr, 0, windows.MEM_RELEASE)
+	}
 	m.Buf = nil
 	m.com = 0
 	m.ptr = 0
+	m.views = nil
 	return err
 }

internal/sqlite3_wrap/placeholder_windows.go

@@ -0,0 +1,93 @@
+package sqlite3_wrap
+
+// Address-space placeholders (Windows 10 1803+ / Server 2019+) let a file
+// view be mapped INTO the wasm linear memory, the same way the unix build
+// maps the WAL-index with MAP_FIXED. SQLite then works on genuinely shared
+// memory: no private copies, no sync points, native memory semantics.
+//
+// The round-trip used here:
+//
+//	reserve:  VirtualAlloc2(MEM_RESERVE|MEM_RESERVE_PLACEHOLDERS)
+//	commit:   split placeholder, VirtualAlloc2(MEM_REPLACE_PLACEHOLDER|COMMIT)
+//	carve:    split a committed (replaced-placeholder) range back into a
+//	          placeholder with VirtualFree(MEM_RELEASE|MEM_PRESERVE_PLACEHOLDER)
+//	map:      MapViewOfFile3(MEM_REPLACE_PLACEHOLDER) into the carved hole
+//	unmap:    UnmapViewOfFile2(MEM_PRESERVE_PLACEHOLDER), then re-commit
+
+import (
+	"golang.org/x/sys/windows"
+)
+
+const (
+	_MEM_COMMIT               = 0x00001000
+	_MEM_RESERVE              = 0x00002000
+	_MEM_RELEASE              = 0x00008000
+	_MEM_FREE                 = 0x00010000
+	_MEM_RESERVE_PLACEHOLDERS = 0x00040000
+	_MEM_REPLACE_PLACEHOLDER  = 0x00004000
+	_MEM_PRESERVE_PLACEHOLDER = 0x00000002
+	_PAGE_READWRITE           = 0x04
+	_PAGE_NOACCESS            = 0x01
+)
+
+var (
+	kernelbase           = windows.NewLazySystemDLL("kernelbase.dll")
+	procVirtualAlloc2    = kernelbase.NewProc("VirtualAlloc2")
+	procMapViewOfFile3   = kernelbase.NewProc("MapViewOfFile3")
+	procUnmapViewOfFile2 = kernelbase.NewProc("UnmapViewOfFile2")
+)
+
+// PlaceholdersSupported reports whether this Windows version has the
+// placeholder APIs (Windows 10 1803+ / Server 2019+).
+func PlaceholdersSupported() bool {
+	return procVirtualAlloc2.Find() == nil &&
+		procMapViewOfFile3.Find() == nil &&
+		procUnmapViewOfFile2.Find() == nil
+}
+
+func virtualAlloc2(addr uintptr, size uintptr, allocType, protect uint32) (uintptr, error) {
+	r, _, err := procVirtualAlloc2.Call(
+		0, // current process
+		addr,
+		size,
+		uintptr(allocType),
+		uintptr(protect),
+		0, 0) // no extended parameters
+	if r == 0 {
+		return 0, err
+	}
+	return r, nil
+}
+
+func mapViewOfFile3(h windows.Handle, addr uintptr, offset uint64, size uintptr, allocType, protect uint32) (uintptr, error) {
+	r, _, err := procMapViewOfFile3.Call(
+		uintptr(h),
+		0, // current process
+		addr,
+		uintptr(offset),
+		size,
+		uintptr(allocType),
+		uintptr(protect),
+		0, 0)
+	if r == 0 {
+		return 0, err
+	}
+	return r, nil
+}
+
+func unmapViewOfFile2(addr uintptr, unmapFlags uint32) error {
+	r, _, err := procUnmapViewOfFile2.Call(
+		^uintptr(0), // current process pseudo handle
+		addr,
+		uintptr(unmapFlags))
+	if r == 0 {
+		return err
+	}
+	return nil
+}
+
+// splitPlaceholder shrinks the placeholder/allocation containing
+// [addr, addr+size) to exactly that range, so it can be replaced.
+func splitPlaceholder(addr, size uintptr) error {
+	return windows.VirtualFree(addr, size, _MEM_RELEASE|_MEM_PRESERVE_PLACEHOLDER)
+}