perf(lava): WAL journaling for the LAVA database

scripts/lavafuncs.py

@@ -36,6 +36,7 @@
 
 from scripts.version_info import rleapp_version
 from scripts.context import Context
+from scripts.storage_safety import configure_lava_journal_mode
 
 # Global variables
 lava_data = None
@@ -118,6 +119,24 @@ def initialize_lava(input_path, output_path, input_type):
 
     db_path = os.path.join(output_path, lava_db_name)
     lava_db = sqlite3.connect(db_path)
+    # Performance: the media insert helpers commit() per row, so a media-heavy
+    # artifact (e.g. a device backup with tens of thousands of files) triggers
+    # tens of thousands of fsync'd commits. Default rollback-journal +
+    # synchronous=FULL makes each commit fsync the whole DB, which can take many
+    # minutes. WAL + synchronous=NORMAL keeps the same durability for a tool run
+    # (only a power-loss in the final checkpoint window could lose the tail) and
+    # is dramatically faster.
+    #
+    # WAL is NOT safe over a network filesystem (https://www.sqlite.org/wal.html),
+    # and examiners commonly write LEAPP output straight to NAS/mapped drives.
+    # configure_lava_journal_mode enables WAL only when output_path is confirmed
+    # local, stays on the network-safe rollback journal otherwise, and verifies
+    # WAL actually took effect before tuning synchronous mode.
+    try:
+        from scripts.ilapfuncs import logfunc as _logfunc
+    except Exception:
+        _logfunc = None
+    configure_lava_journal_mode(lava_db, output_path, logfunc=_logfunc)
 
     cursor = lava_db.cursor()
     cursor.execute('''CREATE TABLE _artifact_search_patterns (

scripts/storage_safety.py

@@ -0,0 +1,216 @@
+"""
+storage_safety.py — Network-aware SQLite journal mode selection.
+
+WAL (Write-Ahead Logging) mode significantly improves SQLite write performance,
+but per the SQLite documentation it is NOT safe over network filesystems:
+
+    https://www.sqlite.org/wal.html
+    "All processes using a database must be on the same host computer; WAL does
+     not work over a network filesystem."
+
+Forensic examiners commonly write LEAPP output directly to network storage
+(NAS/SAN, mapped drives, UNC paths). Enabling WAL on such a path risks database
+corruption or a crashed run — unacceptable when processing evidence.
+
+Strategy (safe by default):
+  1. Only enable WAL when the output path can be AFFIRMATIVELY confirmed local.
+  2. If the path is network-backed, or if we cannot determine its nature with
+     confidence, stay on SQLite's default rollback journal (works everywhere).
+  3. After attempting to set WAL, verify the journal mode actually took effect
+     before tuning synchronous mode (WAL can silently fall back on some
+     filesystems). This is the backstop suggested by @JamesHabben.
+"""
+
+import os
+import sys
+
+
+# Filesystem types that indicate network-backed storage on Unix-like systems.
+_NETWORK_FSTYPES = frozenset({
+    'nfs', 'nfs4', 'cifs', 'smbfs', 'smb3', 'fuse.sshfs', 'afpfs',
+    'ncpfs', 'fuse.s3fs', 'fuse.rclone', '9p', 'fuse.glusterfs',
+    'lustre', 'ceph', 'gfs', 'gfs2', 'ocfs2',
+})
+
+
+# Windows extended-length path prefixes. These are NOT network indicators:
+#   \\?\C:\dir         -> a LOCAL drive-letter path (the \\?\ just lifts MAX_PATH)
+#   \\?\UNC\srv\share  -> a UNC (network) path in extended-length form
+# LEAPP prepends \\?\ to every drive-letter output path (rleapp.py), so the path
+# handed to us is routinely extended-length and must be normalized before the
+# leading-\\ UNC test, or a local \\?\X:\... is misread as a network share.
+_EXT_PREFIX = '\\\\?\\'          # \\?\
+_EXT_UNC_PREFIX = '\\\\?\\UNC\\'  # \\?\UNC\
+
+
+def _strip_extended_prefix(path):
+    r"""Normalize a Windows extended-length path to its plain form.
+
+    \\?\UNC\server\share -> \\server\share   (still UNC/network)
+    \\?\C:\dir           -> C:\dir            (plain drive-letter)
+    Anything else is returned unchanged.
+    """
+    p = str(path)
+    if p.startswith(_EXT_UNC_PREFIX):
+        return '\\\\' + p[len(_EXT_UNC_PREFIX):]
+    if p.startswith(_EXT_PREFIX):
+        return p[len(_EXT_PREFIX):]
+    return p
+
+
+def _is_unc_path(path):
+    r"""True if path is a UNC network path (\\server\share or //server/share).
+
+    The Windows extended-length prefix is stripped first so that a local
+    \\?\<drive>: path is not mistaken for UNC, while \\?\UNC\... is preserved
+    as UNC.
+    """
+    p = _strip_extended_prefix(path)
+    return p.startswith('\\\\') or p.startswith('//')
+
+
+def _windows_path_is_network(path):
+    """
+    On Windows, determine whether path resides on a network drive.
+
+    Returns True (network), False (local), or None (undetermined).
+    Uses the Win32 API GetDriveType against the path's drive root.
+    UNC paths are treated as network without an API call.
+    """
+    path = _strip_extended_prefix(path)
+    if _is_unc_path(path):
+        return True
+
+    try:
+        import ctypes
+
+        abs_path = os.path.abspath(path)
+        drive, _ = os.path.splitdrive(abs_path)
+        if not drive:
+            return None  # no drive letter and not UNC — undetermined
+        drive_root = drive + '\\'
+
+        # GetDriveTypeW returns:
+        #   0 DRIVE_UNKNOWN, 1 DRIVE_NO_ROOT_DIR, 2 DRIVE_REMOVABLE,
+        #   3 DRIVE_FIXED, 4 DRIVE_REMOTE, 5 DRIVE_CDROM, 6 DRIVE_RAMDISK
+        DRIVE_REMOTE = 4
+        drive_type = ctypes.windll.kernel32.GetDriveTypeW(ctypes.c_wchar_p(drive_root))
+        if drive_type == DRIVE_REMOTE:
+            return True
+        if drive_type in (3, 2, 5, 6):  # FIXED, REMOVABLE, CDROM, RAMDISK = local
+            return False
+        return None  # UNKNOWN / NO_ROOT_DIR — undetermined
+    except Exception:
+        return None  # API unavailable or error — undetermined
+
+
+def _unix_path_is_network(path, mounts_text=None):
+    """
+    On Unix-like systems, determine whether path resides on a network mount by
+    finding the longest matching mount point in /proc/mounts and checking its
+    filesystem type.
+
+    Returns True (network), False (local), or None (undetermined).
+    mounts_text is injectable for testing.
+    """
+    try:
+        if mounts_text is None:
+            # /proc/mounts is Linux. macOS/BSD lack it; fall back to undetermined.
+            if not os.path.exists('/proc/mounts'):
+                return None
+            with open('/proc/mounts', 'r', encoding='utf-8', errors='replace') as f:
+                mounts_text = f.read()
+
+        abs_path = os.path.abspath(path)
+        best_mount = ''
+        best_fstype = ''
+        for line in mounts_text.strip().splitlines():
+            parts = line.split()
+            if len(parts) < 3:
+                continue
+            mount_point, fstype = parts[1], parts[2]
+            if abs_path == mount_point or abs_path.startswith(mount_point.rstrip('/') + '/'):
+                if len(mount_point) >= len(best_mount):
+                    best_mount = mount_point
+                    best_fstype = fstype
+
+        if not best_fstype:
+            return None
+        return best_fstype.lower() in _NETWORK_FSTYPES
+    except Exception:
+        return None
+
+
+def path_is_network(path):
+    """
+    Best-effort determination of whether path is on network-backed storage.
+
+    Returns:
+        True  — confirmed network storage
+        False — confirmed local storage
+        None  — could not determine (caller should treat as unsafe for WAL)
+    """
+    # Normalize away the Windows extended-length prefix first so a local
+    # \\?\<drive>: path is not misread as UNC.
+    path = _strip_extended_prefix(path)
+
+    # UNC is network on any platform that understands it
+    if _is_unc_path(path):
+        return True
+
+    if sys.platform == 'win32':
+        return _windows_path_is_network(path)
+    else:
+        return _unix_path_is_network(path)
+
+
+def configure_lava_journal_mode(lava_db, output_path, logfunc=None):
+    """
+    Configure the SQLite journal mode for the LAVA database safely.
+
+    Enables WAL only when output_path is AFFIRMATIVELY local. On network or
+    undetermined paths, leaves SQLite on its default rollback journal, which is
+    safe over network filesystems.
+
+    After attempting WAL, verifies the mode actually took effect before setting
+    synchronous=NORMAL (WAL can silently fall back on some filesystems).
+
+    Args:
+        lava_db:     an open sqlite3 Connection (or cursor-capable object).
+        output_path: the path where the LAVA database is being written.
+        logfunc:     optional logging callable (e.g. ilapfuncs.logfunc).
+
+    Returns:
+        The effective journal mode string (e.g. 'wal', 'delete').
+    """
+    def _log(msg):
+        if logfunc:
+            logfunc(msg)
+
+    network = path_is_network(output_path)
+
+    if network is True:
+        _log('LAVA DB: output path is network-backed storage; '
+             'keeping default journal mode (WAL unsafe over network per SQLite docs).')
+        # Ensure a safe, network-compatible journal mode explicitly.
+        effective = lava_db.execute("PRAGMA journal_mode=DELETE").fetchone()[0]
+        return effective.lower()
+
+    if network is None:
+        _log('LAVA DB: could not determine if output path is local or network; '
+             'defaulting to safe journal mode (no WAL).')
+        effective = lava_db.execute("PRAGMA journal_mode=DELETE").fetchone()[0]
+        return effective.lower()
+
+    # network is False — confirmed local storage; safe to attempt WAL.
+    effective = lava_db.execute("PRAGMA journal_mode=WAL").fetchone()[0]
+    if effective.lower() == 'wal':
+        # WAL took effect — NORMAL sync is safe and faster under WAL.
+        lava_db.execute("PRAGMA synchronous=NORMAL")
+        _log('LAVA DB: local storage confirmed; WAL journaling enabled '
+             '(synchronous=NORMAL).')
+    else:
+        # WAL did not take effect (filesystem refused it). Leave sync at default.
+        _log(f'LAVA DB: WAL requested but filesystem returned journal_mode='
+             f'{effective!r}; leaving synchronous at default for safety.')
+    return effective.lower()