refactor: extraction, tooling, and stability

.gitignore

@@ -43,5 +43,5 @@ dmypy.json
 
 # Logs
 *.log
-!examples/**/claude.log
+!examples/**/*.log
 SPEC/

Dockerfile

@@ -72,13 +72,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 
 RUN curl -fsSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh \
     && chmod +x /tmp/dotnet-install.sh \
-    && /tmp/dotnet-install.sh --channel 8.0 --runtime dotnet \
+    && /tmp/dotnet-install.sh --channel 9.0 --runtime dotnet \
         --install-dir /opt/dotnet \
     && rm /tmp/dotnet-install.sh
 
 RUN mkdir -p /opt/zimmermantools && cd /opt/zimmermantools \
     && for tool in AmcacheParser AppCompatCacheParser EvtxECmd JLECmd LECmd MFTECmd PECmd RBCmd RECmd SBECmd SrumECmd; do \
-        wget -q "https://download.ericzimmermanstools.com/${tool}.zip" \
+        wget -q "https://download.ericzimmermanstools.com/net9/${tool}.zip" \
             -O "/tmp/${tool}.zip" \
         && unzip -qo "/tmp/${tool}.zip" -d /opt/zimmermantools \
         && rm "/tmp/${tool}.zip"; \
@@ -110,31 +110,40 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 RUN git clone --depth 1 https://github.com/Neo23x0/signature-base.git \
         /opt/signature-base
 
-RUN git clone --depth 1 https://github.com/Yara-Rules/rules.git \
-        /opt/yara-rules
-
-# Hayabusa: Sigma rule engine for Windows EVTX logs
+# Hayabusa: Sigma rule engine for Windows EVTX logs.
+# amd64: pre-built musl binary (statically linked, no GLIBC dependency).
+# arm64: build from source because the pre-built aarch64-gnu binary requires
+# GLIBC >= 2.38 while Ubuntu 22.04 ships GLIBC 2.35.
 FROM ubuntu:22.04 AS hayabusa-fetch
 
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-        ca-certificates curl unzip \
+        ca-certificates curl unzip git build-essential \
     && rm -rf /var/lib/apt/lists/*
 
 ARG TARGETARCH
 ARG HAYABUSA_VERSION=3.8.1
-RUN case "${TARGETARCH}" in \
-        arm64) HAYABUSA_VARIANT="aarch64-gnu" ;; \
-        *)     HAYABUSA_VARIANT="x64-musl" ;; \
-    esac \
-    && curl -fsSL "https://github.com/Yamato-Security/hayabusa/releases/download/v${HAYABUSA_VERSION}/hayabusa-${HAYABUSA_VERSION}-lin-${HAYABUSA_VARIANT}.zip" \
-        -o /tmp/hayabusa.zip \
-    && mkdir -p /opt/hayabusa \
-    && unzip /tmp/hayabusa.zip -d /opt/hayabusa \
-    && chmod +x /opt/hayabusa/hayabusa-${HAYABUSA_VERSION}-lin-${HAYABUSA_VARIANT} \
-    && ln -s /opt/hayabusa/hayabusa-${HAYABUSA_VERSION}-lin-${HAYABUSA_VARIANT} /opt/hayabusa/hayabusa \
-    && rm /tmp/hayabusa.zip
+RUN mkdir -p /opt/hayabusa \
+    && if [ "$TARGETARCH" != "arm64" ]; then \
+        curl -fsSL "https://github.com/Yamato-Security/hayabusa/releases/download/v${HAYABUSA_VERSION}/hayabusa-${HAYABUSA_VERSION}-lin-x64-musl.zip" \
+            -o /tmp/hayabusa.zip \
+        && unzip /tmp/hayabusa.zip -d /opt/hayabusa \
+        && chmod +x /opt/hayabusa/hayabusa-${HAYABUSA_VERSION}-lin-x64-musl \
+        && ln -s /opt/hayabusa/hayabusa-${HAYABUSA_VERSION}-lin-x64-musl /opt/hayabusa/hayabusa \
+        && rm /tmp/hayabusa.zip; \
+    else \
+        curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
+            | sh -s -- -y --default-toolchain stable --profile minimal \
+        && export PATH="/root/.cargo/bin:${PATH}" \
+        && git clone --depth 1 --branch v${HAYABUSA_VERSION} \
+            https://github.com/Yamato-Security/hayabusa.git /tmp/hayabusa-src \
+        && cd /tmp/hayabusa-src \
+        && cargo build --release \
+        && cp target/release/hayabusa /opt/hayabusa/hayabusa \
+        && cp -r config rules /opt/hayabusa/ \
+        && rm -rf /tmp/hayabusa-src /root/.cargo /root/.rustup; \
+    fi
 
 # radare2: fetch release .deb from GitHub (not in Ubuntu 22.04 repos)
 FROM ubuntu:22.04 AS radare2-fetch
@@ -187,6 +196,125 @@ RUN git clone --depth 1 https://github.com/redNixon/stegdetect.git /tmp/stegdete
     && mkdir -p /opt/stegdetect/bin /opt/stegdetect/share /opt/stegdetect/man/man1 \
     && make install
 
+# CAPA: download pre-built binary (multi-arch)
+FROM ubuntu:22.04 AS capa-fetch
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        ca-certificates curl unzip \
+    && rm -rf /var/lib/apt/lists/*
+
+ARG TARGETARCH
+ARG CAPA_VERSION=9.4.0
+RUN case "${TARGETARCH}" in \
+        arm64) CAPA_ARCH="linux-arm64" ;; \
+        *)     CAPA_ARCH="linux" ;; \
+    esac \
+    && curl -fsSL "https://github.com/mandiant/capa/releases/download/v${CAPA_VERSION}/capa-v${CAPA_VERSION}-${CAPA_ARCH}.zip" \
+        -o /tmp/capa.zip \
+    && unzip /tmp/capa.zip -d /opt/capa \
+    && chmod +x /opt/capa/capa \
+    && rm /tmp/capa.zip
+
+# FLOSS: download pre-built binary (amd64 only; arm64 uses pip in runtime)
+FROM ubuntu:22.04 AS floss-fetch
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        ca-certificates curl unzip \
+    && rm -rf /var/lib/apt/lists/*
+
+ARG TARGETARCH
+ARG FLOSS_VERSION=3.1.0
+RUN mkdir -p /opt/floss \
+    && if [ "$TARGETARCH" = "amd64" ]; then \
+        curl -fsSL "https://github.com/mandiant/flare-floss/releases/download/v${FLOSS_VERSION}/floss-v${FLOSS_VERSION}-linux.zip" \
+            -o /tmp/floss.zip \
+        && unzip /tmp/floss.zip -d /opt/floss \
+        && chmod +x /opt/floss/floss \
+        && rm /tmp/floss.zip; \
+    fi
+
+# Detect-It-Easy: fetch .deb for amd64 only (arm64 source build is fragile)
+FROM ubuntu:22.04 AS die-fetch
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        ca-certificates curl \
+    && rm -rf /var/lib/apt/lists/*
+
+ARG TARGETARCH
+ARG DIE_VERSION=3.09
+RUN mkdir -p /opt/die \
+    && if [ "$TARGETARCH" = "amd64" ]; then \
+        curl -fsSL "https://github.com/horsicq/DIE-engine/releases/download/${DIE_VERSION}/die_${DIE_VERSION}_Ubuntu_22.04_amd64.deb" \
+            -o /opt/die/die.deb; \
+    fi
+
+# Chainsaw: download pre-built binary (multi-arch), Sigma rules, and mappings
+FROM ubuntu:22.04 AS chainsaw-fetch
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        ca-certificates curl git \
+    && rm -rf /var/lib/apt/lists/*
+
+ARG TARGETARCH
+ARG CHAINSAW_VERSION=2.16.0
+RUN case "${TARGETARCH}" in \
+        arm64) CHAINSAW_ARCH="aarch64-unknown-linux-gnu" ;; \
+        *)     CHAINSAW_ARCH="x86_64-unknown-linux-gnu" ;; \
+    esac \
+    && curl -fsSL "https://github.com/WithSecureLabs/chainsaw/releases/download/v${CHAINSAW_VERSION}/chainsaw_${CHAINSAW_ARCH}.tar.gz" \
+        -o /tmp/chainsaw.tar.gz \
+    && mkdir -p /opt/chainsaw \
+    && tar xzf /tmp/chainsaw.tar.gz -C /opt/chainsaw --strip-components=1 \
+    && chmod +x /opt/chainsaw/chainsaw \
+    && rm /tmp/chainsaw.tar.gz
+
+RUN git clone --depth 1 --branch v${CHAINSAW_VERSION} \
+        https://github.com/WithSecureLabs/chainsaw.git /tmp/chainsaw-repo \
+    && cp -r /tmp/chainsaw-repo/mappings /opt/chainsaw/mappings \
+    && cp -r /tmp/chainsaw-repo/rules /opt/chainsaw/rules \
+    && rm -rf /tmp/chainsaw-repo
+
+ARG SIGMA_VERSION=r2024-09-02
+RUN curl -fsSL "https://github.com/SigmaHQ/sigma/archive/refs/tags/${SIGMA_VERSION}.tar.gz" \
+        -o /tmp/sigma.tar.gz \
+    && mkdir -p /opt/sigma-rules \
+    && tar xzf /tmp/sigma.tar.gz -C /opt/sigma-rules --strip-components=1 \
+    && rm /tmp/sigma.tar.gz
+
+# Suricata: build from source for arm64 (amd64 uses PPA in runtime)
+FROM ubuntu:22.04 AS suricata-builder
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+ARG TARGETARCH
+ARG SURICATA_VERSION=7.0.7
+RUN mkdir -p /opt/suricata /etc/suricata \
+    && if [ "$TARGETARCH" = "arm64" ]; then \
+        apt-get update && apt-get install -y --no-install-recommends \
+            build-essential ca-certificates curl pkg-config \
+            libpcre2-dev libpcap-dev libyaml-dev libjansson-dev \
+            libcap-ng-dev libmagic-dev zlib1g-dev liblz4-dev \
+            rustc cargo python3-yaml \
+        && rm -rf /var/lib/apt/lists/* \
+        && curl -fsSL "https://www.openinfosecfoundation.org/download/suricata-${SURICATA_VERSION}.tar.gz" \
+            -o /tmp/suricata.tar.gz \
+        && tar xzf /tmp/suricata.tar.gz -C /tmp \
+        && cd /tmp/suricata-${SURICATA_VERSION} \
+        && ./configure --prefix=/opt/suricata --sysconfdir=/etc \
+            --localstatedir=/var --enable-non-root --disable-geoip \
+        && make -j"$(nproc)" \
+        && make install \
+        && rm -rf /tmp/suricata*; \
+    fi
+
 # Runtime image
 FROM ubuntu:22.04 AS runtime
 
@@ -195,6 +323,7 @@ ENV DEBIAN_FRONTEND=noninteractive \
     PYTHONUNBUFFERED=1 \
     DISABLE_AUTOUPDATE=1 \
     DOTNET_ROOT=/usr/local/share/dotnet \
+    DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=true \
     PATH="/usr/local/share/dotnet:${PATH}"
 
 
@@ -232,6 +361,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         dc3dd \
         libguestfs-tools \
         pasco \
+        pst-utils \
         tshark \
         tcpdump \
         ssdeep \
@@ -266,22 +396,82 @@ COPY --from=symbols-fetch /opt/vol-symbols/windows.zip /home/mulder/.cache/volat
 COPY --from=symbols-fetch /opt/vol-symbols/linux.zip /home/mulder/.cache/volatility3/symbols/linux.zip
 COPY --from=stegdetect-builder /opt/stegdetect/bin/stegdetect /usr/local/bin/stegdetect
 COPY --from=stegdetect-builder /opt/stegdetect/bin/stegbreak /usr/local/bin/stegbreak
-COPY --from=yara-fetch /opt/yara-rules /opt/yara-rules
 COPY --from=yara-fetch /opt/signature-base /opt/signature-base
 COPY --from=attack-fetch /opt/attack/enterprise-attack.json /opt/attack/enterprise-attack.json
 COPY --from=attack-fetch /opt/attack/ics-attack.json /opt/attack/ics-attack.json
 COPY --from=hayabusa-fetch /opt/hayabusa /opt/hayabusa
 COPY --from=radare2-fetch /tmp/radare2.deb /tmp/radare2.deb
+COPY --from=capa-fetch /opt/capa/capa /usr/local/bin/capa
+COPY --from=floss-fetch /opt/floss/ /opt/floss/
+COPY --from=die-fetch /opt/die/ /opt/die/
+COPY --from=chainsaw-fetch /opt/chainsaw/ /opt/chainsaw/
+RUN ln -sf /opt/chainsaw/chainsaw /usr/local/bin/chainsaw
+COPY --from=chainsaw-fetch /opt/sigma-rules/ /opt/sigma-rules/
+COPY --from=suricata-builder /opt/suricata/ /opt/suricata/
+COPY --from=suricata-builder /etc/suricata/ /etc/suricata/
+
 RUN dpkg -i /tmp/radare2.deb || apt-get install -yf --no-install-recommends \
     && rm /tmp/radare2.deb
-ENV PATH="/opt/hayabusa:${PATH}"
+
+ARG TARGETARCH
+
+# Detect-It-Easy: install .deb on amd64 (skipped on arm64)
+RUN if [ -f /opt/die/die.deb ]; then \
+        dpkg -i /opt/die/die.deb || apt-get install -yf --no-install-recommends \
+        && rm /opt/die/die.deb; \
+    fi
+
+# FLOSS: symlink pre-built binary on amd64 (arm64 uses pip entry point)
+RUN if [ -x /opt/floss/floss ]; then ln -sf /opt/floss/floss /usr/local/bin/floss; fi
+
+# Suricata: PPA on amd64, link source build on arm64
+# python3 now points to 3.12 (deadsnakes), but apt_pkg is compiled for
+# the system 3.10, so invoke add-apt-repository via python3.10 explicitly.
+RUN if [ "$TARGETARCH" = "amd64" ]; then \
+        /usr/bin/python3.10 /usr/bin/add-apt-repository -y ppa:oisf/suricata-stable \
+        && apt-get update \
+        && apt-get install -y --no-install-recommends suricata \
+        && apt-get clean && rm -rf /var/lib/apt/lists/*; \
+    elif [ -x /opt/suricata/bin/suricata ]; then \
+        apt-get update \
+        && apt-get install -y --no-install-recommends \
+            libpcre3 libpcap0.8 libyaml-0-2 libjansson4 \
+            libcap-ng0 libmagic1 liblz4-1 \
+        && apt-get clean && rm -rf /var/lib/apt/lists/* \
+        && ln -sf /opt/suricata/bin/suricata /usr/bin/suricata; \
+    fi
+
+# Download Emerging Threats Open ruleset for Suricata
+RUN mkdir -p /etc/suricata/rules \
+    && curl -fsSL "https://rules.emergingthreats.net/open/suricata-7.0/emerging.rules.tar.gz" \
+        -o /tmp/et-rules.tar.gz \
+    && tar xzf /tmp/et-rules.tar.gz -C /etc/suricata/rules --strip-components=1 \
+    && rm /tmp/et-rules.tar.gz
+
+# Zeek: install from OBS repository (provides amd64 and arm64 packages).
+# Mark as manual so later apt-get autoremove steps do not uninstall it.
+RUN echo "deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_22.04/ /" \
+        > /etc/apt/sources.list.d/zeek.list \
+    && curl -fsSL "https://download.opensuse.org/repositories/security:/zeek/xUbuntu_22.04/Release.key" \
+        | gpg --dearmor -o /etc/apt/trusted.gpg.d/zeek.gpg \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends zeek \
+    && apt-mark manual zeek-core zeekctl \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV PATH="/opt/zeek/bin:/opt/chainsaw:/opt/hayabusa:${PATH}"
 RUN ldconfig || true
 
 # Install Python forensic packages that require C compilation, then purge build deps
 RUN apt-get update && apt-get install -y --no-install-recommends \
         gcc g++ make pkg-config python3.12-dev \
         zlib1g-dev libbz2-dev libssl-dev libsqlcipher-dev \
-    && uv pip install --system --no-cache volatility3 plaso mvt pysqlcipher3 pyhindsight \
+    && uv pip install --system --no-cache \
+        volatility3 plaso mvt pysqlcipher3 pyhindsight oletools \
+    && if [ "$TARGETARCH" = "arm64" ]; then \
+        uv pip install --system --no-cache flare-floss; \
+    fi \
     && apt-get purge -y \
         gcc g++ make pkg-config python3.12-dev \
         zlib1g-dev libbz2-dev libssl-dev libsqlcipher-dev \
@@ -290,6 +480,37 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
+# Install pure Python forensic tool dependencies
+RUN uv pip install --system --no-cache \
+        stix2 evtx jsonpath-ng colorama tqdm aiohttp lark
+
+# Clone forensic tool repositories
+RUN git clone --depth 1 --branch 2.20.0 \
+        https://github.com/wagga40/Zircolite.git /opt/zircolite \
+    && chmod +x /opt/zircolite/zircolite.py \
+    && git clone --depth 1 https://github.com/abrignoni/ALEAPP.git /opt/aleapp \
+    && git clone --depth 1 https://github.com/abrignoni/iLEAPP.git /opt/ileapp \
+    && git clone --depth 1 \
+        https://github.com/DidierStevens/DidierStevensSuite.git \
+        /opt/didier-stevens \
+    && chmod +x /opt/didier-stevens/pdfid.py /opt/didier-stevens/pdf-parser.py
+
+# Install ALEAPP and iLEAPP Python dependencies; strip version pins and
+# filter non-PyPI entries to avoid conflicts between the two projects
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        gcc g++ python3.12-dev \
+    && cat /opt/aleapp/requirements.txt /opt/ileapp/requirements.txt \
+        | sed 's/[;].*//; s/==.*//; s/>=.*//; s/<=.*//; s/~=.*//' \
+        | grep -v '^\s*#' | grep -v '^\s*$' | grep -v '/' | sort -u \
+        | xargs uv pip install --system --no-cache \
+    && apt-get purge -y gcc g++ python3.12-dev \
+    && apt-get autoremove -y \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Copy Linux Sigma rules for Zircolite
+RUN mkdir -p /opt/zircolite/rules/linux \
+    && cp -r /opt/sigma-rules/rules/linux/* /opt/zircolite/rules/linux/ 2>/dev/null || true
+
 RUN python3 -c "import pyewf; print('libewf-python', pyewf.get_version())"
 
 RUN useradd -m -s /bin/bash mulder