Automate corpora testing in CI

.github/workflows/detector-corpora-test.yml

@@ -0,0 +1,93 @@
+name: Corpora Test
+
+on:
+  workflow_dispatch:
+  pull_request:
+    # types: [opened, reopened] TODO: Decide if we should run this on every push
+    paths:
+      - 'pkg/detectors/**'
+      - '.github/workflows/detector-corpora-test.yml'
+      - 'scripts/detector_corpora_test.sh'
+
+env:
+  DATASETS: |
+    s3://trufflehog-corpora-datasets/contents.2025-11-04.jsonl.zstd
+
+jobs:
+  corpora-test:
+    if: ${{ github.repository == 'trufflesecurity/trufflehog' && !github.event.pull_request.head.repo.fork }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.25"
+
+      - name: Install dependencies
+        run: sudo apt-get install -y zstd jq
+
+      - name: Install DuckDB
+        run: |
+          wget -q https://github.com/duckdb/duckdb/releases/latest/download/duckdb_cli-linux-amd64.zip
+          unzip duckdb_cli-linux-amd64.zip
+          sudo mv duckdb /usr/local/bin/
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+
+      - name: Run corpora test
+        run: |
+          files=()
+          while IFS= read -r dataset; do
+            [[ -z "$dataset" ]] && continue
+            files+=("$dataset")
+          done <<< "$DATASETS"
+          ./scripts/detector_corpora_test.sh "${files[@]}" | tee /tmp/corpora-results.txt
+
+      - name: Post results to PR
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            const results = fs.readFileSync('/tmp/corpora-results.txt', 'utf8');
+            const body = [
+              `## Corpora Test Results`,
+              ``,
+              `This test scans a real-world dataset of public content to measure how often this detector fires. A high number of unverified or unknown results may indicate the detector is too noisy and could impact signal quality in production.`,
+              ``,
+              `\`\`\``,
+              results,
+              `\`\`\``,
+            ].join('\n');
+            let issue_number;
+            if (context.eventName === 'workflow_dispatch') {
+              const pulls = await github.rest.pulls.list({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                head: `${context.repo.owner}:${context.ref.replace('refs/heads/', '')}`,
+                state: 'open',
+              });
+              if (pulls.data.length === 0) {
+                core.setFailed(`No open PR found for branch ${context.ref}`);
+                return;
+              }
+              issue_number = pulls.data[0].number;
+            } else {
+              issue_number = context.issue.number;
+            }
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number,
+              body,
+            });

scripts/detector_corpora_test.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+set -euo pipefail
+
+if [[ $# -lt 1 ]]; then
+    echo "Usage: $0 <corpora_file.jsonl.zstd> [<corpora_file2.jsonl.zstd> ...]"
+    exit 1
+fi
+
+OUTPUT_JSONL="/tmp/corpora_results.jsonl"
+> "$OUTPUT_JSONL"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(dirname "$SCRIPT_DIR")"
+TRUFFLEHOG_BIN="${REPO_ROOT}/trufflehog"
+
+CGO_ENABLED=0 go build -o "$TRUFFLEHOG_BIN" "$REPO_ROOT"
+
+scan() {
+    local input="$1"
+    set +e
+    unzstd -c "$input" | jq -r .content | "$TRUFFLEHOG_BIN" \
+        --no-update \
+        --log-level=3 \
+        --concurrency=6 \
+        --json \
+        --print-avg-detector-time \
+        stdin >> "$OUTPUT_JSONL" 2>/dev/null
+    set -e
+}
+
+for CORPORA_FILE in "$@"; do
+    if [[ "$CORPORA_FILE" == s3://* ]]; then
+        aws s3 cp "$CORPORA_FILE" - | scan /dev/stdin
+    else
+        scan "$CORPORA_FILE"
+    fi
+done
+
+duckdb -c "
+CREATE TABLE t AS FROM read_json_auto('$OUTPUT_JSONL', ignore_errors=true);
+
+SELECT
+    t.DetectorName detector,
+    COUNT(*) total,
+    SUM(CASE WHEN Verified AND VerificationError IS NULL THEN 1 ELSE 0 END) verified,
+    SUM(CASE WHEN NOT Verified AND VerificationError IS NULL THEN 1 ELSE 0 END) unverified,
+    SUM(CASE WHEN VerificationError IS NOT NULL THEN 1 ELSE 0 END) \"unknown\"
+FROM t
+GROUP BY all
+ORDER BY total DESC, detector
+LIMIT 50;
+"