chore(release): v1.1.0

.github/workflows/ci.yml

@@ -30,19 +30,19 @@ jobs:
         continue-on-error: ${{ env.ADVISORY == 'true' }}
 
       - name: Lint (ruff check)
-        run: uv run ruff check src/ apps/ mcp/ tests/ *.py
+        run: uv run ruff check src/ apps/ mcp_servers/ tests/ *.py
         continue-on-error: ${{ env.ADVISORY == 'true' }}
 
       - name: Format check (ruff format) — advisory
-        run: uv run ruff format --check src/ apps/ mcp/ tests/ *.py
+        run: uv run ruff format --check src/ apps/ mcp_servers/ tests/ *.py
         continue-on-error: true
 
       - name: Dependency audit (pip-audit) — advisory
         run: uv run pip-audit
         continue-on-error: true
 
       - name: SAST (bandit) — advisory
-        run: uv run bandit -r src/ apps/ mcp/ -c pyproject.toml
+        run: uv run bandit -r src/ apps/ mcp_servers/ -c pyproject.toml
         continue-on-error: true
 
       - name: Lint HTML (djlint check)

CHANGELOG.md

@@ -0,0 +1,128 @@
+# Changelog
+
+All notable changes to this project are documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [1.1.0] — 2026-06-14
+
+### Added
+
+- **Incident correlator** — new Phase 1 orchestrator (`src/correlator/`) that assembles
+  incident context from OpenSearch, Mantis tickets, and enrichment in one call.
+  Exposed via the MCP `investigate` tool and `/investigate` web pages in
+  opensearch_web.
+- **Public IP profiler** — `profile_device` now supports public IPs, with sensor
+  presence and reverse DNS surfaced in web device cards.
+- **Mantis Explorer** — new Flask app for browsing student activity, registered in
+  `run_all.py` and the Hub landing page.
+- **Hub redesign** — list layout, live data-freshness indicators, settings page with
+  theme dropdown, version + git update status in the footer.
+- **Theme system** — 8 community themes (Gruvbox, Tokyo Night, Catppuccin variants),
+  CSS-variable-driven ECharts colors so charts render correctly on every theme.
+- **Shared static blueprint** — `apps/shared/` serves tokens, base CSS, and logos
+  across all four web apps; per-app duplicate assets removed.
+- **Dashboard** — alert trend chart, triage workqueues, per-sensor log time-series,
+  Tickets tab, sensor filter modal, date-range controls wired into the toolbar.
+- **MCP OpenSearch tools** — `investigate`, `histogram`, `aggregate`,
+  `bulk_enrich_ips`, `count`, `list_filter_categories`, `list_fp_filters`,
+  `delete_fp_filter`, `get_notice_summary`, `build_share_urls`,
+  `compare_to_baseline`, `enrich_top_talkers`.
+- **MCP querier** — port/proto filters, multi-value IP/sensor parameters, absolute
+  timestamps, `truncated` flag surfaced on all search results.
+- **Wildcard filters** — `notice_note` and `weird_name` accept ES wildcard syntax,
+  dispatched to `wildcard` queries with exact-match fallback.
+- **Filter loader** — validates category/subcategory pairs against
+  `filters/categories.yaml`.
+- **FP manager** — `delete_ip_from_filter` extracted into reusable module.
+- **Investigate UX** — escalation indicators on ticket cards, Investigate entry-points
+  on IP pivot and notice/Suricata records, public device cards, profile buttons.
+- **Web UX** — sidebar nav with per-tab persisted filters, sticky-column rendering,
+  src/dest/both IP role toggle on ip_pivot, error banners for OpenSearch/Mantis,
+  destination IP filter in search bar.
+- **Enricher** — `prewarm_enrichment_cache` background warmer, parallel execution
+  and result caching on the web enrich path.
+
+### Changed
+
+- **Querier refactor** — `src/querier/zeek_modules/base.py` split into focused
+  modules; silent `None` returns replaced with typed exceptions.
+- **Web concurrency** — overview route switched from `ThreadPoolExecutor` fan-out
+  to `asyncio.gather`; single-flight dedup, shared thread pool, and ETag support
+  added; `bool.must` switched to `bool.filter` context for cacheability.
+- **MCP package rename** — `mcp/` → `mcp_servers/` to resolve a namespace collision
+  with the upstream `mcp` package. **Breaking** for anyone importing the old path.
+- **App rename** — `mantis_web` → `threat_model`. **Breaking** for any external
+  bookmarks or imports referencing the old name.
+- **Pivot/profile/investigate** and `aggregate` MCP tools consolidated.
+- **Mantis Explorer** — escalation detection rewritten; `is_escalated` surfaced on
+  tickets; warning modal added about escalated-count accuracy.
+- **Enricher clients** — persistent HTTP sessions, retry adapter, shared console,
+  `atexit` cleanup across all enricher modules.
+- **Dashboard / OpenSearch panels** — low-signal Malcolm panels pruned; protocol
+  bar replaced with time-series area charts; unified to horizontal bars.
+- **Hub branding** — heading renamed to "PISCES Toolkit" with toolbox icon; brand
+  link navigates to hub; redundant home button removed.
+- **OpenSearch web** — search bar redesigned as two-row pill layout; sensor
+  selector reworked as single clickable button; Investigate button moved to the
+  global search bar; auth history section replaced with search-all-logs.
+
+### Fixed
+
+- **OpenSearch mapping drift** — `terms` aggregations now use a `_source` Painless
+  script via `source_terms_script()`, surviving indices whose mapping for the
+  same field disagrees (keyword vs text + `.keyword` subfield on rolled-over
+  write index).
+- **Zeek notice/weird** — exact and wildcard filters now target the `.keyword`
+  subfield.
+- **Dashboard XSS** — date query parameters sanitised; CodeQL taint chain broken
+  by returning parsed ISO-format date.
+- **MCP dest_ip** — pushed into the ES query instead of being post-filtered.
+- **Web exceptions** — bare `except` handlers that silently swallowed tracebacks
+  now log the exception.
+- **Cross-protocol query handler** — logs protocol name and error on failure.
+- **Querier** — `FilesModule` IP filter flag corrected; `SuricataAlert` summary
+  type fix; `build_extra_must` tuple correctly unpacked before
+  `build_base_query`.
+- **Correlator** — parallel profile fetches, timeline key override, ticket
+  deduplication.
+- **OpenSearch web** — doubled `script_name` prefix removed from investigate
+  HTMX paths; em-dash placeholder IPs skipped in overview table.
+- **Threat model / Mantis Explorer** — one-time-per-session notice modal.
+- **Surface hierarchy** — Catppuccin and Gruvbox themes corrected.
+
+### Performance
+
+- **HTTP timeout** — sync and async OpenSearch clients bumped from 30s → 60s to
+  accommodate slower script aggregations.
+- **OpenSearch client cache** — session reused across queries; query construction
+  optimised.
+- **Filter loader** — mtime-based cache avoids re-parsing YAML on every query.
+- **Mantis** — index pagination parallelised; HTTP sessions reused; linear scan
+  and per-request sorts replaced with dict lookups in `data.py`.
+- **Filter loading / remapping / post-filtering** — redundant work removed.
+
+### Removed
+
+- Root-level standalone app launcher shims.
+- `cryptography` dependency dropped; `geoip2` moved to the `offline-enrichment`
+  extra.
+- `pytest` moved out of main dependencies into dev dependencies.
+- Theme toggle buttons removed from per-app navbars (now centralised in Hub
+  settings).
+
+### CI
+
+- `djlint` HTML linting added to pre-commit and the CI pipeline.
+
+## [1.0.0] — 2026-XX-XX
+
+Initial tagged release. Dashboard redesign, theming, threat model rename, and
+Mantis Explorer (PR #40).
+
+[Unreleased]: https://github.com/liamadale/pisces-scripts/compare/v1.1.0...HEAD
+[1.1.0]: https://github.com/liamadale/pisces-scripts/compare/v1.0.0...v1.1.0
+[1.0.0]: https://github.com/liamadale/pisces-scripts/releases/tag/v1.0.0

README.md

@@ -85,6 +85,8 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for development guidelines and how to ope
 
 To report a vulnerability, follow the process in [SECURITY.md](SECURITY.md).
 
+Release notes for each version are recorded in [CHANGELOG.md](CHANGELOG.md).
+
 ---
 
 ## Development Transparency — Use of AI Tooling

apps/dashboard_web/opensearch/__init__.py

@@ -3,6 +3,7 @@
 from apps.dashboard_web import cache as dcache
 from apps.dashboard_web.opensearch.aggregations import (
     agg_conn_volume_over_time,
+    agg_logs_by_sensor_over_time,
     agg_notice_over_time,
     agg_opensearch_sensors,
     agg_opensearch_top_ips,
@@ -30,6 +31,7 @@ def section():
             "conn_over_time": agg_conn_volume_over_time(time_range, sensors),
             "sensors": agg_opensearch_sensors(time_range),
             "top_ips": agg_opensearch_top_ips(time_range, sensors),
+            "sensor_trend": agg_logs_by_sensor_over_time(time_range, sensors),
         }
     except Exception as exc:
         data = {"error": str(exc)}

apps/dashboard_web/opensearch/aggregations.py

@@ -2,12 +2,22 @@
 
 from src.querier.zeek_modules.base import (
     FILTERS_DIR,
+    OpenSearchAuthError,
+    OpenSearchConnectionError,
     build_base_query,
     load_with_remap,
     query_opensearch,
 )
 
 
+def _safe_query(body: dict, params: dict) -> dict | None:
+    """query_opensearch wrapper that returns None on connectivity/auth errors."""
+    try:
+        return query_opensearch(body, params)
+    except (OpenSearchConnectionError, OpenSearchAuthError):
+        return None
+
+
 def parse_sensors(raw: str) -> list | None:
     """Parse a comma-separated sensor string into a list, or None for 'all'."""
     if not raw or raw.strip().lower() == "all":
@@ -53,7 +63,7 @@ def agg_opensearch_sensors(time_range: str) -> dict:
             }
         }
     }
-    raw = query_opensearch(body, params)
+    raw = _safe_query(body, params)
     buckets = raw.get("aggregations", {}).get("sensors", {}).get("buckets", []) if raw else []
     return {
         "labels": [b["key"] for b in buckets],
@@ -81,7 +91,7 @@ def agg_opensearch_notice_count(time_range: str, sensors: list | None = None) ->
     body["size"] = 0
     body.pop("sort", None)
     body.pop("_source", None)
-    raw = query_opensearch(body, params)
+    raw = _safe_query(body, params)
     if not raw:
         return 0
     return raw.get("hits", {}).get("total", {}).get("value", 0)
@@ -127,7 +137,7 @@ def agg_suricata_alert_count(time_range: str, sensors: list | None = None) -> in
     body["size"] = 0
     body.pop("sort", None)
     body.pop("_source", None)
-    raw = query_opensearch(body, params)
+    raw = _safe_query(body, params)
     if not raw:
         return 0
     return raw.get("hits", {}).get("total", {}).get("value", 0)
@@ -160,7 +170,7 @@ def agg_suricata_over_time(time_range: str, sensors: list | None = None) -> dict
             }
         }
     }
-    raw = query_opensearch(body, params)
+    raw = _safe_query(body, params)
     buckets = raw.get("aggregations", {}).get("over_time", {}).get("buckets", []) if raw else []
     return {
         "timestamps": [b["key_as_string"] for b in buckets],
@@ -199,7 +209,7 @@ def agg_notice_over_time(time_range: str, sensors: list | None = None) -> dict:
             }
         }
     }
-    raw = query_opensearch(body, params)
+    raw = _safe_query(body, params)
     buckets = raw.get("aggregations", {}).get("over_time", {}).get("buckets", []) if raw else []
     return {
         "timestamps": [b["key_as_string"] for b in buckets],
@@ -235,7 +245,7 @@ def agg_conn_volume_over_time(time_range: str, sensors: list | None = None) -> d
             }
         }
     }
-    raw = query_opensearch(body, params)
+    raw = _safe_query(body, params)
     buckets = raw.get("aggregations", {}).get("over_time", {}).get("buckets", []) if raw else []
     return {
         "timestamps": [b["key_as_string"] for b in buckets],
@@ -244,6 +254,68 @@ def agg_conn_volume_over_time(time_range: str, sensors: list | None = None) -> d
     }
 
 
+def agg_logs_by_sensor_over_time(time_range: str, sensors: list | None = None) -> dict:
+    """Total log count per sensor as aligned time series (terms → date_histogram)."""
+    interval = _interval_for_range(time_range)
+    body, params = build_base_query(
+        must_not=[],
+        extra_must=[],
+        source_fields=[],
+        limit=0,
+        time_range=time_range,
+        sensors=sensors,
+        datasets=["all"],
+        public_only=False,
+        src_ip_filter=None,
+        direction=None,
+    )
+    body["size"] = 0
+    body.pop("sort", None)
+    body.pop("_source", None)
+    body["aggs"] = {
+        "by_sensor": {
+            "terms": {"field": "host.name", "size": 50, "order": {"_count": "desc"}},
+            "aggs": {
+                "over_time": {
+                    "date_histogram": {
+                        "field": "@timestamp",
+                        "fixed_interval": interval,
+                        "min_doc_count": 0,
+                    }
+                }
+            },
+        }
+    }
+    raw = _safe_query(body, params)
+    sensor_buckets = (
+        raw.get("aggregations", {}).get("by_sensor", {}).get("buckets", []) if raw else []
+    )
+
+    # Collect all unique timestamps in order across all sensors
+    all_ts: dict[str, None] = {}
+    for sb in sensor_buckets:
+        for tb in sb.get("over_time", {}).get("buckets", []):
+            all_ts[tb["key_as_string"]] = None
+    timestamps = list(all_ts.keys())
+
+    # Build per-sensor series aligned to the shared timestamp list
+    series = []
+    for sb in sensor_buckets:
+        ts_map = {
+            tb["key_as_string"]: tb["doc_count"]
+            for tb in sb.get("over_time", {}).get("buckets", [])
+        }
+        series.append(
+            {
+                "sensor": sb["key"],
+                "counts": [ts_map.get(t, 0) for t in timestamps],
+                "total": sb["doc_count"],
+            }
+        )
+
+    return {"timestamps": timestamps, "series": series, "interval": interval}
+
+
 def agg_new_ips_delta(time_range: str, sensors: list | None = None) -> dict:
     """Compare unique source IPs in the current window vs the previous window.
 
@@ -270,7 +342,7 @@ def _unique_count(tr: str) -> int:
         body["aggs"] = {
             "uniq": {"cardinality": {"field": "source.ip", "precision_threshold": 3000}}
         }
-        raw = query_opensearch(body, params)
+        raw = _safe_query(body, params)
         if not raw:
             return 0
         return raw.get("aggregations", {}).get("uniq", {}).get("value", 0)

apps/dashboard_web/opensearch/malcolm.py

@@ -6,7 +6,20 @@
 
 import concurrent.futures
 
-from src.querier.zeek_modules.base import build_base_query, query_opensearch
+from src.querier.zeek_modules.base import (
+    OpenSearchAuthError,
+    OpenSearchConnectionError,
+    build_base_query,
+    query_opensearch,
+)
+
+
+def _safe_query(body: dict, params: dict) -> dict | None:
+    """query_opensearch wrapper that returns None on connectivity/auth errors."""
+    try:
+        return query_opensearch(body, params)
+    except (OpenSearchConnectionError, OpenSearchAuthError):
+        return None
 
 
 def _terms(field: str, time_range: str, datasets: list, size: int = 20) -> dict:
@@ -27,7 +40,7 @@ def _terms(field: str, time_range: str, datasets: list, size: int = 20) -> dict:
     body.pop("sort", None)
     body.pop("_source", None)
     body["aggs"] = {"r": {"terms": {"field": field, "size": size, "order": {"_count": "desc"}}}}
-    raw = query_opensearch(body, params)
+    raw = _safe_query(body, params)
     buckets = raw.get("aggregations", {}).get("r", {}).get("buckets", []) if raw else []
     return {
         "labels": [b["key"] for b in buckets],
@@ -60,7 +73,7 @@ def _sum_terms(
             "aggs": {"total": {"sum": {"field": sum_field}}},
         }
     }
-    raw = query_opensearch(body, params)
+    raw = _safe_query(body, params)
     buckets = raw.get("aggregations", {}).get("r", {}).get("buckets", []) if raw else []
     buckets = sorted(buckets, key=lambda b: -b.get("total", {}).get("value", 0))
     return {