diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7311fe..edad005 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,11 +30,11 @@ jobs: continue-on-error: ${{ env.ADVISORY == 'true' }} - name: Lint (ruff check) - run: uv run ruff check src/ apps/ mcp/ tests/ *.py + run: uv run ruff check src/ apps/ mcp_servers/ tests/ *.py continue-on-error: ${{ env.ADVISORY == 'true' }} - name: Format check (ruff format) — advisory - run: uv run ruff format --check src/ apps/ mcp/ tests/ *.py + run: uv run ruff format --check src/ apps/ mcp_servers/ tests/ *.py continue-on-error: true - name: Dependency audit (pip-audit) — advisory @@ -42,7 +42,7 @@ jobs: continue-on-error: true - name: SAST (bandit) — advisory - run: uv run bandit -r src/ apps/ mcp/ -c pyproject.toml + run: uv run bandit -r src/ apps/ mcp_servers/ -c pyproject.toml continue-on-error: true - name: Lint HTML (djlint check) diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..6241107 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,128 @@ +# Changelog + +All notable changes to this project are documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [1.1.0] — 2026-06-14 + +### Added + +- **Incident correlator** — new Phase 1 orchestrator (`src/correlator/`) that assembles + incident context from OpenSearch, Mantis tickets, and enrichment in one call. + Exposed via the MCP `investigate` tool and `/investigate` web pages in + opensearch_web. +- **Public IP profiler** — `profile_device` now supports public IPs, with sensor + presence and reverse DNS surfaced in web device cards. +- **Mantis Explorer** — new Flask app for browsing student activity, registered in + `run_all.py` and the Hub landing page. +- **Hub redesign** — list layout, live data-freshness indicators, settings page with + theme dropdown, version + git update status in the footer. +- **Theme system** — 8 community themes (Gruvbox, Tokyo Night, Catppuccin variants), + CSS-variable-driven ECharts colors so charts render correctly on every theme. +- **Shared static blueprint** — `apps/shared/` serves tokens, base CSS, and logos + across all four web apps; per-app duplicate assets removed. +- **Dashboard** — alert trend chart, triage workqueues, per-sensor log time-series, + Tickets tab, sensor filter modal, date-range controls wired into the toolbar. +- **MCP OpenSearch tools** — `investigate`, `histogram`, `aggregate`, + `bulk_enrich_ips`, `count`, `list_filter_categories`, `list_fp_filters`, + `delete_fp_filter`, `get_notice_summary`, `build_share_urls`, + `compare_to_baseline`, `enrich_top_talkers`. +- **MCP querier** — port/proto filters, multi-value IP/sensor parameters, absolute + timestamps, `truncated` flag surfaced on all search results. +- **Wildcard filters** — `notice_note` and `weird_name` accept ES wildcard syntax, + dispatched to `wildcard` queries with exact-match fallback. +- **Filter loader** — validates category/subcategory pairs against + `filters/categories.yaml`. +- **FP manager** — `delete_ip_from_filter` extracted into reusable module. +- **Investigate UX** — escalation indicators on ticket cards, Investigate entry-points + on IP pivot and notice/Suricata records, public device cards, profile buttons. +- **Web UX** — sidebar nav with per-tab persisted filters, sticky-column rendering, + src/dest/both IP role toggle on ip_pivot, error banners for OpenSearch/Mantis, + destination IP filter in search bar. +- **Enricher** — `prewarm_enrichment_cache` background warmer, parallel execution + and result caching on the web enrich path. + +### Changed + +- **Querier refactor** — `src/querier/zeek_modules/base.py` split into focused + modules; silent `None` returns replaced with typed exceptions. +- **Web concurrency** — overview route switched from `ThreadPoolExecutor` fan-out + to `asyncio.gather`; single-flight dedup, shared thread pool, and ETag support + added; `bool.must` switched to `bool.filter` context for cacheability. +- **MCP package rename** — `mcp/` → `mcp_servers/` to resolve a namespace collision + with the upstream `mcp` package. **Breaking** for anyone importing the old path. +- **App rename** — `mantis_web` → `threat_model`. **Breaking** for any external + bookmarks or imports referencing the old name. +- **Pivot/profile/investigate** and `aggregate` MCP tools consolidated. +- **Mantis Explorer** — escalation detection rewritten; `is_escalated` surfaced on + tickets; warning modal added about escalated-count accuracy. +- **Enricher clients** — persistent HTTP sessions, retry adapter, shared console, + `atexit` cleanup across all enricher modules. +- **Dashboard / OpenSearch panels** — low-signal Malcolm panels pruned; protocol + bar replaced with time-series area charts; unified to horizontal bars. +- **Hub branding** — heading renamed to "PISCES Toolkit" with toolbox icon; brand + link navigates to hub; redundant home button removed. +- **OpenSearch web** — search bar redesigned as two-row pill layout; sensor + selector reworked as single clickable button; Investigate button moved to the + global search bar; auth history section replaced with search-all-logs. + +### Fixed + +- **OpenSearch mapping drift** — `terms` aggregations now use a `_source` Painless + script via `source_terms_script()`, surviving indices whose mapping for the + same field disagrees (keyword vs text + `.keyword` subfield on rolled-over + write index). +- **Zeek notice/weird** — exact and wildcard filters now target the `.keyword` + subfield. +- **Dashboard XSS** — date query parameters sanitised; CodeQL taint chain broken + by returning parsed ISO-format date. +- **MCP dest_ip** — pushed into the ES query instead of being post-filtered. +- **Web exceptions** — bare `except` handlers that silently swallowed tracebacks + now log the exception. +- **Cross-protocol query handler** — logs protocol name and error on failure. +- **Querier** — `FilesModule` IP filter flag corrected; `SuricataAlert` summary + type fix; `build_extra_must` tuple correctly unpacked before + `build_base_query`. +- **Correlator** — parallel profile fetches, timeline key override, ticket + deduplication. +- **OpenSearch web** — doubled `script_name` prefix removed from investigate + HTMX paths; em-dash placeholder IPs skipped in overview table. +- **Threat model / Mantis Explorer** — one-time-per-session notice modal. +- **Surface hierarchy** — Catppuccin and Gruvbox themes corrected. + +### Performance + +- **HTTP timeout** — sync and async OpenSearch clients bumped from 30s → 60s to + accommodate slower script aggregations. +- **OpenSearch client cache** — session reused across queries; query construction + optimised. +- **Filter loader** — mtime-based cache avoids re-parsing YAML on every query. +- **Mantis** — index pagination parallelised; HTTP sessions reused; linear scan + and per-request sorts replaced with dict lookups in `data.py`. +- **Filter loading / remapping / post-filtering** — redundant work removed. + +### Removed + +- Root-level standalone app launcher shims. +- `cryptography` dependency dropped; `geoip2` moved to the `offline-enrichment` + extra. +- `pytest` moved out of main dependencies into dev dependencies. +- Theme toggle buttons removed from per-app navbars (now centralised in Hub + settings). + +### CI + +- `djlint` HTML linting added to pre-commit and the CI pipeline. + +## [1.0.0] — 2026-XX-XX + +Initial tagged release. Dashboard redesign, theming, threat model rename, and +Mantis Explorer (PR #40). + +[Unreleased]: https://github.com/liamadale/pisces-scripts/compare/v1.1.0...HEAD +[1.1.0]: https://github.com/liamadale/pisces-scripts/compare/v1.0.0...v1.1.0 +[1.0.0]: https://github.com/liamadale/pisces-scripts/releases/tag/v1.0.0 diff --git a/README.md b/README.md index 644cc9a..c73958f 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,8 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for development guidelines and how to ope To report a vulnerability, follow the process in [SECURITY.md](SECURITY.md). +Release notes for each version are recorded in [CHANGELOG.md](CHANGELOG.md). + --- ## Development Transparency — Use of AI Tooling diff --git a/apps/dashboard_web/opensearch/__init__.py b/apps/dashboard_web/opensearch/__init__.py index bf7fd35..ff6abdf 100644 --- a/apps/dashboard_web/opensearch/__init__.py +++ b/apps/dashboard_web/opensearch/__init__.py @@ -3,6 +3,7 @@ from apps.dashboard_web import cache as dcache from apps.dashboard_web.opensearch.aggregations import ( agg_conn_volume_over_time, + agg_logs_by_sensor_over_time, agg_notice_over_time, agg_opensearch_sensors, agg_opensearch_top_ips, @@ -30,6 +31,7 @@ def section(): "conn_over_time": agg_conn_volume_over_time(time_range, sensors), "sensors": agg_opensearch_sensors(time_range), "top_ips": agg_opensearch_top_ips(time_range, sensors), + "sensor_trend": agg_logs_by_sensor_over_time(time_range, sensors), } except Exception as exc: data = {"error": str(exc)} diff --git a/apps/dashboard_web/opensearch/aggregations.py b/apps/dashboard_web/opensearch/aggregations.py index 00e336e..f34cbbf 100644 --- a/apps/dashboard_web/opensearch/aggregations.py +++ b/apps/dashboard_web/opensearch/aggregations.py @@ -2,12 +2,22 @@ from src.querier.zeek_modules.base import ( FILTERS_DIR, + OpenSearchAuthError, + OpenSearchConnectionError, build_base_query, load_with_remap, query_opensearch, ) +def _safe_query(body: dict, params: dict) -> dict | None: + """query_opensearch wrapper that returns None on connectivity/auth errors.""" + try: + return query_opensearch(body, params) + except (OpenSearchConnectionError, OpenSearchAuthError): + return None + + def parse_sensors(raw: str) -> list | None: """Parse a comma-separated sensor string into a list, or None for 'all'.""" if not raw or raw.strip().lower() == "all": @@ -53,7 +63,7 @@ def agg_opensearch_sensors(time_range: str) -> dict: } } } - raw = query_opensearch(body, params) + raw = _safe_query(body, params) buckets = raw.get("aggregations", {}).get("sensors", {}).get("buckets", []) if raw else [] return { "labels": [b["key"] for b in buckets], @@ -81,7 +91,7 @@ def agg_opensearch_notice_count(time_range: str, sensors: list | None = None) -> body["size"] = 0 body.pop("sort", None) body.pop("_source", None) - raw = query_opensearch(body, params) + raw = _safe_query(body, params) if not raw: return 0 return raw.get("hits", {}).get("total", {}).get("value", 0) @@ -127,7 +137,7 @@ def agg_suricata_alert_count(time_range: str, sensors: list | None = None) -> in body["size"] = 0 body.pop("sort", None) body.pop("_source", None) - raw = query_opensearch(body, params) + raw = _safe_query(body, params) if not raw: return 0 return raw.get("hits", {}).get("total", {}).get("value", 0) @@ -160,7 +170,7 @@ def agg_suricata_over_time(time_range: str, sensors: list | None = None) -> dict } } } - raw = query_opensearch(body, params) + raw = _safe_query(body, params) buckets = raw.get("aggregations", {}).get("over_time", {}).get("buckets", []) if raw else [] return { "timestamps": [b["key_as_string"] for b in buckets], @@ -199,7 +209,7 @@ def agg_notice_over_time(time_range: str, sensors: list | None = None) -> dict: } } } - raw = query_opensearch(body, params) + raw = _safe_query(body, params) buckets = raw.get("aggregations", {}).get("over_time", {}).get("buckets", []) if raw else [] return { "timestamps": [b["key_as_string"] for b in buckets], @@ -235,7 +245,7 @@ def agg_conn_volume_over_time(time_range: str, sensors: list | None = None) -> d } } } - raw = query_opensearch(body, params) + raw = _safe_query(body, params) buckets = raw.get("aggregations", {}).get("over_time", {}).get("buckets", []) if raw else [] return { "timestamps": [b["key_as_string"] for b in buckets], @@ -244,6 +254,68 @@ def agg_conn_volume_over_time(time_range: str, sensors: list | None = None) -> d } +def agg_logs_by_sensor_over_time(time_range: str, sensors: list | None = None) -> dict: + """Total log count per sensor as aligned time series (terms → date_histogram).""" + interval = _interval_for_range(time_range) + body, params = build_base_query( + must_not=[], + extra_must=[], + source_fields=[], + limit=0, + time_range=time_range, + sensors=sensors, + datasets=["all"], + public_only=False, + src_ip_filter=None, + direction=None, + ) + body["size"] = 0 + body.pop("sort", None) + body.pop("_source", None) + body["aggs"] = { + "by_sensor": { + "terms": {"field": "host.name", "size": 50, "order": {"_count": "desc"}}, + "aggs": { + "over_time": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": interval, + "min_doc_count": 0, + } + } + }, + } + } + raw = _safe_query(body, params) + sensor_buckets = ( + raw.get("aggregations", {}).get("by_sensor", {}).get("buckets", []) if raw else [] + ) + + # Collect all unique timestamps in order across all sensors + all_ts: dict[str, None] = {} + for sb in sensor_buckets: + for tb in sb.get("over_time", {}).get("buckets", []): + all_ts[tb["key_as_string"]] = None + timestamps = list(all_ts.keys()) + + # Build per-sensor series aligned to the shared timestamp list + series = [] + for sb in sensor_buckets: + ts_map = { + tb["key_as_string"]: tb["doc_count"] + for tb in sb.get("over_time", {}).get("buckets", []) + } + series.append( + { + "sensor": sb["key"], + "counts": [ts_map.get(t, 0) for t in timestamps], + "total": sb["doc_count"], + } + ) + + return {"timestamps": timestamps, "series": series, "interval": interval} + + def agg_new_ips_delta(time_range: str, sensors: list | None = None) -> dict: """Compare unique source IPs in the current window vs the previous window. @@ -270,7 +342,7 @@ def _unique_count(tr: str) -> int: body["aggs"] = { "uniq": {"cardinality": {"field": "source.ip", "precision_threshold": 3000}} } - raw = query_opensearch(body, params) + raw = _safe_query(body, params) if not raw: return 0 return raw.get("aggregations", {}).get("uniq", {}).get("value", 0) diff --git a/apps/dashboard_web/opensearch/malcolm.py b/apps/dashboard_web/opensearch/malcolm.py index d6a279d..f960cdb 100644 --- a/apps/dashboard_web/opensearch/malcolm.py +++ b/apps/dashboard_web/opensearch/malcolm.py @@ -6,7 +6,20 @@ import concurrent.futures -from src.querier.zeek_modules.base import build_base_query, query_opensearch +from src.querier.zeek_modules.base import ( + OpenSearchAuthError, + OpenSearchConnectionError, + build_base_query, + query_opensearch, +) + + +def _safe_query(body: dict, params: dict) -> dict | None: + """query_opensearch wrapper that returns None on connectivity/auth errors.""" + try: + return query_opensearch(body, params) + except (OpenSearchConnectionError, OpenSearchAuthError): + return None def _terms(field: str, time_range: str, datasets: list, size: int = 20) -> dict: @@ -27,7 +40,7 @@ def _terms(field: str, time_range: str, datasets: list, size: int = 20) -> dict: body.pop("sort", None) body.pop("_source", None) body["aggs"] = {"r": {"terms": {"field": field, "size": size, "order": {"_count": "desc"}}}} - raw = query_opensearch(body, params) + raw = _safe_query(body, params) buckets = raw.get("aggregations", {}).get("r", {}).get("buckets", []) if raw else [] return { "labels": [b["key"] for b in buckets], @@ -60,7 +73,7 @@ def _sum_terms( "aggs": {"total": {"sum": {"field": sum_field}}}, } } - raw = query_opensearch(body, params) + raw = _safe_query(body, params) buckets = raw.get("aggregations", {}).get("r", {}).get("buckets", []) if raw else [] buckets = sorted(buckets, key=lambda b: -b.get("total", {}).get("value", 0)) return { diff --git a/apps/dashboard_web/opensearch/templates/opensearch/section.html b/apps/dashboard_web/opensearch/templates/opensearch/section.html index c2bd447..c651f33 100644 --- a/apps/dashboard_web/opensearch/templates/opensearch/section.html +++ b/apps/dashboard_web/opensearch/templates/opensearch/section.html @@ -60,15 +60,25 @@ {% endif %} +
+
Log Count by Sensor ({{ data.sensor_trend.interval }} intervals)
+ {% if not data.sensor_trend.timestamps %} +
No sensor data in this window
+ {% else %} +
+ {% endif %} +
+ diff --git a/apps/dashboard_web/run.py b/apps/dashboard_web/run.py index e14cd6e..f2fbbb5 100644 --- a/apps/dashboard_web/run.py +++ b/apps/dashboard_web/run.py @@ -21,7 +21,9 @@ app = create_app() -if __name__ == "__main__": + +def main() -> None: + """Entry point for `pisces-dashboard` console script.""" parser = argparse.ArgumentParser(description="PISCES Dashboard") parser.add_argument("--host", default="0.0.0.0", help="Bind address (default: 0.0.0.0)") parser.add_argument("--port", type=int, default=5004, help="Port (default: 5004)") @@ -29,5 +31,8 @@ "--debug", action="store_true", default=False, help="Enable Flask debug mode" ) args = parser.parse_args() - app.run(host=args.host, port=args.port, debug=args.debug, threaded=True) + + +if __name__ == "__main__": + main() diff --git a/apps/hub/run.py b/apps/hub/run.py index d366d09..8c5a10c 100644 --- a/apps/hub/run.py +++ b/apps/hub/run.py @@ -15,12 +15,16 @@ from apps.hub.app import create_app -if __name__ == "__main__": + +def main() -> None: + """Entry point for `pisces-hub` console script.""" parser = argparse.ArgumentParser(description="PISCES Hub Portal") parser.add_argument("--host", default="0.0.0.0") parser.add_argument("--port", type=int, default=5000) parser.add_argument("--debug", action="store_true", default=False) args = parser.parse_args() + create_app().run(host=args.host, port=args.port, debug=args.debug, threaded=True) - app = create_app() - app.run(host=args.host, port=args.port, debug=args.debug, threaded=True) + +if __name__ == "__main__": + main() diff --git a/apps/hub/templates/index.html b/apps/hub/templates/index.html index baf2924..4ddf89c 100644 --- a/apps/hub/templates/index.html +++ b/apps/hub/templates/index.html @@ -180,7 +180,7 @@

PISCES Toolkit