From 4a9b862c3b5ca12d2dca9e681dd85d09dff0ebf1 Mon Sep 17 00:00:00 2001 From: liamadale Date: Fri, 24 Apr 2026 17:50:39 -0700 Subject: [PATCH 001/109] feat(opensearch-web): add destination IP filter to query pipeline and search bar Extend the query backend and web form to support filtering by destination IP alongside the existing src_ip filter. - build_base_query gains a dest_ip_filter param that appends a destination.ip term clause when set - run_query guards dest_ip_filter the same way src_ip is guarded: only applied to modules that declare destination.ip in SOURCE_FIELDS - build_search_params_from_request reads the dest_ip form value - base.html search bar exposes a Dst IP text input --- apps/opensearch_web/queries.py | 1 + apps/opensearch_web/templates/base.html | 7 +++++++ src/querier/zeek_modules/base.py | 10 +++++++++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/apps/opensearch_web/queries.py b/apps/opensearch_web/queries.py index 42ae62d..cf30403 100644 --- a/apps/opensearch_web/queries.py +++ b/apps/opensearch_web/queries.py @@ -18,6 +18,7 @@ def build_search_params_from_request(request, extra_keys=None) -> dict: "limit": int(v) if (v := request.values.get("limit", "").strip()) and v.isdigit() else 500, "public_only": request.values.get("public_only") in ("on", "true", "1"), "src_ip": request.values.get("src_ip") or None, + "dest_ip": request.values.get("dest_ip") or None, "direction": request.values.get("direction") or None, "no_filters": False, "use_cache": False, diff --git a/apps/opensearch_web/templates/base.html b/apps/opensearch_web/templates/base.html index e6a2c83..460321e 100644 --- a/apps/opensearch_web/templates/base.html +++ b/apps/opensearch_web/templates/base.html @@ -111,6 +111,13 @@
+ + + +
+ + - {% for cat in categories %}{% endfor %} @@ -56,7 +56,7 @@ {% for cat, cat_data in categories.items() %} {% for sub in cat_data.get('subcategories', []) %} - {% endfor %} {% endfor %} diff --git a/apps/opensearch_web/templates/partials/record_detail.html b/apps/opensearch_web/templates/partials/record_detail.html index 7b01d50..09a0773 100644 --- a/apps/opensearch_web/templates/partials/record_detail.html +++ b/apps/opensearch_web/templates/partials/record_detail.html @@ -37,7 +37,7 @@
- {% set src_ip = record.get('src_ip') %} + {% set src_ip = record.get('src_ip') %} {% set dest_ip = record.get('dest_ip') %} {% if src_ip or dest_ip or record.get('notice_note') %}
@@ -137,5 +137,4 @@
{% endif %} - diff --git a/pyproject.toml b/pyproject.toml index 435e2ef..7767d18 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,10 +80,11 @@ skips = ["B101"] profile = "jinja" indent = 2 max_line_length = 100 -ignore = "H021,H023,H030,H031" +ignore = "H021,H023,H030,H031,J018" # H021: inline styles are intentional in dashboard/chart templates # H023: HTML entity references (· etc.) are standard and intentional # H030/H031: meta description/keywords are false positives on Jinja child templates +# J018: cross-app internal links cannot use url_for() in a multi-app Flask setup [tool.pytest.ini_options] testpaths = ["tests"] From 0fca72ddb50967d27d02f49cc8850d0366364496 Mon Sep 17 00:00:00 2001 From: liamadale Date: Wed, 29 Apr 2026 09:58:06 -0700 Subject: [PATCH 006/109] feat(mantis): rewrite escalation detection and surface is_escalated on tickets Replace the single loose _ESCALATION_RE pattern with a two-stage function _note_is_escalation() that eliminates false positives: - Stage 1: matches past-tense client-contact phrases (informed/notified the client, let the client know, reached out to the client, etc.) and skips any match whose 20-char prefix contains "will" (future intent). - Stage 2: matches past-tense "escalated [this/it] to [the] client" and skips matches prefixed with "not" or "won't" (negated intent). Previously, bare "escalat*" triggered on "privilege escalation", conditional futures ("will let the client know"), and negations ("not going to escalate to the client"). The two-stage approach targets only confirmed past-action phrases. _normalize_issue() now exposes is_escalated and escalated_by on every normalised ticket dict. activity_report._ticket_ref() propagates is_escalated into StudentStats.created_tickets. student_activity adds an "Escalated" column to the summary table, shows a per-student count in the detail view, supports --org / --since / --until CLI flags, and refactors the graph helper into a reusable _plot_ticket_timeline() shared by per-student and per-org views. --- src/mantis/activity_report.py | 795 +++++++++++++++++++++++++++++++++ src/mantis/mantis_search.py | 60 +++ src/mantis/student_activity.py | 312 +++++++++++-- 3 files changed, 1138 insertions(+), 29 deletions(-) create mode 100644 src/mantis/activity_report.py diff --git a/src/mantis/activity_report.py b/src/mantis/activity_report.py new file mode 100644 index 0000000..5de7891 --- /dev/null +++ b/src/mantis/activity_report.py @@ -0,0 +1,795 @@ +#!/usr/bin/env python3 +""" +Student activity report — counts tickets created and notes written per student. + +Students are defined as any reporter who has never acted as a ticket handler +anywhere in the corpus (mirrors the handler_registry logic in mantis_index.py). + +Reads from the local offline index by default; pass --live to fetch from the +REST API instead (requires MANTIS_API_TOKEN + MANTIS_API_URL). + +When --student matches exactly one name, a detailed view is shown with ticket +titles and links. When it matches multiple, the user is prompted to pick one. + +Usage: + python src/mantis/activity_report.py + python src/mantis/activity_report.py --live + python src/mantis/activity_report.py --sort tickets + python src/mantis/activity_report.py --project bonney-lake + python src/mantis/activity_report.py --student alice + python src/mantis/activity_report.py --student alice --graph + python src/mantis/activity_report.py --org 'bellevue college' + python src/mantis/activity_report.py --org 'bellevue college' --student alice + python src/mantis/activity_report.py --since 2025-01-01 --until 2025-04-30 + python src/mantis/activity_report.py --input data/tickets/indexed/tickets_index.json +""" + +import argparse +import json +import os +import sys +from collections import defaultdict +from dataclasses import dataclass, field +from datetime import date, timedelta + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + +import urllib3 +from dotenv import load_dotenv +from rich import box +from rich.console import Console +from rich.rule import Rule +from rich.table import Table + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +_BASE = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +DEFAULT_INDEX = os.path.join(_BASE, "data", "tickets", "indexed", "tickets_index.json") + +console = Console() + + +def _ordinal(n: int) -> str: + """Return an integer with its ordinal suffix: 1st, 2nd, 3rd, 4th…""" + if 11 <= (n % 100) <= 13: + suffix = "th" + else: + suffix = {1: "st", 2: "nd", 3: "rd"}.get(n % 10, "th") + return f"{n}{suffix}" + + +def _format_date_range(date_strs: list[str]) -> str: + """Format a collection of YYYY-MM-DD strings as a human-readable span. + + Returns strings like "Jan 1st – Apr 28th" for same-year ranges, or + "Jan 1st, 2024 – Apr 28th, 2025" when the range crosses calendar years. + Returns an empty string when no valid dates are present. + """ + valid = [d for d in date_strs if d and len(d) >= 10] + if not valid: + return "" + parsed = sorted(date.fromisoformat(d[:10]) for d in valid) + first, last = parsed[0], parsed[-1] + cross_year = first.year != last.year + + def _fmt(d: date) -> str: + base = f"{d.strftime('%b')} {_ordinal(d.day)}" + return f"{base}, {d.year}" if cross_year else base + + if first == last: + return _fmt(first) + return f"{_fmt(first)} – {_fmt(last)}" + + +def _filter_by_date_range( + tickets: list[dict], + since: date | None, + until: date | None, +) -> list[dict]: + """Return only tickets whose created_at falls within [since, until]. + + Tickets with no or unparseable created_at are kept so data is not silently lost. + """ + if not since and not until: + return tickets + result = [] + for t in tickets: + raw = (t.get("created_at") or "")[:10] + if not raw: + result.append(t) + continue + try: + d = date.fromisoformat(raw) + except ValueError: + result.append(t) + continue + if since and d < since: + continue + if until and d > until: + continue + result.append(t) + return result + + +# Minimal ticket ref stored per student — id, summary, url, status, created_at +_TicketRef = dict + + +@dataclass +class StudentStats: + """Activity data for a single student reporter.""" + + name: str + created_tickets: list[_TicketRef] = field(default_factory=list) + # Tickets they left notes on that they did NOT create (keyed by id to deduplicate) + _noted: dict[str, _TicketRef] = field(default_factory=dict, repr=False) + notes_written: int = 0 + projects: set[str] = field(default_factory=set) + + @property + def tickets_created(self) -> int: + return len(self.created_tickets) + + @property + def noted_tickets(self) -> list[_TicketRef]: + """Unique tickets this student commented on but did not create.""" + return list(self._noted.values()) + + escalated_tickets: int = 0 + categories: set[str] = field(default_factory=set) + + @property + def total_activity(self) -> int: + return self.tickets_created + self.notes_written + + def add_noted(self, ticket: _TicketRef) -> None: + """Record a ticket this student left a note on (deduplicates by id).""" + tid = ticket["id"] + if tid not in self._noted: + self._noted[tid] = ticket + + +def _ticket_ref(ticket: dict) -> _TicketRef: + """Extract the minimal displayable fields from a normalized ticket dict.""" + return { + "id": ticket.get("id", ""), + "summary": ticket.get("summary", ""), + "url": ticket.get("url", ""), + "status": ticket.get("status", ""), + "created_at": ticket.get("created_at", ""), + "is_escalated": bool(ticket.get("is_escalated")), + } + + +def _load_offline(path: str) -> list[dict]: + """Load normalized tickets from the offline index JSON.""" + if not os.path.exists(path): + console.print(f"[red]Offline index not found: {path}[/red]") + console.print("[dim]Run: python src/mantis/mantis_index.py[/dim]") + sys.exit(1) + with open(path) as fh: + return json.load(fh) + + +def _load_live(project_filter: str | None) -> list[dict]: + """Fetch all tickets from the MantisBT REST API.""" + import requests + + from src.mantis.mantis_search import _normalize_issue + + api_url = os.environ.get("MANTIS_API_URL", "").rstrip("/") + api_token = os.environ.get("MANTIS_API_TOKEN", "") + if not api_url or not api_token: + console.print("[red]MANTIS_API_URL and MANTIS_API_TOKEN are required for --live[/red]") + sys.exit(1) + + headers = {"Authorization": api_token} + all_raw: list[dict] = [] + page = 1 + + console.print("[dim]Fetching tickets from Mantis REST API...[/dim]") + while True: + resp = requests.get( + f"{api_url}/api/rest/issues", + headers=headers, + params={"page_size": 200, "page": page}, + timeout=30, + verify=False, + ) + if not resp.ok: + console.print(f"[red]API error {resp.status_code} on page {page}[/red]") + break + data = resp.json() + issues = data.get("issues", []) + if not issues: + break + all_raw.extend(issues) + total = data.get("total_count") + if total and page * 200 >= total: + break + if len(issues) < 200: + break + page += 1 + + console.print(f"[dim]Fetched {len(all_raw)} tickets — normalizing...[/dim]") + handler_registry: set[int] = { + issue["handler"]["id"] for issue in all_raw if issue.get("handler") + } + tickets = [_normalize_issue(issue, api_url, handler_registry) for issue in all_raw] + + if project_filter: + tickets = [t for t in tickets if project_filter.lower() in t.get("project", "").lower()] + + return tickets + + +def build_report( + tickets: list[dict], + project_filter: str | None = None, +) -> dict[int, StudentStats]: + """Aggregate per-student activity from a normalized ticket list. + + Args: + tickets: Normalized ticket dicts from the offline index or live API. + project_filter: Optional substring filter on the project/city field. + + Returns: + Mapping of reporter_id → StudentStats, excluding known handlers. + """ + if project_filter: + tickets = [t for t in tickets if project_filter.lower() in t.get("project", "").lower()] + + # Rebuild handler registry from this corpus + handler_ids: set[int] = set() + for t in tickets: + h = t.get("handler") + if h and h.get("id"): + handler_ids.add(h["id"]) + for note in t.get("notes", []): + if note.get("is_admin_note"): + handler_ids.add(note["reporter"]["id"]) + + stats: dict[int, StudentStats] = defaultdict(lambda: StudentStats(name="")) + + for ticket in tickets: + reporter = ticket.get("reporter", {}) + reporter_id = reporter.get("id", 0) + reporter_name = reporter.get("name", "unknown") + project = ticket.get("project", "") + ref = _ticket_ref(ticket) + + if reporter_id not in handler_ids: + stats[reporter_id].name = reporter_name + stats[reporter_id].created_tickets.append(ref) + if project: + stats[reporter_id].projects.add(project) + if ticket.get("is_escalated"): + stats[reporter_id].escalated_tickets += 1 + category = ticket.get("category", "") + if category: + stats[reporter_id].categories.add(category) + + for note in ticket.get("notes", []): + note_reporter = note.get("reporter", {}) + note_id = note_reporter.get("id", 0) + note_name = note_reporter.get("name", "unknown") + + if note_id in handler_ids: + continue + + stats[note_id].name = note_name + stats[note_id].notes_written += 1 + if project: + stats[note_id].projects.add(project) + # Only track as "noted" if they didn't create it + if note_id != reporter_id: + stats[note_id].add_noted(ref) + + return dict(stats) + + +def _pick_student(matches: list[tuple[int, StudentStats]]) -> tuple[int, StudentStats] | None: + """Prompt the user to select one student from a list of matches.""" + console.print(f"\n[yellow]Found {len(matches)} students matching that name:[/yellow]\n") + for i, (_, s) in enumerate(matches, 1): + console.print( + f" [cyan]{i}[/cyan]. {s.name} " + f"— {s.tickets_created} ticket(s), {s.notes_written} note(s)" + ) + console.print() + + while True: + try: + raw = input("Select a student (number), or press Enter to show all: ").strip() + except (EOFError, KeyboardInterrupt): + console.print() + return None + + if raw == "": + return None + + if raw.isdigit() and 1 <= int(raw) <= len(matches): + return matches[int(raw) - 1] + + console.print(f"[red]Enter a number between 1 and {len(matches)}.[/red]") + + +def _pick_org(matches: list[str]) -> str | None: + """Prompt the user to select one institution from a list of matches.""" + console.print(f"\n[yellow]Found {len(matches)} institutions matching that name:[/yellow]\n") + for i, name in enumerate(matches, 1): + console.print(f" [cyan]{i}[/cyan]. {name}") + console.print() + + while True: + try: + raw = input("Select an institution (number), or press Enter to show all: ").strip() + except (EOFError, KeyboardInterrupt): + console.print() + return None + + if raw == "": + return None + + if raw.isdigit() and 1 <= int(raw) <= len(matches): + return matches[int(raw) - 1] + + console.print(f"[red]Enter a number between 1 and {len(matches)}.[/red]") + + +def display_org_report( + stats: dict[int, StudentStats], + org_filter: str, + sort_by: str = "activity", + student_filter: str | None = None, + show_graph: bool = False, +) -> None: + """Render the activity report for a single institution. + + Finds all unique ticket categories matching org_filter (substring, + case-insensitive). If multiple categories match, prompts the user to pick + one or show all. Then renders the same summary table as display_report(), + scoped to students whose tickets belong to the matched category/categories. + + If student_filter is also provided, further narrows to a specific student + within the institution and shows the detail view. + + Args: + stats: Full per-student stats from build_report(). + org_filter: Substring to match against ticket category names. + sort_by: Column to sort the summary table by. + student_filter: Optional student name substring for further drill-down. + show_graph: Whether to show the submission graph in detail view. + """ + # Collect all unique categories across the corpus + all_categories: list[str] = sorted( + {cat for s in stats.values() for cat in s.categories}, + key=str.lower, + ) + matching_cats = [c for c in all_categories if org_filter.lower() in c.lower()] + + if not matching_cats: + console.print(f"[yellow]No institution matching '{org_filter}' found.[/yellow]") + console.print(f"[dim]Known institutions: {', '.join(all_categories) or 'none'}[/dim]") + return + + # Disambiguate if multiple categories match + if len(matching_cats) == 1: + chosen_cats = matching_cats + else: + chosen = _pick_org(matching_cats) + chosen_cats = [chosen] if chosen is not None else matching_cats + + org_label = chosen_cats[0] if len(chosen_cats) == 1 else org_filter + + # Filter students who have tickets in any of the chosen categories + org_stats = {k: v for k, v in stats.items() if v.categories & set(chosen_cats)} + + if not org_stats: + console.print(f"[yellow]No students found for institution '{org_label}'.[/yellow]") + return + + all_dates = [ref["created_at"] for s in org_stats.values() for ref in s.created_tickets] + date_range = _format_date_range(all_dates) + date_part = f" · {date_range}" if date_range else "" + console.print( + Rule(f"[bold]{org_label}[/bold] [dim]({len(org_stats)} students{date_part})[/dim]") + ) + + if student_filter: + # Delegate to the normal student-filter path, scoped to this org + display_report( + org_stats, + sort_by=sort_by, + student_filter=student_filter, + show_graph=show_graph, + ) + else: + display_report(org_stats, sort_by=sort_by) + if show_graph: + console.print() + draw_org_graph(org_stats, org_label) + + +def display_student_detail(student: StudentStats, show_graph: bool = False) -> None: + """Render a detailed activity breakdown for a single student.""" + console.print(Rule(f"[cyan]{student.name}[/cyan]")) + escalated_str = ( + f" Escalated: [red]{student.escalated_tickets}[/red]" if student.escalated_tickets else "" + ) + console.print( + f" Tickets created: [green]{student.tickets_created}[/green]" + f"{escalated_str} " + f"Notes written: [yellow]{student.notes_written}[/yellow] " + f"Total activity: [bold]{student.total_activity}[/bold]" + ) + if student.projects: + console.print(f" Projects: [dim]{', '.join(sorted(student.projects))}[/dim]") + console.print() + + if student.created_tickets: + created = Table(title="Tickets Created", box=box.SIMPLE, show_header=True) + created.add_column("#", style="dim", no_wrap=True) + created.add_column("Summary", ratio=3) + created.add_column("Status", no_wrap=True) + created.add_column("Created", no_wrap=True) + created.add_column("Link", style="blue") + + for ref in sorted( + student.created_tickets, + key=lambda r: int(r["id"]) if r["id"].isdigit() else 0, + reverse=True, + ): + created.add_row( + ref["id"], + ref["summary"] or "—", + ref["status"] or "—", + ref["created_at"] or "—", + ref["url"], + ) + console.print(created) + + noted = student.noted_tickets + if noted: + noted_table = Table( + title="Tickets Commented On (not created by this student)", box=box.SIMPLE + ) + noted_table.add_column("#", style="dim", no_wrap=True) + noted_table.add_column("Summary", ratio=3) + noted_table.add_column("Status", no_wrap=True) + noted_table.add_column("Link", style="blue") + + for ref in sorted( + noted, key=lambda r: int(r["id"]) if r["id"].isdigit() else 0, reverse=True + ): + noted_table.add_row( + ref["id"], + ref["summary"] or "—", + ref["status"] or "—", + ref["url"], + ) + console.print(noted_table) + + if show_graph: + console.print() + draw_submission_graph(student) + + +def _plot_ticket_timeline(date_strs: list[str], title: str) -> None: + """Render a terminal line graph of ticket submissions over time. + + Accepts a flat list of YYYY-MM-DD strings (duplicates allowed — each + represents one ticket). Granularity is chosen automatically: + - ≤ 5 weeks → daily + - ≤ 12 months → weekly (Mon-anchored) + - > 12 months → monthly + + The graph width is capped to the terminal width minus a small margin. + """ + import plotext as plt + + parsed = sorted(date.fromisoformat(d) for d in date_strs) + span_days = (parsed[-1] - parsed[0]).days + + if span_days <= 35: + granularity = "day" + label_fmt = "%b %d" + bucket_fn = lambda d: d # noqa: E731 + elif span_days <= 365: + granularity = "week" + label_fmt = "%b %d" + bucket_fn = lambda d: d - timedelta(days=d.weekday()) # noqa: E731 + else: + granularity = "month" + label_fmt = "%b '%y" + bucket_fn = lambda d: d.replace(day=1) # noqa: E731 + + buckets: dict[date, int] = defaultdict(int) + for d in parsed: + buckets[bucket_fn(d)] += 1 + + # Fill gaps so the x-axis is contiguous + all_buckets = sorted(buckets) + first, last = all_buckets[0], all_buckets[-1] + if granularity == "day": + cursor, step = first, timedelta(days=1) + full_range: list[date] = [] + while cursor <= last: + full_range.append(cursor) + cursor += step + elif granularity == "week": + cursor, step = first, timedelta(weeks=1) + full_range = [] + while cursor <= last: + full_range.append(cursor) + cursor += step + else: + full_range = [] + y, m = first.year, first.month + while date(y, m, 1) <= last: + full_range.append(date(y, m, 1)) + m += 1 + if m > 12: + m, y = 1, y + 1 + + counts = [buckets.get(b, 0) for b in full_range] + labels = [b.strftime(label_fmt) for b in full_range] + # Use numeric x indices — passing formatted strings causes plotext to attempt + # date parsing which fails for short formats like "Jan 12". + x_indices = list(range(len(counts))) + + try: + term_width = os.get_terminal_size().columns + except OSError: + term_width = 100 + plot_width = max(60, min(term_width - 4, 160)) + + tick_step = max(1, len(labels) // (plot_width // 10)) + tick_positions = x_indices[::tick_step] + tick_labels = labels[::tick_step] + + plt.clf() + plt.plot_size(plot_width, 14) + plt.theme("dark") + plt.title(f"{title} ({granularity}ly)") + plt.xlabel(granularity.capitalize()) + plt.ylabel("Tickets") + plt.plot(x_indices, counts, marker="braille") + plt.xticks(tick_positions, tick_labels) + plt.show() + console.print( + f" [dim]{len(parsed)} ticket(s) · " + f"{full_range[0].strftime('%Y-%m-%d')} → {full_range[-1].strftime('%Y-%m-%d')} " + f"· {granularity}ly buckets[/dim]" + ) + + +def draw_submission_graph(student: StudentStats) -> None: + """Render a submission timeline graph for a single student.""" + dated = [ + ref["created_at"] + for ref in student.created_tickets + if ref.get("created_at") and len(ref["created_at"]) == 10 + ] + if not dated: + console.print("[yellow]No dated tickets — cannot draw graph.[/yellow]") + return + _plot_ticket_timeline(dated, f"Ticket Submissions — {student.name}") + + +def draw_org_graph(org_stats: dict[int, StudentStats], org_label: str) -> None: + """Render an aggregate submission timeline graph for all students in an org.""" + dated = [ + ref["created_at"] + for s in org_stats.values() + for ref in s.created_tickets + if ref.get("created_at") and len(ref["created_at"]) == 10 + ] + if not dated: + console.print("[yellow]No dated tickets — cannot draw graph.[/yellow]") + return + _plot_ticket_timeline(dated, f"Ticket Submissions — {org_label}") + + +def display_report( + stats: dict[int, StudentStats], + sort_by: str = "activity", + student_filter: str | None = None, + org_filter: str | None = None, + show_graph: bool = False, +) -> None: + """Render the student activity report. + + If org_filter is set, delegates to display_org_report() which scopes the + table to students from that institution (matched by ticket category). + If student_filter matches exactly one student, shows the detail view. + If it matches multiple, prompts the user to pick one (or show all). + With no filter, renders the full summary table. + """ + if org_filter: + display_org_report( + stats, + org_filter=org_filter, + sort_by=sort_by, + student_filter=student_filter, + show_graph=show_graph, + ) + return + + if student_filter: + matches = [(k, v) for k, v in stats.items() if student_filter.lower() in v.name.lower()] + + if not matches: + console.print(f"[yellow]No student matching '{student_filter}' found.[/yellow]") + return + + if len(matches) == 1: + display_student_detail(matches[0][1], show_graph=show_graph) + return + + # Multiple matches — prompt + chosen = _pick_student(matches) + if chosen is not None: + display_student_detail(chosen[1], show_graph=show_graph) + return + + # User pressed Enter — fall through and show all matches in summary table + stats = dict(matches) + + rows = list(stats.values()) + + if sort_by == "tickets": + rows.sort(key=lambda s: s.tickets_created, reverse=True) + elif sort_by == "notes": + rows.sort(key=lambda s: s.notes_written, reverse=True) + elif sort_by == "name": + rows.sort(key=lambda s: s.name.lower()) + else: + rows.sort(key=lambda s: s.total_activity, reverse=True) + + all_dates = [ref["created_at"] for s in rows for ref in s.created_tickets] + date_range = _format_date_range(all_dates) + title = f"Student Activity Report ({len(rows)} students)" + if date_range: + title += f" · {date_range}" + + table = Table( + title=title, + box=box.SIMPLE, + show_footer=True, + ) + table.add_column("Student", style="cyan", footer="TOTAL") + table.add_column( + "Tickets Created", + justify="right", + style="green", + footer=str(sum(s.tickets_created for s in rows)), + ) + table.add_column( + "Escalated", + justify="right", + style="red", + footer=str(sum(s.escalated_tickets for s in rows)), + ) + table.add_column( + "Notes Written", + justify="right", + style="yellow", + footer=str(sum(s.notes_written for s in rows)), + ) + table.add_column( + "Total Activity", + justify="right", + footer=str(sum(s.total_activity for s in rows)), + ) + table.add_column("Projects", style="dim") + + for s in rows: + projects_str = ", ".join(sorted(s.projects)) if s.projects else "—" + escalated_str = str(s.escalated_tickets) if s.escalated_tickets else "—" + table.add_row( + s.name, + str(s.tickets_created), + escalated_str, + str(s.notes_written), + str(s.total_activity), + projects_str, + ) + + console.print(table) + + +def main() -> None: + parser = argparse.ArgumentParser(description="PISCES Student Activity Report") + parser.add_argument( + "--input", + default=DEFAULT_INDEX, + help="Path to tickets_index.json (default: data/tickets/indexed/tickets_index.json)", + ) + parser.add_argument( + "--live", + action="store_true", + help="Fetch from Mantis REST API instead of the offline index", + ) + parser.add_argument( + "--project", + metavar="NAME", + help="Filter to tickets in projects matching NAME (substring, e.g. bonney-lake)", + ) + parser.add_argument( + "--sort", + choices=["activity", "tickets", "notes", "name"], + default="activity", + help="Sort order for the summary table (default: activity)", + ) + parser.add_argument( + "--student", + metavar="NAME", + help="Filter to a specific student by name (substring, case-insensitive)", + ) + parser.add_argument( + "--org", + metavar="NAME", + help="Filter to students from a specific institution by category name (substring, " + "case-insensitive, e.g. 'bellevue college')", + ) + parser.add_argument( + "--since", + metavar="YYYY-MM-DD", + help="Include only tickets created on or after this date", + ) + parser.add_argument( + "--until", + metavar="YYYY-MM-DD", + help="Include only tickets created on or before this date", + ) + parser.add_argument( + "--graph", + action="store_true", + help="Show a terminal line graph of ticket submissions over time " + "(requires --student or --org)", + ) + args = parser.parse_args() + + if args.graph and not args.student and not args.org: + parser.error("--graph requires --student or --org") + + since: date | None = None + until: date | None = None + if args.since: + try: + since = date.fromisoformat(args.since) + except ValueError: + parser.error(f"--since: invalid date '{args.since}' (expected YYYY-MM-DD)") + if args.until: + try: + until = date.fromisoformat(args.until) + except ValueError: + parser.error(f"--until: invalid date '{args.until}' (expected YYYY-MM-DD)") + + load_dotenv() + + from src.utils.dns import setup_dns + + setup_dns() + + if args.live: + tickets = _load_live(project_filter=args.project) + stats = build_report(_filter_by_date_range(tickets, since, until)) + else: + tickets = _load_offline(args.input) + stats = build_report( + _filter_by_date_range(tickets, since, until), project_filter=args.project + ) + + display_report( + stats, + sort_by=args.sort, + student_filter=args.student, + org_filter=args.org, + show_graph=args.graph, + ) + + +if __name__ == "__main__": + main() diff --git a/src/mantis/mantis_search.py b/src/mantis/mantis_search.py index d112f02..841ebd9 100755 --- a/src/mantis/mantis_search.py +++ b/src/mantis/mantis_search.py @@ -77,6 +77,59 @@ def sensor_to_project(sensor_val: str) -> str | None: # Fields that carry structured source/dest labels (not free-form notes) _LABEL_FIELDS = ("description", "steps_to_reproduce", "additional_information") +# Escalation detection — two-stage approach applied only to admin notes. +# +# Stage 1 (_CLIENT_CONTACT_RE): the handler directly contacted/informed the +# client (past or present tense). Any match whose 20-char prefix contains +# "will" is treated as a future intent and skipped. +# +# Stage 2 (_ESC_PAST_TO_CLIENT_RE): the handler wrote "escalated [this/it] to +# [the] client" in the past tense. Matches whose 20-char prefix contains +# "not" or "won't" are skipped (negated intent). +# +# Intentionally excluded: +# - bare "escalat*" words (privilege escalation, "no need to escalate", etc.) +# - future-tense client contact ("we will let the client know") +# - conditional escalation ("worth escalating", "if we see more, we'll…") +_CLIENT_CONTACT_RE = re.compile( + r"(?:" + r"(?:informed|notif(?:ied|y))\s+(?:the\s+)?client" + r"|let\s+(?:the\s+)?client\s+know" + r"|message\s+to\s+(?:the\s+)?client" + r"|customer\s+communication" + r"|client\s+respond(?:ed|s)" + r"|reported\s+(?:this\s+)?to\s+(?:the\s+)?client" + r"|reached\s+out\s+to\s+(?:the\s+)?client" + r"|contacted\s+(?:the\s+)?client" + r")", + re.I, +) +_ESC_PAST_TO_CLIENT_RE = re.compile( + r"\bescalated\s+(?:(?:this|it)\s+)?to\s+(?:the\s+)?client\b", + re.I, +) +_WILL_RE = re.compile(r"\bwill\b", re.I) +_NOT_RE = re.compile(r"\bnot\b|\bwon't\b", re.I) + + +def _note_is_escalation(note: dict) -> bool: + """Return True if an admin note confirms the ticket was escalated to the client.""" + text = note.get("text", "") + # Stage 1: client-contact phrases (skip future-tense occurrences) + for m in _CLIENT_CONTACT_RE.finditer(text): + prefix = text[max(0, m.start() - 20) : m.start()] + if _WILL_RE.search(prefix): + continue + return True + # Stage 2: past-tense "escalated [this/it] to [the] client" + for m in _ESC_PAST_TO_CLIENT_RE.finditer(text): + prefix = text[max(0, m.start() - 20) : m.start()] + if _NOT_RE.search(prefix): + continue + return True + return False + + _DASHBOARD_DOMAINS = {"kibana", "opensearch", "elastic"} _TI_DOMAINS = {"greynoise", "abuseipdb", "shodan", "virustotal"} @@ -241,6 +294,11 @@ def _normalize_issue( admin_note_count = sum(1 for n in notes if n["is_admin_note"]) + escalation_note = next( + (n for n in notes if n["is_admin_note"] and _note_is_escalation(n)), + None, + ) + return { "id": issue_id, "url": f"{api_url}/view.php?id={issue_id}", @@ -270,6 +328,8 @@ def _normalize_issue( "ti_links": ti_links, "note_count": len(notes), "admin_note_count": admin_note_count, + "is_escalated": escalation_note is not None, + "escalated_by": escalation_note["reporter"]["name"] if escalation_note else None, } diff --git a/src/mantis/student_activity.py b/src/mantis/student_activity.py index d6b33fb..847f112 100644 --- a/src/mantis/student_activity.py +++ b/src/mantis/student_activity.py @@ -18,6 +18,9 @@ python src/mantis/student_activity.py --project bonney-lake python src/mantis/student_activity.py --student alice python src/mantis/student_activity.py --student alice --graph + python src/mantis/student_activity.py --org 'bellevue college' + python src/mantis/student_activity.py --org 'bellevue college' --student alice + python src/mantis/student_activity.py --since 2025-01-01 --until 2025-04-30 python src/mantis/student_activity.py --input data/tickets/indexed/tickets_index.json """ @@ -45,6 +48,69 @@ console = Console() + +def _ordinal(n: int) -> str: + """Return an integer with its ordinal suffix: 1st, 2nd, 3rd, 4th…""" + if 11 <= (n % 100) <= 13: + suffix = "th" + else: + suffix = {1: "st", 2: "nd", 3: "rd"}.get(n % 10, "th") + return f"{n}{suffix}" + + +def _format_date_range(date_strs: list[str]) -> str: + """Format a collection of YYYY-MM-DD strings as a human-readable span. + + Returns strings like "Jan 1st – Apr 28th" for same-year ranges, or + "Jan 1st, 2024 – Apr 28th, 2025" when the range crosses calendar years. + Returns an empty string when no valid dates are present. + """ + valid = [d for d in date_strs if d and len(d) >= 10] + if not valid: + return "" + parsed = sorted(date.fromisoformat(d[:10]) for d in valid) + first, last = parsed[0], parsed[-1] + cross_year = first.year != last.year + + def _fmt(d: date) -> str: + base = f"{d.strftime('%b')} {_ordinal(d.day)}" + return f"{base}, {d.year}" if cross_year else base + + if first == last: + return _fmt(first) + return f"{_fmt(first)} – {_fmt(last)}" + + +def _filter_by_date_range( + tickets: list[dict], + since: date | None, + until: date | None, +) -> list[dict]: + """Return only tickets whose created_at falls within [since, until]. + + Tickets with no or unparseable created_at are kept so data is not silently lost. + """ + if not since and not until: + return tickets + result = [] + for t in tickets: + raw = (t.get("created_at") or "")[:10] + if not raw: + result.append(t) + continue + try: + d = date.fromisoformat(raw) + except ValueError: + result.append(t) + continue + if since and d < since: + continue + if until and d > until: + continue + result.append(t) + return result + + # Minimal ticket ref stored per student — id, summary, url, status, created_at _TicketRef = dict @@ -69,6 +135,9 @@ def noted_tickets(self) -> list[_TicketRef]: """Unique tickets this student commented on but did not create.""" return list(self._noted.values()) + escalated_tickets: int = 0 + categories: set[str] = field(default_factory=set) + @property def total_activity(self) -> int: return self.tickets_created + self.notes_written @@ -193,6 +262,11 @@ def build_report( stats[reporter_id].created_tickets.append(ref) if project: stats[reporter_id].projects.add(project) + if ticket.get("is_escalated"): + stats[reporter_id].escalated_tickets += 1 + category = ticket.get("category", "") + if category: + stats[reporter_id].categories.add(category) for note in ticket.get("notes", []): note_reporter = note.get("reporter", {}) @@ -239,11 +313,112 @@ def _pick_student(matches: list[tuple[int, StudentStats]]) -> tuple[int, Student console.print(f"[red]Enter a number between 1 and {len(matches)}.[/red]") +def _pick_org(matches: list[str]) -> str | None: + """Prompt the user to select one institution from a list of matches.""" + console.print(f"\n[yellow]Found {len(matches)} institutions matching that name:[/yellow]\n") + for i, name in enumerate(matches, 1): + console.print(f" [cyan]{i}[/cyan]. {name}") + console.print() + + while True: + try: + raw = input("Select an institution (number), or press Enter to show all: ").strip() + except (EOFError, KeyboardInterrupt): + console.print() + return None + + if raw == "": + return None + + if raw.isdigit() and 1 <= int(raw) <= len(matches): + return matches[int(raw) - 1] + + console.print(f"[red]Enter a number between 1 and {len(matches)}.[/red]") + + +def display_org_report( + stats: dict[int, StudentStats], + org_filter: str, + sort_by: str = "activity", + student_filter: str | None = None, + show_graph: bool = False, +) -> None: + """Render the activity report for a single institution. + + Finds all unique ticket categories matching org_filter (substring, + case-insensitive). If multiple categories match, prompts the user to pick + one or show all. Then renders the same summary table as display_report(), + scoped to students whose tickets belong to the matched category/categories. + + If student_filter is also provided, further narrows to a specific student + within the institution and shows the detail view. + + Args: + stats: Full per-student stats from build_report(). + org_filter: Substring to match against ticket category names. + sort_by: Column to sort the summary table by. + student_filter: Optional student name substring for further drill-down. + show_graph: Whether to show the submission graph in detail view. + """ + # Collect all unique categories across the corpus + all_categories: list[str] = sorted( + {cat for s in stats.values() for cat in s.categories}, + key=str.lower, + ) + matching_cats = [c for c in all_categories if org_filter.lower() in c.lower()] + + if not matching_cats: + console.print(f"[yellow]No institution matching '{org_filter}' found.[/yellow]") + console.print(f"[dim]Known institutions: {', '.join(all_categories) or 'none'}[/dim]") + return + + # Disambiguate if multiple categories match + if len(matching_cats) == 1: + chosen_cats = matching_cats + else: + chosen = _pick_org(matching_cats) + chosen_cats = [chosen] if chosen is not None else matching_cats + + org_label = chosen_cats[0] if len(chosen_cats) == 1 else org_filter + + # Filter students who have tickets in any of the chosen categories + org_stats = {k: v for k, v in stats.items() if v.categories & set(chosen_cats)} + + if not org_stats: + console.print(f"[yellow]No students found for institution '{org_label}'.[/yellow]") + return + + all_dates = [ref["created_at"] for s in org_stats.values() for ref in s.created_tickets] + date_range = _format_date_range(all_dates) + date_part = f" · {date_range}" if date_range else "" + console.print( + Rule(f"[bold]{org_label}[/bold] [dim]({len(org_stats)} students{date_part})[/dim]") + ) + + if student_filter: + # Delegate to the normal student-filter path, scoped to this org + display_report( + org_stats, + sort_by=sort_by, + student_filter=student_filter, + show_graph=show_graph, + ) + else: + display_report(org_stats, sort_by=sort_by) + if show_graph: + console.print() + draw_org_graph(org_stats, org_label) + + def display_student_detail(student: StudentStats, show_graph: bool = False) -> None: """Render a detailed activity breakdown for a single student.""" console.print(Rule(f"[cyan]{student.name}[/cyan]")) + escalated_str = ( + f" Escalated: [red]{student.escalated_tickets}[/red]" if student.escalated_tickets else "" + ) console.print( - f" Tickets created: [green]{student.tickets_created}[/green] " + f" Tickets created: [green]{student.tickets_created}[/green]" + f"{escalated_str} " f"Notes written: [yellow]{student.notes_written}[/yellow] " f"Total activity: [bold]{student.total_activity}[/bold]" ) @@ -299,32 +474,22 @@ def display_student_detail(student: StudentStats, show_graph: bool = False) -> N draw_submission_graph(student) -def draw_submission_graph(student: StudentStats) -> None: - """Render a terminal line graph of ticket submissions over time for one student. +def _plot_ticket_timeline(date_strs: list[str], title: str) -> None: + """Render a terminal line graph of ticket submissions over time. - Granularity is chosen automatically based on the date range: + Accepts a flat list of YYYY-MM-DD strings (duplicates allowed — each + represents one ticket). Granularity is chosen automatically: - ≤ 5 weeks → daily - ≤ 12 months → weekly (Mon-anchored) - > 12 months → monthly - The graph width is capped to the terminal width minus a small margin so it - fits comfortably on most screen sizes. + The graph width is capped to the terminal width minus a small margin. """ import plotext as plt - dated = [ - ref["created_at"] - for ref in student.created_tickets - if ref.get("created_at") and len(ref["created_at"]) == 10 - ] - if not dated: - console.print("[yellow]No dated tickets — cannot draw graph.[/yellow]") - return - - parsed = sorted(date.fromisoformat(d) for d in dated) + parsed = sorted(date.fromisoformat(d) for d in date_strs) span_days = (parsed[-1] - parsed[0]).days - # Choose bucket granularity if span_days <= 35: granularity = "day" label_fmt = "%b %d" @@ -338,7 +503,6 @@ def draw_submission_graph(student: StudentStats) -> None: label_fmt = "%b '%y" bucket_fn = lambda d: d.replace(day=1) # noqa: E731 - # Build ordered bucket counts buckets: dict[date, int] = defaultdict(int) for d in parsed: buckets[bucket_fn(d)] += 1 @@ -378,17 +542,15 @@ def draw_submission_graph(student: StudentStats) -> None: except OSError: term_width = 100 plot_width = max(60, min(term_width - 4, 160)) - plot_height = 14 - # Show one x-tick label every N buckets so they don't overlap tick_step = max(1, len(labels) // (plot_width // 10)) tick_positions = x_indices[::tick_step] tick_labels = labels[::tick_step] plt.clf() - plt.plot_size(plot_width, plot_height) + plt.plot_size(plot_width, 14) plt.theme("dark") - plt.title(f"Ticket Submissions — {student.name} ({granularity}ly)") + plt.title(f"{title} ({granularity}ly)") plt.xlabel(granularity.capitalize()) plt.ylabel("Tickets") plt.plot(x_indices, counts, marker="braille") @@ -401,18 +563,58 @@ def draw_submission_graph(student: StudentStats) -> None: ) +def draw_submission_graph(student: StudentStats) -> None: + """Render a submission timeline graph for a single student.""" + dated = [ + ref["created_at"] + for ref in student.created_tickets + if ref.get("created_at") and len(ref["created_at"]) == 10 + ] + if not dated: + console.print("[yellow]No dated tickets — cannot draw graph.[/yellow]") + return + _plot_ticket_timeline(dated, f"Ticket Submissions — {student.name}") + + +def draw_org_graph(org_stats: dict[int, StudentStats], org_label: str) -> None: + """Render an aggregate submission timeline graph for all students in an org.""" + dated = [ + ref["created_at"] + for s in org_stats.values() + for ref in s.created_tickets + if ref.get("created_at") and len(ref["created_at"]) == 10 + ] + if not dated: + console.print("[yellow]No dated tickets — cannot draw graph.[/yellow]") + return + _plot_ticket_timeline(dated, f"Ticket Submissions — {org_label}") + + def display_report( stats: dict[int, StudentStats], sort_by: str = "activity", student_filter: str | None = None, + org_filter: str | None = None, show_graph: bool = False, ) -> None: """Render the student activity report. + If org_filter is set, delegates to display_org_report() which scopes the + table to students from that institution (matched by ticket category). If student_filter matches exactly one student, shows the detail view. If it matches multiple, prompts the user to pick one (or show all). With no filter, renders the full summary table. """ + if org_filter: + display_org_report( + stats, + org_filter=org_filter, + sort_by=sort_by, + student_filter=student_filter, + show_graph=show_graph, + ) + return + if student_filter: matches = [(k, v) for k, v in stats.items() if student_filter.lower() in v.name.lower()] @@ -444,8 +646,14 @@ def display_report( else: rows.sort(key=lambda s: s.total_activity, reverse=True) + all_dates = [ref["created_at"] for s in rows for ref in s.created_tickets] + date_range = _format_date_range(all_dates) + title = f"Student Activity Report ({len(rows)} students)" + if date_range: + title += f" · {date_range}" + table = Table( - title=f"Student Activity Report ({len(rows)} students)", + title=title, box=box.SIMPLE, show_footer=True, ) @@ -456,6 +664,12 @@ def display_report( style="green", footer=str(sum(s.tickets_created for s in rows)), ) + table.add_column( + "Escalated", + justify="right", + style="red", + footer=str(sum(s.escalated_tickets for s in rows)), + ) table.add_column( "Notes Written", justify="right", @@ -471,9 +685,11 @@ def display_report( for s in rows: projects_str = ", ".join(sorted(s.projects)) if s.projects else "—" + escalated_str = str(s.escalated_tickets) if s.escalated_tickets else "—" table.add_row( s.name, str(s.tickets_created), + escalated_str, str(s.notes_written), str(s.total_activity), projects_str, @@ -510,15 +726,45 @@ def main() -> None: metavar="NAME", help="Filter to a specific student by name (substring, case-insensitive)", ) + parser.add_argument( + "--org", + metavar="NAME", + help="Filter to students from a specific institution by category name (substring, " + "case-insensitive, e.g. 'bellevue college')", + ) + parser.add_argument( + "--since", + metavar="YYYY-MM-DD", + help="Include only tickets created on or after this date", + ) + parser.add_argument( + "--until", + metavar="YYYY-MM-DD", + help="Include only tickets created on or before this date", + ) parser.add_argument( "--graph", action="store_true", - help="Show a terminal line graph of ticket submissions over time (requires --student)", + help="Show a terminal line graph of ticket submissions over time " + "(requires --student or --org)", ) args = parser.parse_args() - if args.graph and not args.student: - parser.error("--graph requires --student") + if args.graph and not args.student and not args.org: + parser.error("--graph requires --student or --org") + + since: date | None = None + until: date | None = None + if args.since: + try: + since = date.fromisoformat(args.since) + except ValueError: + parser.error(f"--since: invalid date '{args.since}' (expected YYYY-MM-DD)") + if args.until: + try: + until = date.fromisoformat(args.until) + except ValueError: + parser.error(f"--until: invalid date '{args.until}' (expected YYYY-MM-DD)") load_dotenv() @@ -528,12 +774,20 @@ def main() -> None: if args.live: tickets = _load_live(project_filter=args.project) - stats = build_report(tickets) + stats = build_report(_filter_by_date_range(tickets, since, until)) else: tickets = _load_offline(args.input) - stats = build_report(tickets, project_filter=args.project) + stats = build_report( + _filter_by_date_range(tickets, since, until), project_filter=args.project + ) - display_report(stats, sort_by=args.sort, student_filter=args.student, show_graph=args.graph) + display_report( + stats, + sort_by=args.sort, + student_filter=args.student, + org_filter=args.org, + show_graph=args.graph, + ) if __name__ == "__main__": From 8b44a2de409a75d9e651ce9c212b587106bc0813 Mon Sep 17 00:00:00 2001 From: liamadale Date: Wed, 29 Apr 2026 09:58:21 -0700 Subject: [PATCH 007/109] feat(mantis-explorer): add new Flask web app for student activity exploration New app at /mantis-explorer (port 5003 standalone via apps/mantis_explorer/run.py) that provides a browser-based view of the data from student_activity/activity_report. Key capabilities: - Institution overview table with ticket counts, escalation counts, and date ranges - Per-institution student breakdown with sortable activity table - Per-student slide panel showing created tickets (with escalated row tinting and red exclamation icon) and notes, plus a ticket detail view with an "Escalated" badge in the meta row when is_escalated is true - Resizable ticket slide panel with drag-to-resize handle; width persisted to localStorage across page loads - Charts: submission timeline, org bar chart, and escalation breakdown - Date-range filtering propagated from the URL query string - Dark SOC-analyst CSS theme consistent with the rest of the PISCES UI (me.css) --- apps/mantis_explorer/__init__.py | 0 apps/mantis_explorer/app.py | 211 ++++ apps/mantis_explorer/data.py | 311 +++++ apps/mantis_explorer/run.py | 10 + apps/mantis_explorer/static/me.css | 1108 +++++++++++++++++ apps/mantis_explorer/static/pisces-logo.ico | Bin 0 -> 152126 bytes apps/mantis_explorer/static/pisces-logo.png | Bin 0 -> 21465 bytes apps/mantis_explorer/templates/base.html | 315 +++++ apps/mantis_explorer/templates/index.html | 155 +++ apps/mantis_explorer/templates/org.html | 129 ++ .../templates/partials/chart_escalation.html | 59 + .../templates/partials/chart_orgs_bar.html | 38 + .../templates/partials/chart_timeline.html | 45 + .../templates/partials/org_stat_row.html | 24 + .../templates/partials/org_table.html | 18 + .../templates/partials/org_timeline.html | 39 + .../templates/partials/stat_row.html | 28 + .../templates/partials/student_panel.html | 110 ++ .../templates/partials/student_rows.html | 20 + .../templates/partials/ticket_detail.html | 201 +++ 20 files changed, 2821 insertions(+) create mode 100644 apps/mantis_explorer/__init__.py create mode 100644 apps/mantis_explorer/app.py create mode 100644 apps/mantis_explorer/data.py create mode 100644 apps/mantis_explorer/run.py create mode 100644 apps/mantis_explorer/static/me.css create mode 100644 apps/mantis_explorer/static/pisces-logo.ico create mode 100644 apps/mantis_explorer/static/pisces-logo.png create mode 100644 apps/mantis_explorer/templates/base.html create mode 100644 apps/mantis_explorer/templates/index.html create mode 100644 apps/mantis_explorer/templates/org.html create mode 100644 apps/mantis_explorer/templates/partials/chart_escalation.html create mode 100644 apps/mantis_explorer/templates/partials/chart_orgs_bar.html create mode 100644 apps/mantis_explorer/templates/partials/chart_timeline.html create mode 100644 apps/mantis_explorer/templates/partials/org_stat_row.html create mode 100644 apps/mantis_explorer/templates/partials/org_table.html create mode 100644 apps/mantis_explorer/templates/partials/org_timeline.html create mode 100644 apps/mantis_explorer/templates/partials/stat_row.html create mode 100644 apps/mantis_explorer/templates/partials/student_panel.html create mode 100644 apps/mantis_explorer/templates/partials/student_rows.html create mode 100644 apps/mantis_explorer/templates/partials/ticket_detail.html diff --git a/apps/mantis_explorer/__init__.py b/apps/mantis_explorer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/mantis_explorer/app.py b/apps/mantis_explorer/app.py new file mode 100644 index 0000000..8887d46 --- /dev/null +++ b/apps/mantis_explorer/app.py @@ -0,0 +1,211 @@ +"""Flask application factory and routes for Mantis Explorer.""" + +from flask import Flask, abort, render_template, request + +from apps.mantis_explorer.data import ( + SLUG_TO_ORG, + TICKETS_BY_ID, + compute_escalation_data, + compute_global_stats, + compute_org_report, + compute_org_rows, + compute_org_stats, + compute_timeline_data, + get_report, + parse_date_params, + sort_students, +) + + +def create_app() -> Flask: + """Create and configure the Mantis Explorer Flask app.""" + app = Flask(__name__, static_folder="static", template_folder="templates") + + @app.context_processor + def inject_globals() -> dict: + return {"script_name": request.environ.get("SCRIPT_NAME", "")} + + # ------------------------------------------------------------------ + # GET / — Overview page (skeleton; HTMX regions load on trigger) + # ------------------------------------------------------------------ + @app.route("/") + def index(): + since = request.args.get("since", "") + until = request.args.get("until", "") + return render_template("index.html", since=since, until=until) + + # ------------------------------------------------------------------ + # GET /api/stat-row — HTMX: global summary chips + # ------------------------------------------------------------------ + @app.route("/api/stat-row") + def api_stat_row(): + since, until = parse_date_params(request.args) + report = get_report(since, until) + stats = compute_global_stats(report) + return render_template("partials/stat_row.html", stats=stats) + + # ------------------------------------------------------------------ + # GET /api/org-table — HTMX: institution table tbody + # ------------------------------------------------------------------ + @app.route("/api/org-table") + def api_org_table(): + since, until = parse_date_params(request.args) + q = request.args.get("q", "").strip() + sort = request.args.get("sort", "tickets") + order = request.args.get("order", "desc") + report = get_report(since, until) + rows = compute_org_rows(report, q=q, sort=sort, order=order) + return render_template( + "partials/org_table.html", + rows=rows, + sort=sort, + order=order, + since=since, + until=until, + ) + + # ------------------------------------------------------------------ + # GET /api/chart/orgs-bar — HTMX: horizontal bar chart + # ------------------------------------------------------------------ + @app.route("/api/chart/orgs-bar") + def api_chart_orgs_bar(): + since, until = parse_date_params(request.args) + report = get_report(since, until) + rows = compute_org_rows(report) + return render_template("partials/chart_orgs_bar.html", rows=rows) + + # ------------------------------------------------------------------ + # GET /api/chart/timeline — HTMX: global area timeline + # ------------------------------------------------------------------ + @app.route("/api/chart/timeline") + def api_chart_timeline(): + since, until = parse_date_params(request.args) + report = get_report(since, until) + timeline = compute_timeline_data(report) + return render_template("partials/chart_timeline.html", timeline=timeline) + + # ------------------------------------------------------------------ + # GET /api/chart/escalation — HTMX: escalation grouped bar + # ------------------------------------------------------------------ + @app.route("/api/chart/escalation") + def api_chart_escalation(): + since, until = parse_date_params(request.args) + report = get_report(since, until) + esc = compute_escalation_data(report) + return render_template("partials/chart_escalation.html", esc=esc) + + # ------------------------------------------------------------------ + # GET /org/ — Org detail page + # ------------------------------------------------------------------ + @app.route("/org/") + def org_detail(slug: str): + org_name = SLUG_TO_ORG.get(slug) + if not org_name: + abort(404) + since = request.args.get("since", "") + until = request.args.get("until", "") + return render_template( + "org.html", + org_name=org_name, + slug=slug, + since=since, + until=until, + ) + + # ------------------------------------------------------------------ + # GET /api/org//stat-row — HTMX: org summary chips + # ------------------------------------------------------------------ + @app.route("/api/org//stat-row") + def api_org_stat_row(slug: str): + org_name = SLUG_TO_ORG.get(slug) + if not org_name: + abort(404) + since, until = parse_date_params(request.args) + report = get_report(since, until) + org_report = compute_org_report(report, org_name) + stats = compute_org_stats(org_report) + return render_template("partials/org_stat_row.html", stats=stats, org_name=org_name) + + # ------------------------------------------------------------------ + # GET /api/org//timeline — HTMX: org submission timeline + # ------------------------------------------------------------------ + @app.route("/api/org//timeline") + def api_org_timeline(slug: str): + org_name = SLUG_TO_ORG.get(slug) + if not org_name: + abort(404) + since, until = parse_date_params(request.args) + report = get_report(since, until) + org_report = compute_org_report(report, org_name) + timeline = compute_timeline_data(org_report) + return render_template("partials/org_timeline.html", timeline=timeline, org_name=org_name) + + # ------------------------------------------------------------------ + # GET /api/org//students — HTMX: student table tbody + # ------------------------------------------------------------------ + @app.route("/api/org//students") + def api_org_students(slug: str): + org_name = SLUG_TO_ORG.get(slug) + if not org_name: + abort(404) + since, until = parse_date_params(request.args) + q = request.args.get("q", "").strip() + sort = request.args.get("sort", "activity") + order = request.args.get("order", "desc") + report = get_report(since, until) + org_report = compute_org_report(report, org_name) + from src.mantis.activity_report import _format_date_range + + rows = [] + for sid, s in sort_students(org_report, sort=sort, order=order, q=q): + dates = [ref["created_at"] for ref in s.created_tickets if ref.get("created_at")] + rows.append( + { + "id": sid, + "name": s.name, + "tickets_created": s.tickets_created, + "escalated_tickets": s.escalated_tickets, + "notes_written": s.notes_written, + "total_activity": s.total_activity, + "date_range": _format_date_range(dates), + } + ) + return render_template( + "partials/student_rows.html", + rows=rows, + sort=sort, + order=order, + slug=slug, + since=since, + until=until, + ) + + # ------------------------------------------------------------------ + # GET /api/ticket/ — HTMX: full ticket detail for slide panel + # ------------------------------------------------------------------ + @app.route("/api/ticket/") + def api_ticket_detail(tid: str): + t = TICKETS_BY_ID.get(str(tid)) + if not t: + return ( + "

Ticket not found.

", + 404, + ) + return render_template("partials/ticket_detail.html", t=t) + + # ------------------------------------------------------------------ + # GET /api/student//panel — HTMX: student detail slide panel + # ------------------------------------------------------------------ + @app.route("/api/student//panel") + def api_student_panel(student_id: int): + since, until = parse_date_params(request.args) + report = get_report(since, until) + student = report.get(student_id) + if not student: + return ( + "

Student not found.

", + 404, + ) + return render_template("partials/student_panel.html", student=student) + + return app diff --git a/apps/mantis_explorer/data.py b/apps/mantis_explorer/data.py new file mode 100644 index 0000000..7d4601f --- /dev/null +++ b/apps/mantis_explorer/data.py @@ -0,0 +1,311 @@ +"""Startup data loading and per-request computation for Mantis Explorer.""" + +import os +import re +from collections import defaultdict +from datetime import date, timedelta +from functools import lru_cache + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_REPO = os.path.dirname(os.path.dirname(_HERE)) + +# --------------------------------------------------------------------------- +# Raw tickets — reuse mantis_web's already-parsed list in hub mode so we +# don't parse the JSON twice. Falls back to loading independently when +# running standalone (mantis_web.data will load itself in that process). +# --------------------------------------------------------------------------- +from apps.mantis_web.data import _raw_tickets as RAW_TICKETS # noqa: E402 +from src.mantis.activity_report import ( # noqa: E402 + StudentStats, + _filter_by_date_range, + _format_date_range, + build_report, +) + +# --------------------------------------------------------------------------- +# Slug maps — built once at startup from all category values in the corpus +# --------------------------------------------------------------------------- + + +def _slugify(name: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-") + + +def _build_slug_maps(tickets: list[dict]) -> tuple[dict[str, str], dict[str, str]]: + """Return (slug→org, org→slug) for every unique category value.""" + orgs: set[str] = set() + for t in tickets: + cat = t.get("category", "") + if cat: + orgs.add(cat) + + slug_to_org: dict[str, str] = {} + org_to_slug: dict[str, str] = {} + for org in sorted(orgs): + slug = _slugify(org) + # Collision avoidance + if slug in slug_to_org: + n = 2 + while f"{slug}-{n}" in slug_to_org: + n += 1 + slug = f"{slug}-{n}" + slug_to_org[slug] = org + org_to_slug[org] = slug + return slug_to_org, org_to_slug + + +SLUG_TO_ORG, ORG_TO_SLUG = _build_slug_maps(RAW_TICKETS) + +# --------------------------------------------------------------------------- +# Per-request computation with caching +# --------------------------------------------------------------------------- + + +def parse_date_params(args: object) -> tuple[str, str]: + """Extract and validate since/until from request.args. + + Returns ("", "") for absent or invalid values so callers can pass + strings directly to get_report() without additional validation. + """ + + def _safe(key: str) -> str: + v = getattr(args, "get", lambda k, d="": d)(key, "").strip() + try: + date.fromisoformat(v) + return v + except ValueError: + return "" + + return _safe("since"), _safe("until") + + +@lru_cache(maxsize=32) +def get_report(since_str: str, until_str: str) -> dict[int, StudentStats]: + """Filter RAW_TICKETS by date range and run build_report(). Cached by (since, until).""" + since = date.fromisoformat(since_str) if since_str else None + until = date.fromisoformat(until_str) if until_str else None + filtered = _filter_by_date_range(RAW_TICKETS, since, until) + return build_report(filtered) + + +# --------------------------------------------------------------------------- +# Aggregation helpers +# --------------------------------------------------------------------------- + + +def compute_global_stats(report: dict[int, StudentStats]) -> dict: + """Summary numbers for the overview stat row.""" + all_orgs: set[str] = set() + total_tickets = 0 + total_escalations = 0 + total_notes = 0 + all_dates: list[str] = [] + + for s in report.values(): + all_orgs.update(s.categories) + total_tickets += s.tickets_created + total_escalations += s.escalated_tickets + total_notes += s.notes_written + all_dates.extend(ref["created_at"] for ref in s.created_tickets if ref.get("created_at")) + + return { + "institution_count": len(all_orgs), + "student_count": len(report), + "ticket_count": total_tickets, + "escalation_count": total_escalations, + "note_count": total_notes, + "date_range": _format_date_range(all_dates), + } + + +def _org_aggregate(report: dict[int, StudentStats], q: str = "") -> dict[str, dict]: + """Build a per-org aggregation dict keyed by raw category name.""" + agg: dict[str, dict] = {} + for sid, s in report.items(): + for cat in s.categories: + if q and q.lower() not in cat.lower(): + continue + if cat not in agg: + agg[cat] = { + "name": cat, + "slug": ORG_TO_SLUG.get(cat, _slugify(cat)), + "student_ids": set(), + "ticket_count": 0, + "escalation_count": 0, + "note_count": 0, + "dates": [], + } + d = agg[cat] + d["student_ids"].add(sid) + d["ticket_count"] += s.tickets_created + d["escalation_count"] += s.escalated_tickets + d["note_count"] += s.notes_written + d["dates"].extend( + ref["created_at"] for ref in s.created_tickets if ref.get("created_at") + ) + return agg + + +def compute_org_rows( + report: dict[int, StudentStats], + q: str = "", + sort: str = "tickets", + order: str = "desc", +) -> list[dict]: + """Return institution table rows, sorted and optionally name-filtered.""" + agg = _org_aggregate(report, q=q) + rows = [] + for d in agg.values(): + rows.append( + { + "name": d["name"], + "slug": d["slug"], + "student_count": len(d["student_ids"]), + "ticket_count": d["ticket_count"], + "escalation_count": d["escalation_count"], + "note_count": d["note_count"], + "date_range": _format_date_range(d["dates"]), + } + ) + + key_fn = { + "name": lambda r: r["name"].lower(), + "students": lambda r: r["student_count"], + "tickets": lambda r: r["ticket_count"], + "escalated": lambda r: r["escalation_count"], + "notes": lambda r: r["note_count"], + }.get(sort, lambda r: r["ticket_count"]) + rows.sort(key=key_fn, reverse=(order == "desc")) + return rows + + +def compute_timeline_data(report: dict[int, StudentStats]) -> dict: + """Auto-granularity timeline data for ECharts. + + Mirrors the _plot_ticket_timeline logic from activity_report.py but + returns a JSON-serialisable dict instead of drawing to the terminal. + + Returns: + {"labels": [...], "counts": [...], "granularity": "daily"|"weekly"|"monthly"} + or {"labels": [], "counts": [], "granularity": "daily"} when no dated tickets. + """ + raw_dates: list[str] = [ + ref["created_at"] + for s in report.values() + for ref in s.created_tickets + if ref.get("created_at") and len(ref["created_at"]) >= 10 + ] + if not raw_dates: + return {"labels": [], "counts": [], "granularity": "daily"} + + parsed = sorted(date.fromisoformat(d[:10]) for d in raw_dates) + span_days = (parsed[-1] - parsed[0]).days + + if span_days <= 35: + granularity = "daily" + label_fmt = "%b %d" + bucket_fn = lambda d: d # noqa: E731 + elif span_days <= 365: + granularity = "weekly" + label_fmt = "%b %d" + bucket_fn = lambda d: d - timedelta(days=d.weekday()) # noqa: E731 + else: + granularity = "monthly" + label_fmt = "%b '%y" + bucket_fn = lambda d: d.replace(day=1) # noqa: E731 + + buckets: dict[date, int] = defaultdict(int) + for d in parsed: + buckets[bucket_fn(d)] += 1 + + # Fill gaps + all_buckets = sorted(buckets) + first, last = all_buckets[0], all_buckets[-1] + full_range: list[date] = [] + if granularity == "daily": + cursor = first + while cursor <= last: + full_range.append(cursor) + cursor += timedelta(days=1) + elif granularity == "weekly": + cursor = first + while cursor <= last: + full_range.append(cursor) + cursor += timedelta(weeks=1) + else: + y, m = first.year, first.month + while date(y, m, 1) <= last: + full_range.append(date(y, m, 1)) + m += 1 + if m > 12: + m, y = 1, y + 1 + + return { + "labels": [b.strftime(label_fmt) for b in full_range], + "counts": [buckets.get(b, 0) for b in full_range], + "granularity": granularity, + } + + +def compute_escalation_data(report: dict[int, StudentStats]) -> dict: + """Per-org escalation data for the grouped bar chart. + + Returns: + {"orgs": [...], "totals": [...], "escalated": [...]} + sorted by total tickets descending. + """ + agg = _org_aggregate(report) + rows = sorted(agg.values(), key=lambda d: d["ticket_count"], reverse=True) + return { + "orgs": [d["name"] for d in rows], + "totals": [d["ticket_count"] for d in rows], + "escalated": [d["escalation_count"] for d in rows], + } + + +def compute_org_report(report: dict[int, StudentStats], org_name: str) -> dict[int, StudentStats]: + """Filter report to students who have at least one ticket in org_name's category.""" + return {k: v for k, v in report.items() if org_name in v.categories} + + +def compute_org_stats(org_report: dict[int, StudentStats]) -> dict: + """Summary numbers for the org-detail stat row.""" + total_tickets = sum(s.tickets_created for s in org_report.values()) + total_escalations = sum(s.escalated_tickets for s in org_report.values()) + total_notes = sum(s.notes_written for s in org_report.values()) + all_dates = [ + ref["created_at"] + for s in org_report.values() + for ref in s.created_tickets + if ref.get("created_at") + ] + return { + "student_count": len(org_report), + "ticket_count": total_tickets, + "escalation_count": total_escalations, + "note_count": total_notes, + "date_range": _format_date_range(all_dates), + } + + +def sort_students( + org_report: dict[int, StudentStats], + sort: str = "activity", + order: str = "desc", + q: str = "", +) -> list[tuple[int, StudentStats]]: + """Return (reporter_id, StudentStats) pairs sorted and optionally name-filtered.""" + pairs = list(org_report.items()) + if q: + pairs = [(k, v) for k, v in pairs if q.lower() in v.name.lower()] + + key_fn = { + "name": lambda p: p[1].name.lower(), + "tickets": lambda p: p[1].tickets_created, + "escalated": lambda p: p[1].escalated_tickets, + "notes": lambda p: p[1].notes_written, + "activity": lambda p: p[1].total_activity, + }.get(sort, lambda p: p[1].total_activity) + + pairs.sort(key=key_fn, reverse=(order != "asc" and sort != "name")) + return pairs diff --git a/apps/mantis_explorer/run.py b/apps/mantis_explorer/run.py new file mode 100644 index 0000000..36ca070 --- /dev/null +++ b/apps/mantis_explorer/run.py @@ -0,0 +1,10 @@ +"""Standalone launcher for Mantis Explorer on port 5005.""" + +from apps.mantis_explorer.app import create_app +from apps.mantis_explorer.data import SLUG_TO_ORG, get_report + +if __name__ == "__main__": + report = get_report("", "") + print(f"Mantis Explorer: {len(SLUG_TO_ORG)} institutions, {len(report)} students") + app = create_app() + app.run(host="0.0.0.0", port=5005, debug=False, threaded=True) diff --git a/apps/mantis_explorer/static/me.css b/apps/mantis_explorer/static/me.css new file mode 100644 index 0000000..6efa9a7 --- /dev/null +++ b/apps/mantis_explorer/static/me.css @@ -0,0 +1,1108 @@ +/* ================================================================ + Mantis Explorer — Material Design 3 Dark/Light Theme + Self-contained; does not depend on pisces.css + ================================================================ */ + +/* ── 1. Design Tokens ─────────────────────────────────── */ + +[data-theme="dark"] { + --surface: #0d0f14; + --surface-container-low: #161a24; + --surface-container: #1a1f2e; + --surface-container-high: #1e2433; + --surface-container-highest:#232840; + --outline: #2a3045; + --outline-dim: #1e2233; + --on-surface: #c8ccd8; + --on-surface-dim: #5a6278; + --primary: #4f8ef7; + --primary-dim: #3a7ae8; + --secondary: #7ec8e3; + --tertiary: #bc8cff; + --green: #3fb950; --yellow: #d29922; + --red: #f85149; --orange: #e3763c; --purple: #bc8cff; + --state-hover: rgba(79,142,247,0.08); + --badge-green-bg: rgba(63,185,80,0.2); + --badge-red-bg: rgba(248,81,73,0.2); + --badge-yellow-bg: rgba(210,153,34,0.2); + --badge-blue-bg: rgba(79,142,247,0.2); + --badge-gray-bg: rgba(90,98,120,0.3); + --focus-ring: rgba(79,142,247,0.25); + --panel-shadow: rgba(0,0,0,0.45); +} + +[data-theme="light"] { + --surface: #f8f9fc; + --surface-container-low: #f0f1f5; + --surface-container: #e8eaef; + --surface-container-high: #e1e3e9; + --surface-container-highest:#d8dbe2; + --outline: #c4c8d4; + --outline-dim: #dcdfe6; + --on-surface: #1a1c24; + --on-surface-dim: #5c6070; + --primary: #2563eb; + --primary-dim: #1d4fd8; + --secondary: #0d7490; + --tertiary: #7c3aed; + --green: #16a34a; --yellow: #ca8a04; + --red: #dc2626; --orange: #ea580c; --purple: #7c3aed; + --state-hover: rgba(37,99,235,0.06); + --badge-green-bg: rgba(22,163,74,0.12); + --badge-red-bg: rgba(220,38,38,0.12); + --badge-yellow-bg: rgba(202,138,4,0.12); + --badge-blue-bg: rgba(37,99,235,0.10); + --badge-gray-bg: rgba(92,96,112,0.12); + --focus-ring: rgba(37,99,235,0.25); + --panel-shadow: rgba(0,0,0,0.15); +} + +:root { + --radius-xs: 4px; + --radius-sm: 8px; + --radius-md: 12px; + --radius-pill: 28px; + + --font: "JetBrains Mono","Fira Mono","Courier New",monospace; + --sans: system-ui,-apple-system,sans-serif; +} + + +/* ── 2. Reset & Base ──────────────────────────────────── */ + +*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; } + +html { font-size: 14px; } + +body { + background: var(--surface); + color: var(--on-surface); + font-family: var(--font); + min-height: 100vh; + display: flex; + flex-direction: column; + line-height: 1.5; +} + +a { color: var(--primary); text-decoration: none; } +a:hover { text-decoration: underline; } + + +/* ── 3. Nav ───────────────────────────────────────────── */ + +nav { + flex-shrink: 0; + background: var(--surface-container-low); + border-bottom: 1px solid var(--outline); + padding: 0 1.5rem; + display: flex; + align-items: center; + gap: 0.5rem; + height: 48px; + position: sticky; + top: 0; + z-index: 100; +} + +nav .brand-logo { + height: 28px; + width: auto; + vertical-align: middle; + margin-right: 8px; +} + +nav a { + font-family: var(--sans); + color: var(--on-surface-dim); + text-decoration: none; + font-size: 0.82rem; + padding: 5px 10px; + border-radius: var(--radius-pill); + transition: color 0.15s, background 0.15s; + white-space: nowrap; + display: inline-flex; + align-items: center; + gap: 5px; +} + +nav a:hover { + color: var(--primary); + background: var(--state-hover); + text-decoration: none; +} + +nav .brand { + font-family: var(--sans); + font-weight: 700; + font-size: 1.1rem; + color: var(--secondary); + letter-spacing: 0.06em; + text-decoration: none; + white-space: nowrap; + margin-right: 0.5rem; +} + +nav .brand-sub { + font-weight: 400; + font-size: 0.8rem; + opacity: 0.6; + letter-spacing: 0; + margin-left: 4px; +} + +nav .spacer { flex: 1; } + +.btn-theme { + background: transparent; + border: none; + color: var(--on-surface-dim); + width: 36px; height: 36px; + border-radius: 50%; + cursor: pointer; + display: inline-flex; + align-items: center; + justify-content: center; + font-size: 1rem; + transition: color 0.15s; +} +.btn-theme:hover { color: var(--primary); } + +.btn-icon { + background: transparent; + border: none; + color: var(--on-surface-dim); + width: 36px; height: 36px; + border-radius: 50%; + cursor: pointer; + display: inline-flex; + align-items: center; + justify-content: center; + font-size: 1rem; + transition: color 0.15s; + text-decoration: none; +} +.btn-icon:hover { color: var(--primary); text-decoration: none; } + + +/* ── 4. Main & Page Wrapper ───────────────────────────── */ + +main { + flex: 1; + padding: 1rem 1.5rem 2rem; + display: flex; + flex-direction: column; + gap: 0.75rem; + min-width: 0; +} + + +/* ── 5. Breadcrumb ────────────────────────────────────── */ + +.me-breadcrumb { + display: flex; + align-items: center; + gap: 0.4rem; + font-family: var(--sans); + font-size: 0.82rem; + color: var(--on-surface-dim); + flex-wrap: wrap; +} + +.me-breadcrumb a { + color: var(--primary); + display: inline-flex; + align-items: center; + gap: 4px; + padding: 2px 6px; + border-radius: var(--radius-xs); + transition: background 0.15s; +} +.me-breadcrumb a:hover { background: var(--state-hover); text-decoration: none; } + +.me-breadcrumb .sep { color: var(--outline); font-size: 0.9rem; } + +.me-breadcrumb .current { + color: var(--on-surface); + font-weight: 600; +} + + +/* ── 6. Filter Bar ────────────────────────────────────── */ + +.me-filter-bar { + display: flex; + flex-wrap: wrap; + gap: 0.5rem; + align-items: center; + background: var(--surface-container-low); + border: 1px solid var(--outline); + border-radius: var(--radius-md); + padding: 0.6rem 1rem; +} + +.me-filter-bar label { + font-family: var(--sans); + font-size: 0.78rem; + color: var(--on-surface-dim); + white-space: nowrap; +} + +.me-filter-bar input[type="date"], +.me-filter-bar input[type="text"] { + background: var(--surface-container-high); + border: 1px solid var(--outline); + color: var(--on-surface); + padding: 3px 8px; + border-radius: var(--radius-xs); + font-family: var(--font); + font-size: 0.82rem; + height: 30px; + outline: none; + transition: border-color 0.15s, box-shadow 0.15s; + color-scheme: dark; +} +[data-theme="light"] .me-filter-bar input[type="date"], +[data-theme="light"] .me-filter-bar input[type="text"] { + color-scheme: light; +} + +.me-filter-bar input:focus { + border-color: var(--primary); + box-shadow: 0 0 0 2px var(--focus-ring); +} + +.me-filter-bar input[type="text"] { min-width: 180px; } + +.filter-sep { + width: 1px; height: 18px; + background: var(--outline); + margin: 0 2px; +} + +.me-filter-bar .spacer { flex: 1; } + + +/* ── 7. Stat Row ──────────────────────────────────────── */ + +.me-stat-row { + display: flex; + flex-wrap: wrap; + gap: 0.5rem; +} + +.me-stat-chip { + display: flex; + flex-direction: column; + gap: 1px; + background: var(--surface-container-low); + border: 1px solid var(--outline); + border-radius: var(--radius-md); + padding: 0.45rem 1rem; + min-width: 110px; +} + +.me-stat-chip .chip-value { + font-family: var(--sans); + font-size: 1.3rem; + font-weight: 700; + color: var(--on-surface); + line-height: 1.2; +} + +.me-stat-chip .chip-label { + font-family: var(--sans); + font-size: 0.72rem; + color: var(--on-surface-dim); + text-transform: uppercase; + letter-spacing: 0.06em; +} + +.me-stat-chip.accent-red .chip-value { color: var(--red); } +.me-stat-chip.accent-blue .chip-value { color: var(--primary); } +.me-stat-chip.accent-green .chip-value { color: var(--green); } + +.me-stat-date { + font-family: var(--sans); + font-size: 0.82rem; + color: var(--on-surface-dim); + display: flex; + align-items: center; + gap: 6px; + padding: 0 0.25rem; + align-self: center; +} + + +/* ── 8. Section Card (table / chart wrapper) ──────────── */ + +.me-section { + background: var(--surface-container-low); + border: 1px solid var(--outline); + border-radius: var(--radius-md); + display: flex; + flex-direction: column; + min-height: 0; +} + +.me-section-header { + display: flex; + align-items: center; + gap: 0.75rem; + padding: 0.6rem 1rem; + background: var(--surface-container-high); + border-bottom: 1px solid var(--outline); + border-radius: var(--radius-md) var(--radius-md) 0 0; + flex-shrink: 0; + flex-wrap: wrap; +} + +.me-section-header h2 { + font-family: var(--sans); + font-size: 0.9rem; + font-weight: 700; + color: var(--secondary); + text-transform: uppercase; + letter-spacing: 0.06em; + margin: 0; + white-space: nowrap; +} + +.me-section-header .count-note { + font-family: var(--sans); + font-size: 0.78rem; + color: var(--on-surface-dim); +} + +.section-spacer { flex: 1; } + +/* Filter input inside section header */ +.me-section-header input[type="text"] { + background: var(--surface-container-highest); + border: 1px solid var(--outline); + color: var(--on-surface); + padding: 3px 8px; + border-radius: var(--radius-xs); + font-family: var(--font); + font-size: 0.82rem; + height: 28px; + outline: none; + transition: border-color 0.15s, box-shadow 0.15s; + width: 180px; +} +.me-section-header input[type="text"]:focus { + border-color: var(--primary); + box-shadow: 0 0 0 2px var(--focus-ring); +} + +.section-table-wrap { + overflow-x: auto; +} + + +/* ── 9. Tables ────────────────────────────────────────── */ + +table { + table-layout: auto; + width: 100%; + border-collapse: collapse; + font-size: 0.82rem; +} + +thead th { + position: sticky; + top: 0; + z-index: 5; + background: var(--surface-container-high); + color: var(--on-surface-dim); + font-family: var(--sans); + font-weight: 600; + padding: 7px 12px; + text-align: left; + border-bottom: 1px solid var(--outline); + white-space: nowrap; + user-select: none; +} + +thead th.sortable { cursor: pointer; } +thead th.sortable:hover { color: var(--primary); } +thead th.sort-asc::after { content: " ↑"; color: var(--primary); } +thead th.sort-desc::after { content: " ↓"; color: var(--primary); } + +tbody tr { + border-bottom: 1px solid var(--outline-dim); + transition: background 0.1s; + cursor: pointer; +} +tbody tr:last-child { border-bottom: none; } +tbody tr:hover { background: var(--surface-container-highest); } +tbody tr.row-selected { + background: var(--surface-container-highest) !important; + box-shadow: inset 3px 0 0 var(--primary); +} + +tbody td { + padding: 5px 12px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + max-width: 320px; +} + +td.num, th.num { text-align: right; font-variant-numeric: tabular-nums; } +td.zero { color: var(--on-surface-dim); } +td.name-cell { max-width: 240px; } + +.loading-row td { + text-align: center; + color: var(--on-surface-dim); + padding: 24px; + font-style: italic; +} + + +/* ── 10. Badges ───────────────────────────────────────── */ + +.badge { + display: inline-block; + padding: 1px 7px; + border-radius: 10px; + font-family: var(--sans); + font-size: 0.73rem; + font-weight: 600; +} +.badge-green { background: var(--badge-green-bg); color: var(--green); } +.badge-red { background: var(--badge-red-bg); color: var(--red); } +.badge-yellow { background: var(--badge-yellow-bg); color: var(--yellow); } +.badge-blue { background: var(--badge-blue-bg); color: var(--primary); } +.badge-gray { background: var(--badge-gray-bg); color: var(--on-surface-dim); } + + +/* ── 11. Chart Grid ───────────────────────────────────── */ + +.me-chart-grid { + display: grid; + grid-template-columns: 1fr 1fr 1fr; + gap: 0.75rem; +} + +@media (max-width: 1100px) { + .me-chart-grid { grid-template-columns: 1fr 1fr; } +} + +@media (max-width: 720px) { + .me-chart-grid { grid-template-columns: 1fr; } +} + +.me-chart-card { + background: var(--surface-container-low); + border: 1px solid var(--outline); + border-radius: var(--radius-md); + overflow: hidden; +} + +.me-chart-title { + font-family: var(--sans); + font-size: 0.78rem; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.06em; + color: var(--on-surface-dim); + padding: 0.5rem 1rem; + border-bottom: 1px solid var(--outline); + background: var(--surface-container-high); +} + +.me-chart-body { + padding: 0.5rem; +} + +.me-chart-body .chart-empty { + font-family: var(--sans); + font-size: 0.82rem; + color: var(--on-surface-dim); + text-align: center; + padding: 2rem 1rem; + font-style: italic; +} + +/* Full-width chart (org timeline on org detail page) */ +.me-chart-full { + background: var(--surface-container-low); + border: 1px solid var(--outline); + border-radius: var(--radius-md); + overflow: hidden; +} + + +/* ── 12. Buttons ──────────────────────────────────────── */ + +.btn { + font-family: var(--sans); + background: var(--surface-container-high); + border: 1px solid var(--outline); + color: var(--on-surface); + padding: 0 14px; + border-radius: var(--radius-pill); + cursor: pointer; + font-size: 0.82rem; + height: 30px; + display: inline-flex; + align-items: center; + gap: 6px; + text-decoration: none; + transition: border-color 0.15s, color 0.15s; + white-space: nowrap; +} +.btn:hover { border-color: var(--primary); color: var(--primary); text-decoration: none; } +.btn-sm { padding: 0 10px; font-size: 0.78rem; height: 26px; } + + +/* ── 13. Student Slide Panel ──────────────────────────── */ + +.student-panel-backdrop { + display: none; + position: fixed; + inset: 0; + background: rgba(0,0,0,0.35); + z-index: 599; +} +.student-panel-backdrop.open { display: block; } + +.student-panel { + position: fixed; + top: 0; right: 0; + width: 520px; + max-width: 94vw; + height: 100vh; + background: var(--surface-container-low); + border-left: 1px solid var(--outline); + transform: translateX(100%); + transition: transform 0.22s cubic-bezier(0.4, 0, 0.2, 1); + z-index: 600; + overflow: hidden; + display: flex; + flex-direction: column; + box-shadow: none; +} + +.student-panel.open { + transform: translateX(0); + box-shadow: -8px 0 40px var(--panel-shadow); +} + +.student-panel-header { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 0.75rem; + padding: 10px 14px; + border-bottom: 1px solid var(--outline); + background: var(--surface-container-high); + flex-shrink: 0; +} + +.student-panel-title { + display: flex; + flex-direction: column; + gap: 3px; + min-width: 0; +} + +.student-panel-label { + font-family: var(--sans); + font-size: 0.75rem; + font-weight: 700; + color: var(--secondary); + text-transform: uppercase; + letter-spacing: 0.06em; +} + +.student-panel-name { + font-size: 1rem; + font-weight: 700; + color: var(--on-surface); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.student-panel-close { + background: transparent; + border: none; + color: var(--on-surface-dim); + width: 28px; height: 28px; + border-radius: 50%; + cursor: pointer; + display: inline-flex; + align-items: center; + justify-content: center; + font-size: 1rem; + flex-shrink: 0; + transition: color 0.15s; +} +.student-panel-close:hover { color: var(--primary); } + +.student-panel-body { + flex: 1; + overflow-y: auto; + padding: 12px 14px; + display: flex; + flex-direction: column; + gap: 12px; +} + +/* Mini stat row inside panel */ +.student-panel-stats { + display: flex; + gap: 0.5rem; + flex-wrap: wrap; +} + +.student-panel-stat { + display: flex; + flex-direction: column; + gap: 1px; + background: var(--surface-container-high); + border: 1px solid var(--outline); + border-radius: var(--radius-sm); + padding: 0.35rem 0.75rem; + min-width: 80px; +} + +.student-panel-stat .sp-val { + font-family: var(--sans); + font-size: 1.1rem; + font-weight: 700; + color: var(--on-surface); + line-height: 1.2; +} + +.student-panel-stat .sp-lbl { + font-family: var(--sans); + font-size: 0.68rem; + color: var(--on-surface-dim); + text-transform: uppercase; + letter-spacing: 0.05em; +} + +.student-panel-stat.accent-red .sp-val { color: var(--red); } +.student-panel-stat.accent-blue .sp-val { color: var(--primary); } +.student-panel-stat.accent-green .sp-val { color: var(--green); } + +/* Ticket sections inside panel */ +.sp-ticket-section { + border-top: 1px solid var(--outline); + padding-top: 10px; +} + +.sp-ticket-section h4 { + font-family: var(--sans); + font-size: 0.72rem; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.06em; + color: var(--on-surface-dim); + margin-bottom: 6px; + display: flex; + align-items: center; + gap: 6px; +} + +.sp-ticket-table { + width: 100%; + border-collapse: collapse; + font-size: 0.8rem; +} + +.sp-ticket-table thead th { + background: var(--surface-container); + color: var(--on-surface-dim); + font-family: var(--sans); + font-size: 0.7rem; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.05em; + padding: 4px 8px; + border-bottom: 1px solid var(--outline); + white-space: nowrap; + cursor: default; +} + +.sp-ticket-table tbody tr { + border-bottom: 1px solid var(--outline-dim); + cursor: default; +} +.sp-ticket-table tbody tr:last-child { border-bottom: none; } +.sp-ticket-table tbody tr:hover { background: var(--surface-container); } + +.sp-ticket-table tbody td { + padding: 4px 8px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + max-width: 260px; +} + +.ticket-id-link { + color: var(--primary); + font-weight: 600; + font-size: 0.8rem; +} +.ticket-id-link:hover { text-decoration: underline; } + +.sp-ticket-row-escalated { background: color-mix(in srgb, var(--red) 8%, transparent); } +.sp-ticket-row-escalated:hover { background: color-mix(in srgb, var(--red) 14%, transparent) !important; } + +.sp-escalated-icon { + color: var(--red); + font-size: 0.7rem; + margin-left: 4px; + vertical-align: middle; +} + +.sp-no-data { + color: var(--on-surface-dim); + font-style: italic; + font-size: 0.82rem; + padding: 0.5rem 0; +} + +/* Active row highlight when student panel is open */ +tr.student-row-active { + background: var(--surface-container-highest) !important; + box-shadow: inset 3px 0 0 var(--secondary); +} + + +/* ── 14. Ticket Slide Panel ───────────────────────────── */ + +.ticket-panel-backdrop { + display: none; + position: fixed; + inset: 0; + background: rgba(0,0,0,0.35); + z-index: 699; +} +.ticket-panel-backdrop.open { display: block; } + +.ticket-panel { + position: fixed; + top: 0; right: 0; + width: 520px; + max-width: 94vw; + height: 100vh; + background: var(--surface-container-low); + border-left: 1px solid var(--outline); + transform: translateX(100%); + transition: transform 0.22s cubic-bezier(0.4, 0, 0.2, 1); + z-index: 700; + overflow: hidden; + display: flex; + flex-direction: column; + box-shadow: none; +} + +.ticket-panel.open { + transform: translateX(0); + box-shadow: -8px 0 40px var(--panel-shadow); +} + +.ticket-panel-resize-handle { + position: absolute; + top: 0; left: 0; + width: 6px; + height: 100%; + cursor: ew-resize; + z-index: 10; + background: transparent; + transition: background 0.15s; +} +.ticket-panel-resize-handle:hover, +.ticket-panel-resize-handle.dragging { + background: var(--primary); + opacity: 0.35; +} + +.ticket-panel-header { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 0.75rem; + padding: 10px 14px; + border-bottom: 1px solid var(--outline); + background: var(--surface-container-high); + flex-shrink: 0; +} + +.ticket-panel-title { + display: flex; + flex-direction: column; + gap: 3px; + min-width: 0; +} + +.ticket-panel-id { + font-family: var(--sans); + font-size: 0.82rem; + font-weight: 700; + color: var(--secondary); + text-transform: uppercase; + letter-spacing: 0.06em; + display: flex; + align-items: center; + gap: 6px; + flex-wrap: wrap; +} + +.ticket-panel-summary { + font-size: 0.88rem; + color: var(--on-surface); + line-height: 1.4; + overflow: hidden; + display: -webkit-box; + -webkit-line-clamp: 2; + -webkit-box-orient: vertical; +} + +.ticket-panel-close { + background: transparent; + border: none; + color: var(--on-surface-dim); + width: 28px; height: 28px; + border-radius: 50%; + cursor: pointer; + display: inline-flex; + align-items: center; + justify-content: center; + font-size: 1rem; + flex-shrink: 0; + transition: color 0.15s; +} +.ticket-panel-close:hover { color: var(--primary); } + +.ticket-panel-body { + flex: 1; + overflow-y: auto; + padding: 12px 14px; + display: flex; + flex-direction: column; + gap: 12px; +} + +/* Meta row */ +.ticket-meta-row { + display: flex; + align-items: center; + gap: 8px; + flex-wrap: wrap; +} + +.ticket-mantis-link { + font-family: var(--sans); + font-size: 0.78rem; + color: var(--primary); + border: 1px solid var(--outline); + border-radius: var(--radius-pill); + padding: 2px 10px; + display: inline-flex; + align-items: center; + gap: 5px; + transition: border-color 0.15s; + text-decoration: none; +} +.ticket-mantis-link:hover { border-color: var(--primary); text-decoration: none; } + +.ticket-date { + font-family: var(--font); + font-size: 0.78rem; + color: var(--on-surface-dim); +} + +/* Field blocks */ +.ticket-field { + background: var(--surface-container-high); + border: 1px solid var(--outline); + border-radius: var(--radius-sm); + overflow: hidden; +} + +.ticket-field-label { + font-family: var(--sans); + font-size: 0.7rem; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.07em; + color: var(--on-surface-dim); + padding: 5px 10px 4px; + border-bottom: 1px solid var(--outline); + background: var(--surface-container-highest); +} + +.ticket-field-body { + padding: 8px 10px; + font-size: 0.82rem; + color: var(--on-surface); + white-space: pre-wrap; + word-break: break-word; + line-height: 1.55; + max-height: 240px; + overflow-y: auto; +} + +/* Notes */ +.ticket-notes-label { + font-family: var(--sans); + font-size: 0.7rem; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.07em; + color: var(--on-surface-dim); + margin-bottom: 6px; +} + +.ticket-note { + border: 1px solid var(--outline); + border-radius: var(--radius-sm); + overflow: hidden; + margin-bottom: 8px; +} + +.ticket-note-header { + display: flex; + align-items: center; + gap: 8px; + padding: 4px 10px; + background: var(--surface-container-high); + border-bottom: 1px solid var(--outline); + font-size: 0.76rem; + flex-wrap: wrap; +} + +.ticket-note-author { + font-family: var(--sans); + font-weight: 600; + color: var(--on-surface); +} + +.ticket-note-date { + font-family: var(--font); + color: var(--on-surface-dim); + font-size: 0.73rem; +} + +.ticket-note.admin-note { border-color: var(--primary); } +.ticket-note.admin-note .ticket-note-header { + background: rgba(79, 142, 247, 0.12); + border-bottom-color: var(--primary); +} + +.ticket-note-body { + padding: 7px 10px; + font-size: 0.81rem; + color: var(--on-surface); + white-space: pre-wrap; + word-break: break-word; + line-height: 1.55; + max-height: 200px; + overflow-y: auto; + background: var(--surface-container); +} + +.ticket-note.admin-note .ticket-note-body { + background: rgba(79, 142, 247, 0.05); +} + +/* IP role display */ +.ticket-ip-roles { + display: flex; + flex-direction: column; + gap: 6px; +} + +.ip-role-group { + display: flex; + align-items: flex-start; + gap: 8px; + flex-wrap: wrap; +} + +.ip-role-label { + font-family: var(--sans); + font-size: 0.72rem; + font-weight: 600; + color: var(--on-surface-dim); + white-space: nowrap; + padding-top: 2px; + min-width: 72px; +} + +.ip-role-chips { display: flex; flex-wrap: wrap; gap: 4px; } + +.ip-chip { + font-size: 0.78rem; + padding: 1px 6px; + border-radius: var(--radius-xs); + border: 1px solid var(--outline); + background: var(--surface-container-high); + color: var(--on-surface); +} + +.ip-chip-src { border-color: rgba(248,81,73,0.4); color: var(--red); } +.ip-chip-dest { border-color: rgba(79,142,247,0.4); color: var(--primary); } +.ip-chip-unknown { border-color: var(--outline); color: var(--on-surface-dim); } + +/* Row highlight when ticket panel is open for that ticket */ +tr.sp-ticket-row { + cursor: pointer; +} +tr.sp-ticket-row-active { + background: var(--surface-container-highest) !important; + box-shadow: inset 3px 0 0 var(--primary); +} + +/* ── 16. HTMX Loading Bar ─────────────────────────────── */ + +#htmx-bar { + position: fixed; + top: 0; left: 0; right: 0; + height: 3px; + z-index: 9999; + pointer-events: none; + overflow: hidden; +} +#htmx-bar::after { + content: ''; + display: block; + height: 100%; + width: 45%; + background: var(--primary); + border-radius: 0 2px 2px 0; + transform: translateX(-120%); +} +#htmx-bar.htmx-bar-active::after { + animation: htmx-sweep 1.4s ease-in-out infinite; +} +#htmx-bar.htmx-bar-done::after { + animation: none; + transform: translateX(280%); + transition: transform 0.25s ease, opacity 0.25s ease; + opacity: 0; +} +@keyframes htmx-sweep { + 0% { transform: translateX(-120%); } + 100% { transform: translateX(280%); } +} + +.htmx-indicator { display: none; } +.htmx-request .htmx-indicator { display: inline; } + + +/* ── 17. Utilities ────────────────────────────────────── */ + +.empty-note { + color: var(--on-surface-dim); + font-style: italic; + font-size: 0.82rem; + padding: 1.5rem; + text-align: center; +} diff --git a/apps/mantis_explorer/static/pisces-logo.ico b/apps/mantis_explorer/static/pisces-logo.ico new file mode 100644 index 0000000000000000000000000000000000000000..dd66c2316549dbe7302182a4b8cbf60f01d4f830 GIT binary patch literal 152126 zcmeF41$>;x(Z=^Zv24j~%a)nM%*;%-B!ev3GBb;rB{Q>(f+H~=OR~&PoY+YkYVx&B zlcsH;X*e2hlK99SE_;T1A^?%e?Kz#+&S3rFQ)K@@#1=LqSeFfB4Kz#+&S3rFQ)K@@#1=LqS zeFfB4Kz#+&S3rFQ)K@@#1=LqSeFfB4Kz#+&S3rFQ)K@@#1=LqSeFfB4Kz#+&S3rFQ z)K@@#1=LqSeFfB4Kz#+&S3rFQ)K@@#1=LqSeFfB4Kz#+&S3rFQ)K@@#1=LqSeFfB4 zKz#+&S3rFQ)K@@#1=LqSeFfB4Kz#+&SK!xh1vV||B5M2Zxds(^=b1GNyL2MF#lw8; zWh26UyM{%FwhfL6k{Ha5?9yK{_q=vOyo?h!ql1AFYIA1vlDRW3z%KsJnVS9o9`^hG zEwvWRlv@Y5NHg87<>kIEiUNOk(I$U4>GBXS`Rp(sML>Y7#3saDIyKT?GInODc-WlC zUfsFj7dUgnE;2Ckzhp7Nl|*g7{cDgJ+(yz@onz(%%M2ntWy_q+<*(bD$v<;6mwn=7 zA^)qhrTil|TiG8y?4)ma+Df~99ArX27sOIKuov&1IMt8;RsuV`d+K&M_p zOgm>*u!u>H?qSw1JV%Y(vu@m%^jRBj>tIbV5v&!~K{hPxB?}f_CrcJyBq_gQQ?viu zXP^EXs`!ue=^7d6qBvr2BKyL@O#YLj`8{*MeS0(69S2jzkM^eWZ|zK_UpirYxLL{H z^Rknk^>q}V_H~qG`8kWDf?PT+qdldg7ex0ASf6}O14ix{_)iA^tbzYz@ShC+lfi#7 z_)iA^$>2X3{3nC|U$LqA|L@^HCZ?9-=g~3Q%Td||_HR=5oh%gn43Ft^bWB+E8NF{N zzXSWx!BqYYZa;UmlD_X@BNqGG%eDu($`*xrh&^Kcq~qoVck95&ykH5L7uE(7lle2+ zV0Ex2vS5aYERAo4f&baye-8Mc1O6BMdTl0rhuVJo*MR!}nK5-*{@!h_&Nk8;juu@k z_s*7zpY9sQf-|^h&619de;prm%s7}S?%0`PJebSB!dUp5kDd5@fRnVs&p{Cx;nrn7 zD^N0cR#=+`_-DXB?6=E*9Qaq;Z~q!l{%1#Z>W2AA;v6j%zjrWy0RR2R!UOLAe6CQi z+R4F8hWSDEBXrUiu9ot5d~6j$UkA~eC@+~C_#XxSHP!t8_WAj(`vA&+T(D$dxVLn@ zlcnNAM{}j_x$FBM!22VA#_A^J9KmX(zIg!}=?5os$tM^G3O`3#fxkoYa~{~i3}-;omZbM)Y=KZ ze4CAb!Rdqfe)Qa+;u~vcKOqN1bE8-@fYJ2?RX(+EY$@jiJg=M!esVOGedlZ`dE3iT zu{YF95imcpYjAS3f-^T%!NAC`%@XH!5w-nSTm!AhP?5cxW6OC5b1AFSDaTLCKg< z5AOlq#b3KxNnUri70X?%C9gS}D!z6w>r#$^esfFz7=gwIc4pE$_U6*h-E5>Xe`m?+ z7(e->83A3IOJ?;l8y23YhV|iJhn>nZCjSd|x_Au!&e5c5n>9JJ#gd#ZoIG*NBl6~r7 zCqEtN(m5+4pwT15t8;dsQ%|+CW!EQk9htuNL1Rm~ra)t4*-v(+&_ou}_dV_8yF$IX z+~o1S*Vb&$|Fe{?YlV5}VFR)Ml#^>Ol0sjp6XTU!L{u%JkfPV)3GvJ>A{|xx2 z_8%Ju3*)cp#s*)Ebh5zOml^m4|ET$SAfpfBmOe+>rR!gICX$<8j@_bJk=MNEM_Cyf{vivM5}&_g$Az}?^$Zc)b4O%csra(73?$& zkNcS$sXZM2+_pUVtlr!iU9+96#BVx)f9n6Ag@5X|?MO{G8hIiEeUcERhD*UrDnSv~I@voNNY3Gon*_O=(D!CLUQY;Dma z_*c#Y_wq)4wRgAiz1|LRkQK)Jip_&vI<*3vIvE)GPuc$f{|x>{2Kq-6`d_y?XShLi z(by?f`Nn~~jA4uT*@N@=xdw-M*}4t9Yz=u`j`r1tOucu7>3SatGPOUgP1XF8o2vD_ zAf5ArAcOf)kj~uV!`S;796Z+eUXY>jwID@WBg^qXZUp`dDuvtjK- z!H4`Ty^r{r8lTmrGv5f($d7^ya;H9%{8R@MrjwrpF#gZ&4xWQCxB2Ou@A>K4pYqam z-sh%kz0Awf>EP!M+{w=$wSr$T&X-p@WxTL-{J_S-$y&T3E9Pj08#VH@wsuLceo|cb z9B1nPp*N`hR@S*aT&F)+vnXQ##i%6+J*4N|?oYhihJfp67 zcx+9!{vv*+!B$@801@Ve4}=-acLK^k4CCP^K8%O^^p-xRhOxvNJ3i9-J}-m0!%x@v z0%PHQ%nz4&IfMAZ{GqA!xug7Rb4M6g7mPQkE49>WDtBaH2-73?^Pdb<}dhm&&$=a8>^E8_e zduScc8$F;lZ>%vdZ*#iII~@=2ACb9LeKnLENX&KLZg0a9MB;c8*_2&=YI3q#34Cymzq_N3{618F?u zL>iB{kj7)Kq_NVCwbuP1oYo!g2F=BjTrggW1(|xU3DcP$Fo)co2kz&e&Jho^zA?y- zOI9y&zUHNA{ehpUE9K`7F5+bk53I`{GpfF1l5TzJ6sCU56jEDZO7`X0k%|--QnbOH z`||M+Mm&JfIVn1 z(dQnvvg5(7J+?8v=e3d|&y&Tm=gI1X%VcBn%OrKlb&|g9b&|Q_cO+}&Td=oZm|0IB zLXyCK{wf=CFxQ1tZ}BAcyOrE4nGYbHM}8gFjyg`Q> zN{oF3TP6>#%A3G}F}$s2q;9V*X`t*MbtO&UzUjF8zgXuEH+=p?BYwfCWMR(0tB8*g zBlmqK{oMZ6$AAClSZ&mQT+r{UoD=Y!nVW(%&L@HlovZb^!}bXCMuZBBMi@2hoT_)S z+=^4U!GprIWQY<9va#?Pa1teb6To4r}@EeG@N7sCJ3GW|5KSIskY9 z@!<%sZtGQZ&uhY%&5K?J_c!ixpS|jBlD+Dk7ivjSlgYLe2Xdwy-0$%vt?>07hbixY zM0E7mWo-w%h_Kj%!z-9LgqJtmt}buT1~4K6dmjoiXnsb?9m_rLLt{YOXujW=KS3VMn@&{)%ZW(Xdw$oC#d51Ye*%$1zA`J(jxxsyNr8{Xp?m?PQ zJlHySxHH1C$&9dk(varzF$;xR1Fv#7(|pjqdEjm=&`%TLJ5n7+Q*u|oqLH)u4U)6^E!aD-cYhHR9}6PC z5#u!Pb{SBeJ!UE|d-$r_Og*tML*oOgJyg7__@@l=(#W^mG@Xmw%;6bLrDIc?atHO+ zr)hn{Pap77P3GX`f})Y5nvOYZw4C&0VB}d%ymva&v2)7M`a+|XwW<2QtKH17Y}4<; z@=w|C$G^R+Nb>;HIZoImi>n7=`T{Zp9E`40Ki+p6!+#?<&7WsHFLOV{{Pn6C9M zH+zuaMBb#}EvZi0x$Arx80l}Jp7NmquJVIe|NZhw^7Ai}mC;Y9ZM6lJ82)9_{)&j4$C#Q18GBt(cHS5YI z88?>g!Iljk( zRNggrJa)tLuH@uy7t+4VnrYc;HlikdNP6uitqf7hW76e55EKb^A0bA^GxRAPXH`1`x1G=4Z??YMRKQZ)Zqoo;YMP%$OEb<6n0 z&^+gO8#KQ>yr@{XXX$hmKn!{<>X@j|G@LqnZ#wS7=cjLR=Z}|Q{ z3NsC^Hx!IXs?HxXo?kF#w6JLSr0V=(*4*4tUcAiVA*VL#gq}_v6k1z2KA^V1$WD+k ze0)RR$l*;}rVQY1anan8;>F~H|I{U`!F~zY-$$0rK28?SIyWFTq-Pn}{}Zh_piDiF zU;cOht}5?pJx%t5qlLI9z*7-4JLZyZ`qJMqx$6=4_VJ&&_H~lF;yT#B3ii(_*`Fr{ zbQZ#PQ|{D zF-Z)cKvvG0LQ)r)kb>prWa~OhvNy$=9LcmLCv)NR3*hsM;qy!3^ULAmx5DS|@FuOh zeM$RXKhnO>hqUd*c-U!AP$z@A0d<%q#-w&T;?CW+q-h)SEoBare~w_ww9ySkV`mF9 z^_s!W-}q?+>#p~hVsk3=G5jiib`v`Jt-9xe z55WIP@P8WoSAqYU-Oi-DZ3!bf9jaAdI?0}!GT?OWCgz{`PRiU<-x#IilxmvV%^E)l zG6!F%FB%inuzjk|sV$~Vb%iCTzRXdJmus%GZ`B~3!<#1P)NS|GuHWyaQGd{rsVt5o zJJXht;x(zr(-o183wDsz^A3__bB>e6v#Q9#cph1md{rkt?0hKpslWB88qI$<*H80# zbj;GRYcpN)rKh8U7Z-Zb0{m;1WW7r=HvW#}YAMPuX8~kz}mALb6uBOtRO!a+iNr_izZ&(c%yTeJwI_ zs4kg2b|A4eQTqL8u%8q$k*uCQm835;B}FSO$o36ZWPh3sISQYD3O>I^<@3wnTPwi$ zR`~oK;C;6r>DY&u`al5b*zXPg;rq9mLm!RB7@0__i-wch11_ZXfGcS$_huT4Y;ill>wphw!Z>W$>rLwRc!Pg$!rkQ!-|r3I?@dnQ{z*J{ z96zhXYby7__G2vU$9w?(PwoZ#m^)PLb9ZB|gSGBedxZ@U2ZZZ04qFd_%`3 z&Fj$bulgJ47~rPhyJZ{l1!aiAkNIhwF&P;!?6<{%*1=;Rsne=M-5qtA-@U^T~qvda@$voJN$d!ou5Db_sL99lF-~ zv^5Y)E{InJ^fOJ|1wztJ+oq7 zC+5FaaNY{;+mSb=?90Kvf@OaN_P4A_g60AHsmm{s%+*&(&f05;n_h)o|26T?fPbcA zkHetGeAES`4?N0E)A*43e3l(B&-(kmTb1rX{QF&fw$AzLoDqw#b~=K)W%9V1{K4fl zshrPvDH{K5${AL5_JEJorP?^nOT8<8QT~M(11(!^n5JzO1M0UGah&6RvrwjrA%=7ZUJfWM3^puYda+cR(t@dj+24Kg)Z%qd&*V}BT zNpCw$m%I_?rHG#&dsZL(Gyl!}bNAW}7nT~&t;;eH3sW_}KrDM#=l5ryy$`1NDdhVz zn>gLVlHqfNTP6+bJsL2$zG%Yy>J*)Kc$+xi)nyqfq&v(*Upg77`)>VOrt=WyqF>Pe zQ~vAeJaE8{5mZbyIFUXisVZ&2<=Rw@AJ|+@-`u5|2Y9K>kF^%WkH2Kt9{ zD&Nq!reN3#UY6b!cV;uEXYXoG?EaL2gRclewIn+&y&Ch3 z6x@xR?}e$_Z`bD=E$`T7YH;DW4!`W!t5?A7Q|^UP zarmUS%Jt#C&jY*cxVsyFKbsDtV*q;*ezKV+`fG?+cg5VO^CQ7O^GoO-Ht$1oo_d;O z_+VW!am+wM*M_~3^NIE~Ci5dFk~MRtk<3MAq+~VfBsN)-gXy-UGRK~rM$B7_m{*{R zd9jAvybUogw14{^Khn8BfQS!ao#AjG^uIfFvMu2jjVG0vhG1kAM_4j;A~(k%gO{f9 z8hrlUHKhA}K06lR_j#L8vy-WJ{&fDJgacWIgLkYOz-ilVGP*v0csX~o#y|MTe+sh) z9B3$+VBECRfzyJxwH2D58rA;I`+Z2`ULUOYc_aSyM(q0l?SEJIL;D{Jb(YV2nOQWuhb*1jK~^RBe>HBC8XX`A(|+v(~&sa@W5?@;6*3`Ri|xymdEN{ZC_G zO$-1%9h&dgA>)k(60500h?o5+us@b8h?+#!B}^yTOUy~xS}U?U#fBWtv?C{S9Y|H7 z6Y@7MMA$#(-HsS{C-OardqoEViR5rFksb{plA{5nZI3N!-aegFrVeB((*|;mXBtf6 z<`@~OUDm-lM~?oDqLi}E{$C><${k_f@Q_iFr)(UHfb-;wh=FFM=2?i&# zhQ{*JwOcR-zMy_z^>@htHvx4_VcHok)t<-f;71vU-}aBJIyJz5{pJdk7GUf^G94p5CP z)Sj}Rg<8>Xd>j=wyzIMQb23-lRE>$J8B?m)(%+>z$acEqYfnen$*91y;}*}k%9)Q? zcV%K7S+{^sR?cZ8i=sQxe^)^k$6v(r=MWQjkyVI~*DdHJsmm^r?6ohG{Ee@Y!c8=W zevK5Ye+_k0ud)1dkoeTmqB&1(taS|_7E^{0Pdm!~IFcManXI2@LUNZ`kc#!zWN(@+ z@;z9W%6CM*$C>cq^Xq7yv(M+V`JP?Aq+^di=|s*)b~Kd8D`Ce&iTJ25X{;E>+P*Yh zPGze8_jL5>e(L}4`hL_n3BW!V>{s{C`LOK!f_oV1 zjgI4YPk{fE2e2jp-(LmaPub@|;|O4N@beAeylI~cX@RxDI$)jDFitu!PaMk|&D7+L z&==$m@#bgh9|7+l2vQk#tkbxlHVv^(-iV{Zg0Uk-JN=n6#eU>)?o6^XbtNfUolDXd zZ6|9J4wGf^r;z8Y1?NBa`Sau3@mwbi{g|=dlNi^g8SX81_O_E=#$K3P_F%{nZ2sc> zW1OJ}&`-|hlDEUWJG0P-`T%m7-#!ojbR5v%r}ffyX5u%4JY>r;jcA|Pe*JiW0 z@M_J}B{hZ{7w|^S4QbV$6(XnnGrxBJ`GsRS^`*uWYH|h^*Q9g)Nc}!#{cfE=f4@(k zr@o)&3O91T=4TE$(^xXm^6V)e&g-3vn77*JbK18~vZ~J=+`->W>o2w66BHV+tSdJ$ zke!*$$ict(SK$9C|IOQ>>-RY^7@HbZsF^!eJSw6(OaBOJ@m@!*=eM;fT5nY4ObR?x z?52G<*B!opCVc-&`2Jk@{_XJnhsnYjr^x*1T9OzgB=aI0V9jJ+R4bVm-A)o?I!V$D z`2U$wk{pNpcYM!)C~tYJlbQHkJKD>qPXnoR#f!dW8KwCqGudZe4xQU$eC2LF_KF(B z;K~>TW8+yrQ&lHQb&!L(`29dn@k*TAG+<*w`-FL+tqH+y&7tAGQp0)JM+yF!U$OsZ zcm?B(d09iU!RjBO>6L!JAOH9JeVPa2ZbF_g)j(QTJk?KrCR+FXbEyn(e=s9IW{do$oXXJ*R^jMH@Ww?t47apPx5QyzkZ7c_7j9MRXZ)QH*=Z}zkKSXx}vcO zbvcHoPNy2kDzhgoIF@g&|04Kb6yME>^pQ=(c`m2zOeJ3_*Mpy(!AG;J+%susBERWv zr|60B6M2Stb_IFbDlUWnThFb3`t}~MeI(3e-+S3B>gNPqnzm-Ac(hdGDq81k$ zNn%{c#i7d*E^xp<^Q8KRjeRvZWS}1E9fs)>(@BGghpkfkFN~Q&HYS;p{1uj1f3zX{ zGwetuV%{?kjCqk?LCnjp_3rg2qW!3yJ`hCCo{1)k6QK;Xj{R;%!omsbYBuTo4!L`F zUF(s4AHJVzetw$X|1}hjU)p)VcW{p|iIJesPTgK_j<9Hq8G18_Y0QiLgdYeCj8-?4 zP8xW%)`#gn?n@*`5Qjj+w?WId9zg87ANvRP`J(Q}7qt&QsCPr&2elbzaLcauQtgk| zRbrn_=w0|?Se(Qc$(hlo{iKy+veZ&gwsF7$v zkB0`-{|U-bV*zb{IzJeBCmTk%&4TGX;HcNQWxB!|N9 zAgkx@AxmN_NpegzNs8iQpHe;iKJxsLE%5y~gC)8HzF!32FM;ou!S^c=A9o{8=wg=6 z>D3GKlEpb$h~KyGTMJOds&~)9pgBrvVC-3p`hChbax{0WO(gF}`(N0Q7}!78ms+vwQ0^%4|Q&X zu@{W8Kb|a}F%|tv%}CKIE3$L5E%I>oVG;Hwv>oukXW0&HC^Ak&zVid*JCXA-5H*xewd&aCr_pf$-v!nNYwfcZ z+%{t`TO(?a>QIZs+v%fZAN-$09DH2GKA5jO>Q9ctP8{(gCl8?>0-sj}o>g%#WxoOJ zH-UXN<^}H^2i)*?yP`2tf<TD zEI?iW^&Z>IISt#)wK|S^=!j1EaJHwdBdH53$ckB1^PC~`qj=!Hj?9Z_ghBH}Li0qm zgZ)mhF9!QkurCMuU0}ZlXAbp}WeMlVx`lEk)KfCa+g@@UHI?Yu16EnBj^04fL&`Dm z?EI_Pr?Dw&mp}1wJ-d5h!u2V0gU_4!+AB^XcmEyjA*$s6*~VA@F-dh0odX<9Wk2{h z%D7QMvI&c3^ceWr%QrchNio(WH+-B#YvY4EjMpV!;1s34!IY)HMfR4yN4Dp`LrPQM zRIn~VAn&6?CK($N7c0vC1hO=CI`VB6$aPwiJ!y92XpRH=1)^sY@|*(H zbJlOcekjznwXivUGB%p{FzAorn2e5wJG|&s0;fU z`w!xu)%@VU2IGXAHc-}7G{NV!wj|Bh+U7Ij%4l%sz%&#aPpHcsdIsyfU-LI>eu#R? zw5IZ@gG2{Im=5@Fx@WWXz&&fh{ic2X$kF?u))_qyxBHMY$kU#L?`PSk`5y3Ic`Se& zuM8w7aC`DFa`4FeRDtFD*=O^d@cXRKr_3L8ClYF?-4SE1=tPZB+b--K+wM;4kQ?Kb z`XE;vNRH=3VBI7hwc-m%DfE5r(#@o+CYb4O$}`B|%}k{@kN#c%pLOE)f?dv#3V#y2@zGk?#V-1jF^ zp7|EZTk|@}S^gTyhEeu2QA7Gm^WS^MTeE4)I9pzhVO`Be=3DC1S-$U!bMNYY8t+ow zgV&)p;4i$Kk?UHD%!j-!UBO(aok1x74ZAFLQ41fzOVfUhx0!QOkfDFMrF4Re^q`BD z_%O|P{NjS?ls*9Nu)jc}YLs955UH*xSDfUHz zNk=*cXiQAU0J}E)fsg;p-n64O#7*I`Fy@MGLgcy0&>{P=ulF0uDShqp z@P7aA((lgB0p^mg;Oh_1j_5L49CLn@zhid=&OP|n3S$mGe?8E(XZhNsYeOrt-lqIB z5941?M-w^!fyBWawf(*m$?|wplD*W5Y~5%}4rJPsuz&P>`~-&9z<;@-TS^b68jE~MhFT9Z^eGYPq~{l|4~;odSCOl z>7&lp#4@5|Az14Qz*-j!KEDmO)LKw`+JN~if6C_v3Fkj&(3L*MAhgzY@CV zcn~=OI|co78hQVFzTXj=-I3Kj&HJ25n@aOgpMS^$dJlHc1ODFw+Q%LG%nfzlm@9F; z_+?mQCqh$YCVJR)y=!Z#K;3B9O&4?7OQG(H8S`gd&|A9T zGP7a9HIlaAI>}nx=l@Ylnz8aa89GRpSWO>JB77#2m9x!B!Ad)_JJo?4MVx!O2y0wr zSmUa2LHve)Ps%;^u~7EgcR*wB0`q%N*RVerF&}h4G)MRGDAIe9S}fDB!(ou1VEi)V z?cYQk%hv8cjD0o^relNh&(^(RovUzoVlOvT?{ZT-BRLs}nqW^NIvlN0Uoy^&pRLzL z*N}KATEDL=9v|DX&snb?`?DHQtKW>;=2r0Ditp5MC=k6X0!b5UFocLtSoXJoeb{lt zz;unPvJ&i{2!KKNoD3zWjt7%7&^}e*xCUCEw+nj(b~}K5N4z)KN1b*jxEE9A4|@_B zwL_lZ-xJru1G*pe%h3G|(ET*#tts{=C-cIvW*&<^((_6A+La`aWj`P6Z$WNzFLeG< z==@XA`8D8P0PC0Eq*|YHAJzF#-LrJ=IkJ4-1+p?p$^Dw-D})*y1M8MvWng+Z8*GNN zx#%UEX{tUxN^PQC7wbO`?)hw`uUE#nlsov=I}RrDZ=KDhZ+kmP%4P=i*e;%NQFmTs zmrjsJx3{OALS|3*udwm9iv1^_TU4As?iTf}Ke<@QJ_vD_Z;lU93|KnrlB2JatkKR) zcFPVu+A+?5aIuz%0(_*tGiFG&bCRKbV0p`xv2W&@8zgPjtG^ik*bk^xpEuk=kgnUq zPq|mm_E`Sm`)Ln{`aHwK%{!({Zrf+c^yk0(Xs~|AcB|Rkbe(rm-+Z$%XK+t*k+DV7 z9#4*%{|EC=#{?Y%vqR*=e4Qj~v5xwYjS1EN_k6mlU$inlro5|6ncJkA!_GwU18PeD z=4B&k2yzt9kMWUBm>b!pS&|~xn&Bt5buklhZKp}Tr2VJ+zxUnu?8{@)IQDUTKknyj zA1-tG58n3Ts`!wO@oN-0UW7UHwMq-PY^o5+8%o+kdn%~5nXCR}C?_Bk--e*ULU z7)-*vv9Be;l9a7;M9qmCIfc50no{iJsz6QKR#*5j!w;lCt9q^fL+k8pqF4VH^ z35LH5fe#FY|Mx+Cn?Jc+GnZUE6UAvL88@DnHDHG@mH7wt>8w3S@AN&Vig^(q+~H^G zzFAi?GNJRBw~nkjifMtasN3a81O-l5JDxPMzIgP(npDky)Na=NqOE+=PRSl?qk8ye zA^dJ5-CqXX(*YgUy3Y^2`>7rZLY;y?d_VFp74ZG=$;Y?STqpegDSuLV8ovJ&@-ioa z$?4Nkr0Q58sYVQ2gU{scv_r1c4s~q~$n`iOuY}y@A!r`z^N)BDIkm%H@c&-WKAzBl zsP9AE+lcipA@%^(BIk1|KZG2~no0JgB#{c_HS?FJlCJ4HjhE`oPp2h zl7t8$40)b@`AwSVq4Dm#Xz>GMUh4a4yh}9?_4#WTVqYMP-e1M;6S;)FG8eJ0=Wm?3 zGp!J{4||!&&sC8g?kRSh z7u0FEK2E{ejQ%{CYua_9eZ|4fW|G&er=V7iU29h6eb{l($34zvMchGiP%nUgd`~(i z-7KY7qP!&Gb3&zhGXoXz=+XGPgNagm(VVQkiTF=}j*66}QCCK!FL{G0SpO!;S^qjo zTYH^sUU`jWf5QS>vLh9HS&Kcf9~?DdTO5c0CftTRC-N69+dYUF`9jpD5Yb-b(#!pb zbbl!7T*5H-hoQbF40_jvNGpTL+b!#v?n9miE#;Gwt5db#0?*1`ACF+4&IxQ>0N;-o z;d^fGz~c=&r%!L(W5LwWem`5xN$nOZ(o*iBA{=0h%e zr?IC}fZS%=!9a{d=&@r_qzQVdr#gx>>Ns~ z^$+BHcHp`pN5(Bh?iBgXquDWJZ)zghvVJ8gSdoVPb4Bc$?$Wpe(EG>1eKko4=fmp3 zKF#g7VvQGTx@>-v=6RIdC&ozsCEv4jF3t5^WOWW@e%+#%Sflr;_F0*D3AKu zAMY=ce>$#ceK$1F--11)88HE<35sa7_O%l=Vg5mEqKxgahvt^6x#X2#*RJp-vtQNO zlKnQx+;D?zT>T2!xa{f+=fAhoSF>ZwG&^CIVJBOQ38o+C`&Il?-!I6}dat3ta7oJ^ zbG^1oub=Uco;$kDCDUTLshaO{)1dwH^v`pPhL|1MW5ZGNk3I-b!2j}vH<-mq*YpG3 zByr9boiD>j-a_64{o-gWtoRY@x0ikGJ9c^4wr=uq5XA?%NStH*WD^o1dIu&&ozsX% zK8dowcKOTjp-TSKH=bkCQ?F~ziaKZQ?Ib&DGhK}Fql~Q`+4Y8d{+^Dp7vcZuJjJeQ zo6CRnv6mm66W(JqFQj$2pN)7A`m_H)$2HY->>iqFo!|I5cb%RadBqI;bNchYFV~kG z%Ex*r{9`+OdmGkA`Q@ghZmR>SN32Ut1nb^|Sa(+tVq9<1kmFCzotc4J-w4usJd7w# zg%g|uL3;S{O#5#0;f*C@3Q*7Tu`1X3FwL)eUZs6%vDcBCp(AK68D-mAI+{~+Fp3sevrLbW5iH5Rq#@A0o6T!YveD_eIIHokw$yw+> zVM%UCl1U}zwVxS!eN|#@qO!fuFmF{>hQmThlW-}i#-99|vgyhmEW2!3-`|P4d{*;N zpMT5;IWE{yANYSCXdl!WVQ)Y?*1MW_(!I^-rGh!;Okn^ymJ>zxrOhK-;rk0$rjv|C z#i(Q6iM^_au%3I8%nh$4bHaqUZG`W~nlAD@iHLQRpm&nkwJn=8FDt)zaUnb z>!-e-YI~}4sNY||_+{4UeOCM6XUpea#D3TdWFdV2{OEJYE1g5{nsa1Mh+?>(lQ_lR zLi`tNT2})up(lRyu#xqIc}wO+_;rp8@od(Q_UX`w@kKpd*aealdx;cnyiS%ceuHG8 zjxs;xFC=5_AIYwgkI1H__)haK>iD>HdOMhQijn*Ok+Saqdr_LFe@vj%Y3y3q9ZwsX zXm0db=UD-Ay#Ra38gTysJp;kHFUO1XZ0Ac?=bt+Weu7xb(RHV&t?qc$ee?>!u(9bx_oC$O&pV_q=-sBIfQ z0zJcbSEq5%WAMpz51a2qd`#$41Qv~QgtkM zR$^gN^hM9S&2Q*}|0nYQl>f}-SDB^p=O+ZZ$+p>=h(4wJ7wH-^``#~jE@1h`_l4H~ z+SgHDJ}>&L;rv z^epzQP1Cr@>+4UY(l?L1rTKm~=g-`%%QmPM?{k_eKkCEOR7aEk{GTZ?(W@>PI}5$v zeuupOEpFz3m$><(oLUYAX{h;sdj1zL?nS)a&CChyw)M7GG=PsERXHD=|MHXFl+G{w z>?KE&VlG%^t$AJZN&SD?nkx)UGdi@}1Lvl8*iP;Ik&T7HA=N@JO8-+WMER$_9r5}t zZ#&uPc`-ec5@XH}@pn>ecd=4@OV7cf^N;Fvw$sEvAdYXD9ol2DE~%UI0{uVok`qTm zyX?eXj!#(KLvx;w$@$Qqv+T1vN&8b_?%=G}ZPWD>C&T`Q|Hh(eh7I}SQUqz*A7Iby zH~icowSvMilj{#fF>3ywiGOBpT(@SpmoyHV{~dee3{__aP+sh&qc8as5zccLm4^FE z$2}?k%M!cE%K5!yF?4pQn_`hGYENxV#CUI|7I_icM->OC)+jJ%-S)7Rh-ZaK++zc} z^+H{`H@RALeQbwwK$Q6>fVpu&fnP4TA)7vuj2Q4Z&m!G7UPb)jp$lzP)BTBfJ8Ui zL!RhnA^V*jG!Hwb*|`E^0rLUowyy)7yH3sYJ3ATtGcO?iWtxhn8rJ2GDaAe%oGsL+ z`=7%9ga6xtOoP_e3J0h5ZSEQm;$KiOWTYV9pc?0JeT&%l)4Kc-sSV|(!_@pgWBw;f z%FK$CPV;n>9J4i({^+1O*9ffi*A`eq{Q+m{oelTv315|bS^r7-&s+Hxvp(^ zC>Kllt2QR`+b=o~#A+!zSAl<5E7>2yyyPop_{nu+eWfApma_BEbXbR=I!QUM>0E&4 zZsRQC>%r~{5Ad(~y!`XGS!f7K##st8btUM<_Q)C^mCt8&k7~_}YX1z4@32<9k5^pf zfBd}stL7;>cj0}1^s$p2iudmv9TOoQ=jkjyi9P=~2Z#E9W&Ir;2doA{ZML72s4Cp6 zbK3Lp-?S%;Q&(zg%+EHcfe&ZTp?DbY_dl=VpXN!q>AJ5}WsQkFlRHtfrZVhb_;1~9 zuT@_@!GoKH_yVs-&xns|(g)5sl0HRO&HuCUAL4eF6YO@@%iX4{8~*c_YCj}pMdhEB zd?}ehFWOtEr+g#C_iX&q3*Zur&SbUA7uer*sU3k$b)LmBMhB?5|RCzX^ zN|!%>44~Bb;QBV=`1&ZnF3Sl2P6JuuH~TdC{h#%;kJ<66 z{C;@@m)Aa#%4p6QIDC;Ji&1D~idnlGYEB{?5qL{|A zse=Uhqsnm3(5H0$_DSk|RP(@H{^?pl)_~@g9gfy*`}{c%;{Qx4qg9hn2{`>z8R0GkN278wO9^oz9^Q`>W@Al;gOD(1g(grpOQaQJ&|EEl= z_D z8DWh~bV$=wPY2Nv@bCj&CsNibDEW7xF=HS9R5Pm>vY95iiM=mZVgfFOfqlIv;eY8I z2{SWTq#5SZIm^Ym6M0DqdSu-<)}CVwuuLNdp@RQWPRh61()f7#P;%Df-gt+&q za{hPq4_HPl`=cO7??7G2xS%sJ1@i|rlc=Oj&Wv6TFia>DY@=IHy_1UD|=5_a3!T){Gnx|*LXr;5MwcT=2< z4eOn>e(5WWs{ZX^{(r?>8O|s=$0W!0O!RUPpR_j@Bj2j@%XjCR=i(n@7r9P1Yw7DD z-jdiT?_S*qkMr}KtrWkb^&HBvrsSV~9-9X>ll|cDEPwYI`Dd#0jdZK?2B+~ewLW|r z-J_ZpSpTo&|0`a$Zke!P*nsx^K@a4=xnP)9L+(I7UM8dR|DQIM8mD&ba~P)P|JnTi z@`UqDd`P!Oh?{JNyN&d1=mEA@gEIF6e~)@N%884${Cm_6-|)6ke2D$-w<))_w4TzU z^SucF?t~Q!UK#-Y8Cw7ODE`ZHUO^3!LMOmOvd-B`@*&kybWV9u{L}qjZnm;_!-B>0 zLcH2^{heB8A}92!Bld!+<}TIeDf2*zJ2-Ru+h^gwKHG~U%rZCPWf>k9W@&z}V)|)r zS?vS^WBg!ORNp(U=+B?eA$P>qgCvvNsAolg6dbGY(jukynEL zf8X1qImzEcpzH4{h<34*U2`<2`6lIOs6L0j9(D)y(ceD{|AI^tO+n@)cVW7ImoS65 z^EBM6UQ7Asrm{UwJ_E}+4W(msW&3Q&gY*w~ucrpLVuCd&=a@<-GP$|g+) z|BPDyJbnEG;#cfJxQz9wi$nZfaAvBdlmU9F|F(Ob~o$x&~0i72{?ny++4ypT7t5;GV`A&&NO0R60tpp?GL^U6%GI&&xmJ=f7~X#%$(h zKivPR_@5ngmTX%660Dp@y`l4%ud5OH8t>g;yDM4RqR*O8l{^|RD>i?as zI^T1#<0gjrHR(nNiDD4DT(f8O|Gn7v-aPYc{$F~)lWE&BWq5t=z=OgJjfb99)Zf=X z=)Ow-@H6y3YbzO>+g>(a?_v7qWnmJj&K)zTCVNByFHQ4fHV2x<`Lr?LsHC;bWQ3ak zXXk%*P!EX-WY@x+J**`S=$G(4_P#^=uw4Gt&#i0unuS*e>?(W>XD?mTO$a+5?P@7{ zjqdHjK6mV^l6~*#Aa4!z>YB20&SlP{_+LG@i7ZKI;mnR`o8V<9uCg~*;60ShgxQHI5sx?^F2^2#cOn8rb7lrBbff)y z;;}FC28~nh?gRVT7*O(mBtO95c!8Idu*7Cs^A>wu@XtId|6jkwUZcLu+ym$MNWtNA z@y~Lcfqm>bx@8ST!v~^=$V2(BDY4h8&7a`J&D1%M8i+e|?4joLt=5upfn5h&b=3Sn zJO8tTuxBOy5|bQx-pJ2Rk!LkU^ns13=x>4U($wV%UBlKa=w>MY)aHkEYsL8VxcfMC zUj+Z?4Fvvi)^MP+lo#bI9Si;$y8q=t`iJ`eW%D|)*S|+QF1ULE;_A0mJ&2yq|09k- zUpnOfZv?rEXGHk)qSxK|WLGQsJJ{EWbqyNVD|0~i<`yOY)g?ybxkaN2gt_$mDPvcD z>9m31pYyQ(5B$POn*986@i@2SReCN9_2UoHH;;Qx75@q{G=CK480>G}YC2SMG~%KB zA1iidY6^`<@$v^B#U6mK*xygp_#9_xRB-dgi~|3dyVU(3Xb*@d>i^Ks|5N^F2X&EI zfmb=v-WMjjShVf-b?W5K2|w$avHB$qaDSKooK1gZ=7gUciu(CXtOI;Z_qEVohpv{= zH(~gLP{eU$KWuGtAYnAaY&40?j<(3#t6rz=szYXztsQp(wW=n_jbB<>*TTGefG@u3VYPQ1^jIP z7u7$Me_ol1K6;=mf@b<7%RhPm)u!w9*5;115$tf$Xxrg}-Xwmksr#Rx_uHfTpFDv7 zitIl#Ym#5qj1Q5TCq{PJubq3vaO3=!7#L9*<$qqJR3pT-%ihDL>%5Jr^cIZ+oJaB6t&_cM#Pu%ba`Yxqvj437D{GqNchGbF>`Z@&@0`eU zI`I(~!aW_NRW6pDH(br7e{(YH`VKWt%Fj~94B%h(ZDXO4CC<+hqlV{u)YgB)&CtHi z&l|dxpFi4%S8OzfUphr!wB411{y$8|F;}Lse5_u5&fvnj44p5ZHTOfu0n0yK13>*J zdj2#tmP|Hpsj%d1FS2|f|K~9VxLaK`xdp~{7^j!%xM6Ef(>4DjC@@+sD4%Z7wr#qa z|L5@kvx5|5Z}A(9OhgP~gW2d59?=j-nkW}@C7uYZ=>2bO;tAJZ5Z z{MV!nkP8dUJnM?=G`6SXjHL(of6709moFnInLLu8Z?wNUMe}pE<}{7-ogjN)b6w$x z>22GlYCOdMGw6S%)n`nly8jiS>tCw)SEYX_|FeSRFd0lrW(7;h>`*D08zDmvMg>vz z-C8o|ELl7EoMxD(%pSdVuF(AhY@Xdr`u?oo-j$1HTpai){x>ZZF`+&pJ%2ZGDRMNQ zQ~vuj$y4V+sislZQ_F9AIY@*vgF4M-g>`aZByB|xvvFaU)}lz+kWd%NbYDmDVsE=H z5pu47hy95C4Byq~>N(bD>%3A&{V8~44P0X^+~TG)UvSek{|~(kd+YPZtZm!vHLT&F z(*!}j;mNv8&aJy=nLj4i(_i=e>^{{%Y~QN^FV`1Mp3zWZqPacY6#HNPRsT@_siDuP zZhgt5NSp)w2F4Dnq3|8w6BG5Z%yd`cHw`e|J!r&kNP3>YCmff;icGXKV9+#-6IAc{8gBH z&&l|pi$>rd8tCWwe^vZjlzx#!1WI*6e5Gm5R??5?*m)lQmE#A!pdF<9W(12y%#G<} zV5GlA`LrUo=^PyC+2!o-qS)eM)BTRKmH0burnVD5Q}eZY#xoKmzKCce@aG%xptH!|Hv6iX(Ct<kS4JU3o=Qa59JM5mse!iUYN%_a##%~&P^(&+mV+~shMq3IqwCm{naCf}) z`yCbAPjrj-r(+23|7}f%{?3#6<`g_-D~DV_vk0abv%^+ zaNizn>`_UAetCoDP@l*Dqj$?4Hyi2u*w4E((x+1^4(l3^vW0UlYs{O`Gb}Ew(*ph4 zoY1rC2Ic>LW>tJ6-||^)=D_TR;z|0QI~|>QSqANhTkq>4yL1$9B$29oyeldZwOCj39sy2Fxb z*=9PPmpO0`jqU3&mTOZr{#lbfV0+_^31i#$TQI$qK}`Kl3(|!1`kK%?SFL}ZO#f5= zzZmCk&k2?c3wDvFVGZh2tmo3&(0}^6i1r72bqr37YhoVe|7V0=)d~-|G|Smqex1gy zRL8Sp;9=`r4|rac&&1ff<8CKjJTjS&4kFaqGx|JgVqRr@auMMgy}J1j?tPR(XIwp(*r%EtxOPixa4^X&cpa$o&2WO zf?3z&U9k_H_GebHd~bc~@wE@C-$7pIwwJB6V{Wk6WqyQM<8kcw|9NeOwiEojQvXn2 z4Bn~HvCzjR?N9OvFMCjWb^0*f%5}rtcxhUl&@IYyEvSa+d;PQA($AoE5H;w5Qky$? zJ+EMtHuz^A<^SKTPa-{4bG4c(%w2hzhF5s0oSSS-Tnf2UyIJR@mhvf~qP-3}ybMc2 zUWVZYe!AvARJkh5H8@k7!Fj=h>Ke;UY#X||1e zbo}yoDW^Z*kGn6-(|4%L(YXd4eVg`H;6H$W+cm5jT<2>H+d}o7aG?8e|z5n7uB)-KX>UMqKG9ntn}V{@4X3Fu=if* zT@l3|W4eizWm%dE%CeOO1vRGUrM$en#FzYEx_RcM8DsSSox6JxeGsuFF~8l<=WyAx zvvcRpe&@`YGiT0WPVVy_jy3N_ezrF2hf3#wnhiSNHk6Lof9i4P$wzW0+3>Rmy@>jM zX2&$zOWREI>mh&GziWzzEvw&dF}Pu;wFdv#|GgLd=PZQ(T!?Za&H?O4?w9ZB`AD2Q zc{Mdc8IzUNsoNj^lcMF?F(L9O_~@Lycl^Wehn{b=mwXrODXmP4Rt#F2)q%72ch-Nn zGVnWdYAiN9jdLu&u(3sM!#^vZ^K`zdE*zYDFlVU#(<{upk7Vn#qAaWbJP7>LdjO-? zYx4EwH5Hk#zQ#$cfs5X3{y?|xa{srox`)T}a{eLu7nlGy^8XdyV zV|BHgap_Qowi)YztF;A&!un04oQ`cV=beE5->lL92WmcJiWup4E&o-z4~^p>xdFeQ_hUm9^OE74 zv~=DX;J*v{|G?XmSSpoA{^WTXjF!dv!l-EHTaOcs4>;ga3P%|4Wug^rM2r ziC(bfJ6f>k%Rh?pQLKm$XgBN+|IrZ&o$z3J24ai8L1j3n%MVolr{(FnM#Pl6?&lyC zrN>G=vQnkc{Rf77uH5l0lP5Nojd%w5SNW9n!aqA6fEDa7U8>GEE^OXmZwULlm!Qb> z6h9mCA6R=3_-F4!_3~WaRY9TgzGK_%$DY{bdPn>p-sC{eHOPv1IH59CkgNTDeX()nsa+PvO^>@dRu>pI;_U9t_3wSk ze{P&XFaNbR->6Hlaa?r$7AtM&KMc+Pb-Vqa(|;)b-|8$tepHts4V%W!t|-}O z$y!jq_>T$3{;$7eIrhIkg#GgF{T^TsrTe~KcCs&`d}VowAu@xU#qztxf8CCt!Gf}J zv-x>AR|VWqY;Za~ixXCs@_w$**WYuZe4^=*Dr<}C0wX?RP~G%@xKI3ZIuL8a$9hRx%=c9ObI-zIowI4(Hny(`iw2#m z-#9V|_-8cw|J~bvsQ&Ngc9H?vdOC__E;fp*ZuXLoT^-xjg8IWh6Ye9@5AhTs_kiRx zdX{*gdJk8QmH*iH^0AkmPYDx8XGDv2N>-n~Yy2O3B1xyVWK@VCU+*8l$}O_82Rpgg zTgo?j{#d2!m?K*(#?=%UzEGF<;PFrIC)fa8J>Nuhe2b-TU8M~V@}K!F`A_jLem;Z| zKNqamR52z7?R=#sTeX&_IJg78G*>yf$;o6ElZfjNy`;LO%57@(#xbVw z*WX>Ar*-AN{oi`6HF_{f_pL8T3mz8JxXAGa8ZyfAtE zY@IKtZycRZ==1c8f0Pjx=zm^cW|}EE7_1G_;D3PrpS|rrfu6@1kd3!@iXF$ zd>rJiSR3lu_xg9iOa+q`BOMXq-gXG*W`1GgE8ZpkRX!84k0bpR>*GTd`U|Fs?ppR! z{0pl58TA_Y1?1KGUO;6dhf83G%22x3E66o!sxPy55^Nl6#LpjGhCD>yK8Uq1r;l*s zO?|(sNAe6CUMw2p)cm+N5AvVs?f=tT|EKbw;=i`iio8;v&THB_)mBh6OmK7q^P_5C zL5+XGSiJ>Wmgzm4(tXtDEh5-FW6 z;a6`fdpFi!G&4D*wNLwhdaOhzIYJVS^IF}k?gLxSK+nEV{L}9M|KA6>OP);%Q;wQH z?JP4lvy05h=pwVz?~W+`MLR>a8_LJ|)fF1QUZ2CL>^dA)fE#K*gwM>26H`L@9eui^x*Qt-_?d@RkPxsvN3_jo&n#N0COyPkv_`e7Ir^P6k0C%Yi@@l<` z_=wjK-}AMEu-3SYxK_PB?Eg%BfK)#kd3a&F`519)S=&FiuD#Qh;vca}@g3<~F^E5! z6rkoElXu?BTptbF;1|3Gbx z?C0EAc|ccMCu?`j(fOjT)GWVcuhWQu;{PK>2}7xIo0Wy2!1S4#b-Zt>-)FtLZFH_; z3HK1-%v)X2kTZ=H;{#4U z|AwtLOv6@d59+GlJUR^c*9I}arTG_<*H1M^cO z>ZD_U`Zv*Mqr}gFA6&7?!Q)$}SZVNo@Am)cNv+5?Ax`zSmw)DgGv>jF8#E(MZn1iv zQtNj7Pm5Rb62cX>*ehFuuQT_J|84F}!!~!V zhO)k9{;L4NffI}j`1DP9nBQ?>wY|bxsjHu=7ec`@W z9kfnr!@)V<%XKAYdqsQfCUpzqcmu)zppj?=|GUNpwfI=O`g$N`b^wdWuFOi zf*@b(a&3umUSpMosRsY|j{l^%lfy&(TWKEQZ#?bVui!jDby}#*tYAs^?f(BvdQ7KI zV(6JfoY%Ub(qjg$|4_Xj`qfo`=l0`qfy$|f&!V+(W*50%{L?i6k0GS7%5Hc=@vt2D zLVPVy%Saa6EdJqFRy0T?*lpv%-)L?G-^pC$`~95SSRNMsX}vY@X|CRYcqhd}vs#~W z8Z=P+AK7k41UoF4nr)MY@JmdV@^f|mj`8#>U9(ets226s7aG5Rykc}}^KN?`4gT*P z|FJ>hNg-a+!-y~P1M&x43iT6LMFzDQuK+fC%l~=P8kp3`6UL$LqEb&=>7@bdK6HHy zqB#RxtR??SitJvKm3G1CiHgse2Z{eCgVj=|5EdQ~b9)m4Golj;XI0 z@mO`PQOVH_TK}f!U)X(SdJZPf@M2x*$b~0%SQ=~af6w@5!aZb;0Z#G`S1ZL;#MJpP zE>ylGEly!@JN`2hPw-Nso9+BvP6^>F3cJ55HYcas_jje(SH<`dU-5KoZcd6(*ezXf zLF+-|UsszT&g>-QwHQ){{oQX^JptIKSGpdp$XZ zl-_fDJ!~6rZ~l6%ui+O_)m&vc3HaxMnA`22ec}JD=9!Fgf3Vi^Eml+cxklUh+017` zxnPIA+2N9*Pb2@)kL*5hFWvoi z4Bzj2+6Kt&U-|2GK58r(xl;1D)6jRMs|JF9rg>kSR&9l^Vf99TAi4lZP}0 z*t&6#8tiA)kEI>S zADUBBJa}hQ31S)L>-@yxpU!L0gKE}mf7w(rszSQUc?|H+9NjV*`)U?MxXqR{?RFx~ zPq>rTr@V>y89yR@E&ws+0*QjsbAd$uY#@<79YEUl`H_>4dz0h4JW0bgFYHPCk;CPo z%;>J6WgE}mg8ymJ9lYdlWstX{tOI)duWbJ70oZ-m{Vpryr?-~A5$z*S zOpk2W-&KB*d06;wKb*yYc#S(8ZL14Mym)k-R*yfWY9GwtJwb`t^u}$L`pvtnqxl7f z=lE)$;|I16+{ZZL#sGf<@5h?_A< z-GY4GPjI&2Yv^j1VaKA_@A30v=N7DmYd7e9ASfI%Y@YvdX7{{QJ`bFZ)7O>G57b8F zY$5qN%J1~9X_03}El5@{OQv^{2Z?_MwjXURLJaj8VrIf46CcwiC-!>iHE$UkQ(s`* z&0o)B&jwTbANyFxkoz(!^Y;f4rJCBG3nH>-0*UxZe{yQC z4>_^hlQeGkAi^!)*L20%U)ph-5+TXC+HOQjkjSXC*7x7=&}vItVBGSI&KjtXc3fS-bER zme$btpWrReLfon8-DKgkE;29q44Iv9n#_znO{PbkM$D4aIO}_d_=mU0K1^*V^U~>k zPa}qTC-Rw|BQv6o8AW-CW_sGl|K(z(xaQ|9ktByp?G|Pzv{%jUBe zVQrNy^!tg!g{7k+g+)V-AfM9bh+ldIW!ZBu6yF^F`x;xfjsf0x{9NP1jg^z_x~pTf zCC_>j$@9>8fOq)|!K5A30qUgJ{lP@>Y!Hz=6+l||`jTc~ziEd%soUy7_!WNSKxrg- zDleVvSi6{PTt=~9i5SXHkVR?xf%PLKCA=2YMA9Of@t*2Ct-w7f3D{2-1LsnZ3?wI6 zlz{!LpZzL>n1Z9iJwNoNi{_(z9gwyt8C+)>&Y``@XPG@lR#H4%WScjd*0taBX5YNsk0N8eDQi`p{ka z!`(pn9RBH<%Olx_?>1CTohv&MHmv=ik8Z=(u~XqIosAf{XK^Oyzp(AIx(&C-J7Ap* zg3=-6FDSSst*zm8_ZdBw3ny5OSYS5<-L|E~p9kNBpQL z5t)_g74U;G-$Uiv!dHL?L? zU@hi>RdZe<%kcT43}7P_*iXD&?o;gdf&U2DKO=DdMUDORXbG^7@dxa~mcyW|c^R=C zzW&ZKkrQ;Jf$aBE{3Ev{*1WA>g}aM4Wk$4*&R%qZw-V2`WY&4IaK<_8d7XvNL-#%T zy!G~x<3aXQy=SDUHnEgX8DIV5 zRg>-ON(S%5*-hl&R>vcyWhqhJMoj}6XX@`^zwOuRT>bZI%15O)>~Ymv&6`N}Y*;`lmQ(C+L*A@s$l{D6 zWL{c5i3@HZp<0Qbo0{OZ+$ko)& z*bw2Zu$-UJrL`ynV|vDU$oz9;-UGru9S1tv3^5(7MW&9^BYuv+ey|BynmmjY%pO6u zE*nK!_xK=&prbD0#`+!287!z-uXS0NO|HUE;?%LtQ#=rZ%CKqkqp?d^5TAyP->pJJ(c0c!~_oB~mWF7A( z>;s-?+GgQ&T9~HYw#SQ{Js3#N9tbAg7zZ8uLy7pQKw#g8G*X=h*cWbbfZpRlUM!6! zPvp)el`ARscaT*x_d^e<0d|`p$4?P2hbH3VNM*m6EQT&UH$h6)FKHv0(0?)#C7914 zZxLU7F7zB$=BFz0Jy3E7nUCLV#oSk*Q~Zq-LB1BQ{u_(`Y%2Sq_pE?y9vJpH{422* zXeXJ#erA-MjdeaPs*R*aQ=5|va<-E!gPy%=;RVpk?3!ZbjF-vcIj5K?pVkS8-(TZq z+5Ur@P5Zx-!{j-0QshHQa^GN{E&GV<%>RHXS@yPWQebC{x4rDV<5VeftKJFws(miI z$!$O5Tv%svcvtM|xo;ahUG*Pk*}Rv@qS-GIHYw_&g><72dsbeGafih z*5+t`)l@oSwd4s8Gx;;Fc$nzM!YotVJWix4%SL$BZ<#QZUovzR;@EzOdFX#)_TCNF`z(ul(y>&7c)EvjjnCI^ zoDd}1<)-(b@bBw%jOXui!Z6lfqMsR|&<6e)$baS$lK)J6sCam|k951Iweo8(Yxy?` z{>nYmqdG_AtU60&KT8XyT^JVX+rHY>PW&(A5x(Acd}lTGX>K~2Z`(q0-D#@m<4{*= zVOmJ%=+z7V#;jWS8u>HeA7>HCQ_Bgne}jQ8ztqgPvC!lYja71_XmCc|c85Vtn=Czr zc_vCBVoyD#>Q(NpJs@q?a71jPkM-cREaMyZXO!pYwSD z?rE-hI|*z(ZPy*9wtnqte`-&dXX}J%!AjoJSy<0MWc(ZI^N9HY_ojPHwZEe4=FJ^R*ka|4YZk zU9E3#tNTvvqW6aX2jUUwUufAnKIruGKDy^%1L`^uL1fQ{k>b-aztn!7Q?Mbv|_e-DJh|gJeF|yt9(AubTwhChVDW63^%)1Sma&oMkd!JFz6h zU6d0a+UkkCwMH39?YtF;-?td{&DlvEWM-0r%t`OW_vl&xb|0)gbC$nNDhmEV%Cp}i z1uNeoIj{k(gDq%f)_LTa7|6Ppv;TCcbpNP!GCdmle++Ct(EX=Hi?A-2z~-Zb+&xcL zFM1g{;$Oq{6|ywFi^+`YG>Z@D*zRhf{K|2P>~gS&`0?0a@#wro=a{wg&y%#M4qj5o zX&Zk>>0!soq8~UP=Ueq275-VB2l#if5#Ml_a_Sood(n##f$a|KSG}cOTJ!-^l=pYC zVbxn?#iCcS=1}YY55VTb>HhROe26x2vo<7YAw$UO^xzF+xI zzfbM0Y;2a&p-IhKEVF8|_1=Y@vS)6&Fa7E_#sqtBI`%ebUuq~Bg*jmI@K+AUF`b8^ zNy`%|9?!3Kl#`mZew2rip#6L=j0(!1@;c3$xQY z$=swaCfrLhF2GrG!fCPuelU{HJsiX>zV70cDKQF*xv8DH^Wx7i8PUKcU4z2EVS0j+ z%ueni>z2F;?Ei~AvF&40vi5DV0kUx&)`hF6y&r4B1=QyAKl@x9-fz8TAn!F@_s)oE zCo^J!|5zDGi8x8pV_V^WAqU>ilH9d#lC?`-BROl{#JU^n&y+K|X`vmVzV`B$9Vg4L zB6h>uk$!FAY4L6Pi?hy<<#R7EsZs4?g50HLh-v;Ma>T37w)f%ZKy5%2|1@5%i?!$n zoE<(C9i(twG5-y{C-#29P|Dx%E?EKHe<8*;wfW3}U8^^CAH@GQ6ZKi_XU`l#wyhXV zp3WId4ir(`k0&)7#{=`@3C@%M57lk8W14n(X&rka#I*W}gdtDtovq!pb+Spr#?d7; z>i8q94)hxs8;A@B{(r$bqob~TRMN4HX8f95ogW^CovE+3&@q5mLOA5!GJG+3CEIUM&q#Y+SqJgD28A**-kkkY*)`RV2 zQ6{xdz6zUvyIG9CqRPcW{U z5$k+Lv>ZN2O6*A}NCsqnd`JW495I=jc!q2$eUp?GeMnZ!ewh^JzfI<+bTg^JO0#G$ z#pBLXq?hd{Nxq5n7Vpi7lsuN6)W#I7ew|s6dCn}-SH2Lr;opKT->1EYjq|`_AHEF~ z`-mCs>)h55AFl9Sx9m0j!gYUV9t5rjTF1TOpN@&+$Q9qTH;g&5H<5XK?_B)vsV%IS z;KI)}l;LdT&lErRqoc4gnEId=o63%Dom5;`sQ*UKx&n3gh1mzmWA9JrfNY&lPE<}> z*Sg!u1o&s}7ynD9pEF7h?VROdE&UK@U|D-O;s#!Kww7Fmf0#JHRXQg!P;8c!ent!U zXZ{5EXXYe#8Ki`DB>37(Uv-!wx{C9Q7vckDA**J!>*mf8GppyGH%1)vG%p)*7kuG< zR_$}$Ztqd~d{W;x8Ydd(c)#^`lhh}Mw+F9Uc+Q|;-5b9v_URb8AN=1wFC5$F%QTcv z8Z0atwYnx-`y)H+R-9_w zJZ_TcIe*^$;(z7B(^^Tv66X*X@oAiwy+&;#bibdM)_)&5)$RAV(|g(my511v>b!G&^O$ha zgV?34bA_ZR(LlsiH_B(|yHN*&;z z`GfQyCN)Af7&e);F4k=q?WRb64EAh4mX*-$x-0i>ot1OW7)JYwgFUQTPq|vkeq!zX zx1DKG={zc*9>hDLzRVwc*tgT}vPFY?6iv^NlzJr;im zIeHN%`lcTmaLegK+;t%MyEzViU)fLF)`S0SozGgT%(KOhTN~dm{;3b!ybQ5!T!1vt z-$C{c@>gAVNA5T}53u$?*usIM>+t*i)Xhq?D>3A>#frIa=m7uB@2vmhjMd9}>CtDx zeQYF3hskX}xmdNn9TlidTb%W(@#4%*L)d$Q1Ki}ta4sGDz$*Qww{M#}Yuks+$GE2J z98VkRJE0Ep;xJdy)Tof-+B1^dnfV#re;NGKIiUGS22;P?dW3M}W2M-){haC$+!-;7 zdo~7%dMw-t|9^(hSsf@x>xaft(}Rs0$BY2}nfqD)Qv4UMeu+sB>Uu27UHQ1XmG~Q1 zowl{Njh`FKSkwH9ar}*!o%BSkZ%0~IeAlp9QD>OHGXCippmcKg6oba{;ql0A z`U=jStMbLG@jp=g=g%PjX&dMo=1`9Ad(Y;L&U!k3lHvX0e|jS9j|m;vZ&H|rc}ugA z$LbTR2YFx~K>jS~K`P$>AvrUMI7<@-gcUtke0Jzg`u4%v@^!Zp&iqFY!zo5C&_?u$-4pWirFFFatnWV(Q;RtQx{wiyO2A>%VJ)w z=d0{k@?SAFF8evlJEA;fi;@C5%%{hn)|s8r#em44i5@ooyKB?5ns>Wd3QEWBJBsrI zI7fD)fnr`g4?GAziQmHQgZPXcufV@BPwTsylEM3H%7>ZNRoF2g(wl&P2Kd(o{*8ct zvzP#>d6=KVJuILtGA>M+mKOQa^t6bx>EXVj*f3vlP`FRKL%5&ZJT62sE-_pl;_EJ| zakLWsSG6a=;a{~brgjy277+PPPNfAY!xyD=8mydtn#snPdw}?7BK>LyC59?u{ao5x z9j3JZ*UeJ-&j9E4)qbubH#fW1a`-B|4PEUjotL<^E%jZa?`a!hyZs5Vt={!@lI@8K zkcDQ1%7@JgRqzn|fyRHpd6+*JJxu&_^N0AkFx~pfDPGliCN2DRydN5XB@X}GT<{=p z`q1j=8LvF-MeDa9U(EQrN?Tst7F+m+-=y^m&oV1#caMzqZjT9d6PJW|NCo~bikG~d zr0=;qO8)6)C;Jfbq5kRVAbubD(%<%Xk#z;S%WH!?+IB{Gx2}osl~njR$=`?WfW1@B zM@(%OqI*U(4#*VIx4yQmhf>01uGllu$zFMZc|iDQ62n^t`McFexLS&u9H%NSIZTzk z>uD#;i}6*)c{_;rJ6W~itcwKaccJf7c@JW759}i@+ckG<$)|pf;+AM{@%(U4iFtaW zSbyFuF*9#k8_r>h$UL0woR|3LqPofn< zM0EsK$G4GQgUtL9IXwU4X(Rc}*FpZVucPum@QG*rW;mUiUR8QE_697m9+)b+YhHv={m5PXXUMzbU88Q>pzBhKkW_{v6ydWz>Fe!mi9 z_A>G~z7ygjI~3$0e$mrj{+f%m{6}h^rR`>AzZ&-#6JK~+i@PJ-B&FET_KWv#9-bI{ z3^x6f42V1m^vcl$@;egKRAFgWlQ(23{3<`FQ|H>CI?zMYT`;~LRNZvW;^*rA?P&Sr zq$69L^*}fyeJiE<%b4TfKjxw}7uy-9cqksE4N%lks$lZW@gS#06Sy3mDFH30@7uMn8 z?7lN+Pf(kqDL@M)l9+qoFOw$0Jr zqJSjdw&6=N+ja9-on}DfQ6kYZL5OejyAlui-NzuWMJ~?n{sZS1Z{`txfLMJ{nD1Q= z9jCQtQl$}y<`cq657XXSb9zeL{y~2 zwH71+Ny&zU7R2vq<4sR!Gm7zTwFq?=&j9wDTH+8V+=PHjjE68X-E5s%1>KBb!;-VFV#8T!|wM~cIdlU|5J?i5ae~w|GoHks-wTpS^VeoK5Qs4 zUeZ`P)Br^8M2j;e%-j^IR%)2sDAq^xSX@ABT%@~r2hL5&AoJgKv5{W_zObgGYXfc$ zpksi_f4WXk$$ORlPygrMtHuCy6S^k&!Pi0Za)gg;Q>33PEHR?fGA-`h@Z>0^L5N?i zPD;E)e_DdnG}^aiVzj5&J;GZ&J;=3fzo%{6Kb)=M(?Bs!=OAkZ_MFiM_-48zw=OXM z9en&h^hG|GAXmxrVV=?~e0NHme~a;)M3GMJf_4T(9%XuA<74CxBJCFoO~xP1)6c2P z)&8IX`>QH^^;#F-!`gt>L48IMr|wc+p6+|dQ~A%@T&?eV&dlKR{_uS#KHUQl@cI`&RK7m?V*PVxqH>wjb2 z`J-wa&~+x}0lFrj`Vvbjorzt4Qu~W~9$-JWLLO0T$u~ZZ(pQ2#6-R@;+6x2SrL+B9 zPNf8TNM?li$g`1i=;=U5xg2rWKE`*x$GYTK_~fW$JY9=eL)WpEe~tV8J-|_VI^0#X zKfzzRG$mBw9UCZpEZR?M6c;4Z0wI3UUxS`29ZvorVqPd4sfBZ8<8j6|N0_JgH=OCc zN_A{*Pk@!(1JQ{nrq%KtaXx>k&M|oJXu;53jhn{JZYX_hNo~$he$58m&wyWcjNJ+6 z{e4c`CdkwOrnX|z4&fFXV-Vt=+&wK$ZeixcinLNfr26q;64RteiCv_RBr(KAk{jqO z-tXriZ$~`wH$ChW{{j}j!5aBzS6c~PH$Wz{#Oh6~{RLx$;-Ag~^gU$vHLPK;fcKr3 zz3gjmJISZ;G5x~Bw*6bgX#3ICvi&N`(fG5}-{L!08~JCLYu@#75O+hjsSop#Y>xJq z%uWiHdZq=5M`nac4Hu+#YA?-FGHN2Y{$;3kvo+4{{V^omZp#R(Y_x=%CXYUpJ9shT zN}NRw;wyB2faYl7FirKL{xHe%)%d5r*44Rs|E@0|S6aW>a!k_}OM`~;35GR=Hf~4r zhE^j^))jV4-Rm5H?;#K1r9(xdw;d>%U<@MnOUo80nXG9tt@uERUbvgYC^=9zYDPqt zcXDu7Mv#|eowrL{m5;OZFk;c%5nGINRVnzudyBerJ$Q+lw=qno=J|P&!q=wshD^ zoMr!l(}Vh}xRAUf%(6l18lUbT{UMDm1tbT}(*wEj-b#QEe z^0B1w4%?(?nSXRZ`?L@r#mW%Bw)`+J#in3aMHT#l_Ck+(-q*GD1s|7FFTh7(e}KFA zX+KxVju7wm$`C(Ead@C;U2L!m#@SWJk-CDL1B9`4yXG{P5a;UKAgT4bwEW?U0* zN!AHw^}>^s$o~c6aQFvbnOfwa z+h$dDVz;v~2r-`@9wmgGV3NXHn0X23nXHs9US_IXJ3dsY6Yj6j1D^HagQ*|psW3xGLek62maxc7yPsHvOouR=vasiEeW2N$u1)iB@We zgf}hZ7?TlNPl^0rC96rsBspv}&QVV$6>F@?WfH!e(c3aEl8$wAFk4!}fPM5A5E}=G!#{WJk<}djQI4Oh_o1+A{L1>L zTutluIBPX+9;YR*8b7(I*!aoXTBj70nF$RAB>7k4vj zmz-hNE|)Shrq?nlsYjRvbDEe{3uMgNg`JcT7hOaB^}(M7|J)cmg)z{)!&UoOmHm|3 zqQNT>SFxQBeF)e?92Avps0aUQ-1q23c#k>Y%8`77b4QAYZ4g#YipCk7B;+vIRiCT- z0oJeV`j>vcXYBML=llDN`atswwEiiqFkM);b)*4eV31?*e`wyOp+~h3{%rWC7*i*R1&(Zx1aZ}FM<{P|@ zeYS6C{xEjj0PlBPPxmt>==b^gx_{?ynmCKU#ZF&?|3`n^{~7U5$3We7OT-zOh`ijk zI!#+`toZpua*+$Z9XiZ6unqmno~NX8oFfji>i1MO?)z zuimOIw@9k4bl0of>JItu4*9R4N3{?BtoY~jA>@=}j%{<(s^4roympg$n4o0nj=BP) zH(?|Cf#!JW!9Os{+KMQy>HoLW-QfMUvORSH|AIW-SA=Ee!F82RI>)v;lV*(7liOU$ zscmi=dIb9*IeauJTw+Of=VRTw(TTiRN!PrtSo0#61Af;(10CPt$23;CG1YqSeD5x?MN}@6=r7ti|GfyBlfU zj{J}sdL;YcD1N_3hMvM0*lo+yRk`ZbSK8Xt7n^1w-`C-~eBFz}9Nv$>r%K0%oUYAd z*fYD_oY4Dvw_||AKi1a2V$8KQl#ZMr+Tp?!-qMJ^>JP47(>hh zcJ-yh=GT>&9u(vo{0)0x-(gPZStoOI0T+Xd!!mot+%S-}F!vq!5&ck8IOwpTc<6|> zU49IRXz0=IgBIZbQKnP-Jb5Rda2?cCWodD=Xn5k$V$)r z)&i7z+m3F>fA9H$`=6U9*s^pDovZaFzu06;b*bqP5Yf=1-v^H}_G#OJe+KwRJa$WF z|7Lsr1DmIg7nG0kYbr8ZSywPbP@Sj$ZcPsFKZxazvA|rTdBPe%z3oTbSfJ-Id-fRA za|1gT)NASLJe>~@l#W{QLiu=O5Yf=1-v=$*10etZQpDhSv|1`{hYHKb*c>ey7LD`j zIl|(>hlGVjuQV3ue$kN6T!z2(ParlX5vSkKE7gnB(< zATA5UGKWtt}Iu>$x*XoN0i|Q&TO&076;(>^U9{oP}>terW zTr{)4U~Ir?$x6{G2(Q-)nQl+1vLh93PsI8fzD{*My#+-5ti zr%J{d9WERTy^1?sWru8T!bZK!iB{nGMY9HT~J>-a$SAN$cm@L1I(fk4`_No(*v3w(DZ<&2Q)pP=>bg-XnH`?1DYPt^nj)ZG(Di{0Zk8R zdO*_ynjX;ffTjmDJ)r3UO%G^#K+^-79?fk4`_No(*v3w(DZ<&2Q)pP z=>bg-XnH`?1DYPt^uRs#zzxky(*t+i1DBb;zjB$F^!dR>LIV5x;5YbvR)q4TtdEL* zi?7#D`Mw@OiTX(&IBGlD*#S>T>oyDZ7n-oe(|x6>}B8=fSQJVnT1hsQ5h( zfU@R&@lmYo1}%RPfAHek$v#v0O@5i`15|;&>ctQ2;iCykJn%)rzSMl&7LRff{h;QX ztI8``{$=#Eny-Ff@KNFl{;2wc1219w{uo@tTUy?p4_&56EgMFi1 z5BE^L%;|en%J!7Sy~Xq%HczQ(@#<15wWQ30NPZlJn2 z{Q=mofRD$$*u%d}n11+LEFTTO2$+>|^;}c`_W#Q-V}CPtjNDrOCSU7jJ=aMf$1f+n z_^6-#4Nwn$g>vq;cz71@6uJZC>rvRCJRL*5>(T0m&(Hya^0>W8@ACce2^}DPl<$|X z8r<|}xut#*sATK;4ZeAKPyeIw<`=p0ee%(*z4&-26IFT2zuEsc$|snxKLGt}q73Zm zXpT?iPH(=_Ne?Z=h+z5VJ>@T|`2=?{QTF1m)nd!j@0eV?$-i9On{TqUr=n|=uL|*} zcUez0!s&_B6`qvBs=`B!v-3-ycTtJ+HcmT~+W1oh~#;Oc|V{`gw> zH1O8)eela^JuIKWuW_^dZ}Cl3Cq|)+fsGi*w~K_ijJGdHSuQPeS;ZfuDxhhRO*}-&if~q|CU?z}6w7im{884Msq;IaU zZ7pXX7av?yRYduLc-zBQ-zCnNQ3xd*0Viq+Nv)zzz^sGdJmKO*}PElAt0WUuO{!Y3SG)ge9A ztA80Zk1b!uRj+=p>VuoD#K+~dAo#RLdKFjQkbO+{m+z^F{Uup`4~zYWZAc${wxNCS z*#`E-rw!_hf3Y7vy>VaV=?!{*j^1tUkB=wovpjmHPd;wYH=o69?>ln;U%dJIaV*VM K(*t+xf&UMKOS8WK literal 0 HcmV?d00001 diff --git a/apps/mantis_explorer/static/pisces-logo.png b/apps/mantis_explorer/static/pisces-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..8980d6ae9a7c435e75e2ed59784cc6e01819fffb GIT binary patch literal 21465 zcmb?i^K%{E)4un{YV5SJZ8x@YW7{@w?4(I!+qRv?wi`8Stj6EZ`#-!pb9R5&nKNg0 zclONAK2Magq7(`u0U`hZK#`FaSN+fS|KEX!{ZFfI*}(w-YWXtaBI;fn7u#^jIiv2> z|2})bhCU7KY|?1Z1M)Ih5_J61>R(dr#%;%|T6$7f+4-|(N}HTLEdNyZoa0~lc*MIB z!N_UUSDVRbG_y&_PEkmM#2wrMPGi7NwH=^NAR0-~B`iY!mDGy25tLEi;?>SoRi_~g zGRpsh=r>>fo9-)2`}igG{22JKcm48tG1O8OaPV{5_VXzrUhj_iJ6FT2VW7`h=6~)N z&d*mvmT)FQw^zqEgpb6+psIGYcD!B{%s5kJ?2rxwwu2OtYQ$j46Mv-W1`-aDsaq7P zcRC2NSQ<>{g<$|%8t+*a8Yc=1Z0ll3b8DnXPpCZ)bdloZSB6KM1tX+HJG41*n2(6? zPV!c0wzKTG{AI7I$0(lq&q)PBYFq$KIFmO{8Y@T|h?MWWqzTUVZ_3V;UYYskc6ppyXLjYLVWw+q|p4*kp;R<((!`j3JP9s&ah((pn3ci1{| z;7;1Q3Hsz=0}ux|Gi+(qHEz#-Y{xO^2E${+?-z88+KU_Qp`=UVNCLwrX7B`tLE4TqD#qw&prRc57>lyO* z+nO);;d=jY;A}!rgnFXI=nXLd$|Gv<&~vvuzu#(*!(}LA$J(HbcqsSTjGWOc!-^C3+w&YK%hj)>V-iw$`JC&a{?6nWHOqD zd|a&Xf4OC4Vk3}O{$t!d2Yz38i^v)LNa(@1ayD!2{{cLhx*v@L$E5 zN@2VlbJ~7#n028dhSioF*Iq+!k`m6Uviui5$cRMYcICXPhB4~}bWp{1`;W<4%fYMi z&{}|z2;jb2@Fy{*i15eQq4a&wI8~sutXpN!fK`|Or%4~C%xT~_tqsXwW#nLRp@u49 z4Eba`{M*30Aqgo7d;i-$@&uby5o>g64Jk#)p749dy!+=ZvoksgoA8-`63%5wSj@+e z`jjpGprkNDv>JI1JiSKm6kTM?+f3{HiO@FJw_R@DnGd*EsrgBq>bqs;vW!oc`ddJz%E6!U z3#@itcO9Ap=kUb0Z3BGa7I^6qf)*lc{QA)0Xe>TJ75Rr$4h7C^Qzz*}&fHsn3!G?O z#qfvCsdE=~^EDP<$E@P)nv^Dt`m)uj=qi1bJ?76GD8Lv41O!BeuNs9`%6FUN&c((Z zB;NfLjP7nrV$5*Zurvg7(O2?G45aJCS|4!@zM>Rc_1mJ%L^L)h=ZnewrPl!0kE@Gd zMO_bItN!i^JxoFbKp>dhO3{ytc& zXGRzKNXSesng>MWkAEIL-3X9U4HZ<_%T)c|eplVm^y+|}8jyVwpREa~qwh1q#;ph> zYkgUlRP9>jRk)q%u_3utE7?OKDQt-ank0=(qr4of&3*Q6FwHscJeW6{4{)8sOn^~- z*-0{QMs;u0oOxQy8v-cOAW!MN_1^`MY<*O^RAd>i0Zd(H`3j$# z#CzT*i@{vD-~HMaPgVu~(NQXqj3DxTCY8{UkfqiSn^|BJM4lyUa`cCmReuLUqo3Y~ zYrI{NO@4AizR%1%ZohjMcFUF0h%MF;7;W1XcPm!DaXDX7sihKXn*}2jRVKls(!pDhY#5c~M0Xk=ONiF6H;V#9Ekywp?#jJao-wR4jnKze0%6~L(`bFe?nci%g>#EywQ8bqXT*aKMU+* zpx&Ri;QFfVJcIj5Myxeb0F${p{yMs#@?e>#WvJJz)Dz|zbL&H?N57zQhK#PBcKN$h zoz7*m<*Evz?zwh9B=u5(b_Iy>*v*sV+(BJ2L{vnpKuctwG+4dp(AZ%~2(HvIwB?@> z9xPJF-TP_4Xk1)89+}$Y@$1E$G3amIO5+>t;kuIWM2$QaJF6A|8x;*|n66(qE(2^@ zIu48aqa~1eYM0jDYbrkO_+}W^vtk$BE)mBoe%|DgE0rlgllDV1`5zscm;2A9T*`KY z^LQ+9C|#U1_Uvycr5%Oua`xVKofhhC4W6D(8r|~x0Zr(hFhdEVBjs%W#Oq=;{?MbJ z6rakw{h`2Bfx+R8>MM3W5VJ73y)?TE#|L05f`MZ|2CJGT;?v)x8CP5OYktdC&rM^C znoX|uU&xADI%T!NnaARD;n9viu2a-gxzXUKrS?zS5%zq!)M?Z~)SP=R_L zc6Z~B(F9)(KWh;EZ39VGxdyk$NjO#V(4ND7^rn6qXD zu1M!x1n}GeyDuPWqpMle-PWaZ)}6EP*Am$nH&X%K_UW0yoo1qjVu9=j7!43chltG} zttSMrc)waY$NjI!itGvbJzNfxVSBD)w=}2~!OLq^K1w$yUGBQ%c&89p$!gXP}{HDUm=74yqZtc;@{$RLM&%Q@wFCDc6>^__tFu~`^KCw>+ zD~J&*@3-Ewd> zRfbRCjz`Gb%t6Ya{RvstKVz*P7Q_@)JIf}>DnC9XTsCecI~c8&b22^ais!E{cIRJS zCFV8Z%7GK%++?xePye=%&u(#M(7c(($==91;hN_;a5Z8{N;Jwpn~L zZREEI*TMOs%sKZ^J$RzmCnUYU(N^@RB^-0taOz~SihZ&PVL_9VSdNAiP$sbdJ=qnG{XEvz7?5yDBZv!HK>fh z`?wd5gB@W}TZ@ynV+{LjE8JQjUN&>57F0z?EppA0(EJ^3<$SkQ^ce=8W;5>azbKYD)jJh7ig2wV zsB1hQg|aT8wgcW&e<^ohHo91{raT%onM*E(s>c2Xqx?&@)OE9V^AC_WZV}jWJ1yI1 z^92Zpxsn6kZi>s}xjo-(GnpTLP0TEGy)6vB;&r2&!9%0b0)i|1J7kyAN9~)QU+0I1 zsfWfru0&x~PdaX+#jp!VH1*@i`Zn-(^c>s1-PHWymOTm5atG}=2HOD0=AvK$!_Xuv zm0Qr7JH`8>{<(~>t@8~8$vi@7J3f#!1C=6NgO+)nliGzz<`hNGDg_`0pBB%*T%TZT`nC7C zPc~FUjUN{?_dKthvo_ctnAqPP8fI9$%ZW?Ij9e@y=J>3}XB)BCyDs5DrYS3hRoG_?#6EnuMW zZaqijA^CAj=lP?0!+vEDPWM8x^&7C=>#x%fS)Vw1SWkilbXaDkAZ0X?N@b!4J0Yy* z(GMPngbOJQ!L$eWuj?HO!8cLw^M|FV5-qYE72fnUGd?dr3JktpDHJRSwW%Sg1 zmtZh`zc>#}&PuMo0V-5{5F$y~{_}vA+sv*lDa=absLmmO2s)}i%Cid;dm=`@@(=hi zU*KI({rtat>z2=9!mc_Ztl3e3>VQaw@2B8u^*g7i?^LHt#Geu0TSbrQe~;TK>-9Ij zOqHLC5(BGCwzXtD_f{-$hEBoI25Sp=#$pWsgEYk}6A3JEPZ1u@p{P^x#&BjVx_p2s<|6B;tO8{{%GbvxfA zzDibX&zQMP>RZ?>_ARq~q@CcDbN?=w#vR#tZ?wEkpD8=uG!KRG-i)n(Yqj}Bu@PP9VaV6JA^vuxSGG& zkKc0auNLD+*NqNaUZ`@l3QPHZ`YG(pi!QVc0A{#Kls_`5u1Mo^)oSTeny|(H!aghS zX~EdlM&53X2Q60qtwb_zbg2uyjpK8ryUEG4V-BTVkPENlEZPy~CuUy+$wv@#C5Dd; z23oxyB^g!gFy`O4|N6#4An)4e;RTbzK|hP4hYGduB-;bMF)0CfzZEjA7wtp)MKdO5l-$-zt5Lxv<_Evm5x1V#aN09hINQ_F^#I|+K8Ib@i89(4H$ZG z5{wd1bFy!R5x~ds))7}!1XZ2LOmyVRevo8-wJ6Xh3Ds3z@D7`F0yVG{j+$w4ZeW}A zW>Srb6!B%>l`%$l9lWXj7KyhNLL-3O4`ZAT2D>Bgn=-~^k@QO1MJjIBx*V+zlUUKr z=%2=r)xL!I8`p64y|-qW+r3(J(qIZpOAiE<{cbH`_%tumQL=gP#qm*fcX^7*(Ytb! zXS|-68s#J<6Yf9kV@svjMQ_=RUeG5OpAVCPU{lZ;_oKFCj`wW}@V@W>5;z-0f#6F5 z#>rKrEGX5wK;d7KpMr%aF(Ql^4dmpw7xJ=(73K6@b{7`1aI4nfcq_b8=*}`OcL`$1 zi-9BBn$s`MfRNc3+jKMtzPQ^Tcfu(E$F;U|m^!#=VsVrc{&8lU~0J8TM4?=}_<4mo5`caK8C1L=x?r?!;nN_5v&# zdu09P{pj>Rn+oiRK>LhO3zOpY5x)F^{z~aAti>g(z*I>|A?oRupDeB2T2P;6B6$3a zw^J$4F*4>|!9VLc1NthKQGrfsng$WSN-wtEjC+~Q16tZm*a>YT;mdd*6!Qse!TKuT z+%9%GRLjNpqw#Ru5xo9QKF;$866V8rMY>NjWerbIXBqb0drZZGs7jB|?+2S%7HTl> zYCHgpDEKeMwPbfq%11M44i5Q{r28x^^L2KoEAt_Nv*vT^7lmFM`#@*YV(qT3-(bX} zY4-4LG0ETS;ZQxqfln+8HI!z*u~l0`%SL^*PI}r&!>f3OwmD>)a_$?{nrfxEZS43Z zAxRY{^=|(2ct2Gqc=eC6Kb>P7ao8{Ic`QiLrDT4NmDbZ&w} zQb1;-lywcpH?JneC+$*Eh&mFDj<0h$iG z67HBV%AGX~?{&UMtHB-UY}@=8Fi#raACd16A+?$$&~81C2Zkchbyg4l4}Ylndvbo( zG^{foT%dB9-l0Qf1c{0?2PpabXK<5uR^<*GZuQvT`RqnxLkxn0`s6cPRJ*@38M_hZ zhozokE8bI2J}VYweWe26_5BIsYrJ@Q7xBMmCz5cgK>m6hH%{*kV(a(ggv`_!+p|E7|(*<~k zNZM%I^ODz-CoSp6x$tF3ME=&utV+z_LAYch#ZwcYXDDMe;$6o(mm;;EII<%-*tb}F z^TZV&r+0e{U)#n3SY~^G{Vtx;qu!HuPts`DhqIz92jGZ3U&{E-AgV>zD5BdF2#ldH zf(7=o$09_ow>PHWgV)D8d%AFOUt$3Fyb6$~&lsUH9sAHPMEsqmrN@C4$5XR>Luvqy zMmLi9DML=J%R5nSMh!1bm?lhl+)_F*K)&+)8tzDP%g4#Em%X97T>PQLN4rlO7Ft_Q ze$8*RVz;yJg`uuwm4DqmiqYR?nzUb`Auz}}^U|inY1)}W=;=Z`n`lSuCR zo_Bh<{7PTSXhbFKXLkK?6aG#?*Oi>|(v(rQTJ!hF8>rcimgw4>gTw{w4&0Qs4_vvJ zf|>HgyDujVRbHnt08!$h&C_nP_>|V7_8Y)H2n(@mpniBgtyDmz4Q8$!%Sq!jHUhDf-V9=P*pXi*(E&O&%O%- z4?t(vTd$4o9+7{Fm}&wX22=I>23|Ryl&*HiCAh(L@ox{*xw$?qVnCzAL&_y#l~zD6 zW*BR50;e&!w{I6ECWPtMywa;*YltAE-)$xTWF?lM!s!TlFAULo+S6y6R(z`<@oX3o zBwZ9pWD}DC)0b-$1B79r2N4W7$aaWf-gITQ0}+Iy3oOA1zPc!}qE~Ya+ZN&!4zU6R zN#1`Cv0a&>L}KHSeX_1qM-gG{AxJtf8|<6jl`W-R z%Svu^or{FJx(Ig5km;Rqp9@I&fg;};W4;0j6~t+vTCYUk>|}tXu@9H)Oo(r&QQWv_ z8^zw+%~Zcv8TG!65a;u`eg3m?dfAbS{JqG6Xa~EVRs*)DQp4S$87IJN^xyERVcXpq zFnlIOa{8q@s?b-0=D}c8{Vh&4Ma5~zVh}OTFXh((=585Fw-S()CVw_ymD;p_iv0O) z-!|Id(?iyG{&9PC4n___rZ7FA7$Bx|wkdan!Xi)KMfL;gZYNRlPT8(FHRXuYz#GG? zA)5j>I#TPTKm+UjIdwkqs(VkOiw)qDictiRcKqu2wbQ@Rc^1qX=pZ&2h>rh4FZ_gF z!Jy=hBV;vt9~R|t9c9a%fRgZCP>5hjM^NmMl#2^)PH}PPvRVH8X#R^Lj-q0D4doMQ zbnhtF7|HSS9p_n$e&%Cu0R4M%kP<1B-OV24JyY zaeM^(v`)TITl^G7m_yM~59zU|;NAo4b5sD^U^YiYwXfEoG-6mK}{I8aos=s^4xhI2SNeIooHPN;q zn^5&|t~yX~l#Yje>Gfjy?OwNtuBT8N;cCmDyv7!EFN<%Axvf&s0ZRd^%%IkKf%Ul>KQdF9OO35@YJkS1LdsVP0Q_Rc)k+Yh=9u303;F+dWbkTHv63r zZ@T{0+lgoI$axX~3iuMNbLK3zYuv^L$`0<2l`ZC5$aj0L56-}^(21hh$Fcb0SnUEm9d64?niA;C+;&Br z)WTHJdTv`Rm>*4SUcc%s~~R-MS@CqOk(4|$Q%kW-Kuga#yrAhl}O_gBcu9%>T_LX8kNqyGQ9*Z4%jY|7PO(b7!`EPFvbw5~Ts6Dz>L(u>_1zxG( zCrAQ}&-WGIcdUNC`<^gg0Eh(%U~ z^nCHaTciL%%81)4hfAFtOob*lb@wAb3-sw;ISkb8_^#<{=|d@j(81!-Hu5j{2OQsH zMZVFFw2Tf|NMavHz*pNSBpOniKP(KqsgC7ZDInArFWqBoWrI7l*Y6gCGo?xSFkfso zj@`Xd(4g|&Q3K#PjB;93{R7n4z3X$9V2+KR9=PYMwpY}*>0iTMpQp}4S)@rV zFD>b$qX=#6{O3b(na|?7nGMVx5qPanrMr+L(W5J_?Tab*PSvxb(j?y^*@!`V?RVXn zk(cE@r&TH@(3M(@uM@lfSiV-=qDy^9fNFxU0D>3)RG`k{oR;*PXUsBU`3oOm1R^K3 zlN_bWd(RgPI*sovvSUKz1bz0kJdQ|KJ&%G3mk5xOEcMdzhc(ur@eSF!?hV{vCB6Ve z=YplNe$c2}iwGRZgUFqhN$WG9@WC|mF8%1a*z6=a2i5H>tqp|QTyXuy*gQs(+>@E! z;7e7}fA@{O!TE@#9J2dCgHn_ktq^hulk~0-#gw*fFUF6*8Cm=;ibztejznd|Uvsnc zrXHrZMFV8lsQ^Of^2)({;p$Q73R6zE$2NFuUX`q@`Oq*7*xJME085i|PEyl9{%8#F zO{33oPyt){&<~m^5+1gxkM`UZ=o&$~&Q>T^g=lsfWHIbSV=_iCG~lZR9}@TQW-J@$ z;9AQkDL+xSt5To$9i4?4MugnyU*oKm*`XmjzgMcAu3Yk2uX!1!DQJLDUU`yWCKXOU zqIxQ!V*Ue-rLO7BTtR*_g$OO;KXTSGyMtt(DDoUs!@ho=gIV?Eal7IdIM9wy5L=&v%u z7tkdZM4X}bavK)M0NHb=vkbhE&}UI-$Atth9&W!OqLUj`bvJliMY39Er+&G6RnHxAKn?<<49ZC zJ&$sP+4LZJKr@6MHqlE6qa_KVfLwj|k&Va_Wx3)jne^|&2kXTZVIss8>SyW4S#$M{ zezXHDelUjU^_v#nmLYj&czN#4D=QfG&pe7TOGp8jUDLr7CAc$KQaxE4AyI(_@tlVc zQES`;VAUJV2-5g9xi7^zUFRaQMAxaG%{(J+`(+K5gvT@!T?Ap@5BxRDFzK}DPLI!c z&W8z@kJ+Sv3nE8#HL|3Z+VtYm9nHEA8MTn&_xHcZ@2@eT0YOVHQKts*5ZuB+t@Hq( z%hbR_9OnZ{(5>4ebCpNmQT>dlmgGXfGVnXAXC)juJcLkn$m1#Hs)b-PK)yJnJj`=5 zZUkE-g>44&i5|rzp_PqMDS-b7jEyXc2CeR$aQf`c*saN8ieDr46p=4~Z)9@%F_Ui? z7BtdCYc#c=-{(q7zoQ;3H2__11?Y1ub`VPZ6VjFA*;AHqjsVRw>7$7^MxMaLfLtb0 zC*tl^OO?`<)Sb`|i^$}M=BAH@Fk|B8M>OrRiQc3M9;rlw7Den^JPaYU+px^B!<-3m z-;dR-1P=44F=}ATBM#x%LCYdjLO~jGO#xy$tcy^{0Td=aE0Y6wj0>11lC~<>;m^%$ zHS#!Q@X$RPf-f6df-fdziwT`3@sK?cmO0E44J&E3EIhv&?5v*f_xjI1*oUQ-oK?b} ze|XRk8k9n(T-&JM3^=JT(S=09$}IDAN|)+r=<4rOyGnIjrZR_LiDsa>8cnYB@&P&a z6tyIgG$hbuP@uZIM`}bUA>2j*m1wut_>~(w3)qy4`>LF~p0fwgH|?Djad##5FW+Z| zBW$pUOopDcNy6q_0)9&46Mj63Gk(OYEFSNX*MsSC(%{XP$(UE^I8@P=<{+Rc7eTb_ zuvIhz##4geNB|4^nr-ZYqYF_2b_UuEl&?x;wb}Aw5(EJhRwavqY-Q6dQNY{LU#P{e zs^LmPwPAa|{xqD&|H!i)$YAd&_c1Oaqlu6hhA29CZ^i$ihp~T|!h${O`HSq+AJG7FN*-*+wOdAxc;pJXC|z~Q`Zk{$10ziu4g{&r*pU%<};vex6LA= zhDq*j65TyUFzckDv5cbHp^#)Ade;xw!-ojzFUF)z-H%UM-oivB9wk;R9go`Xe$lcoT8gfi1~xbQE*gr_plksjj?j_nyl}P= zALg#B=&wA2(1o#ZK2EF{U-YEckov!VI*lR4!=(0NBff{;EiXTynX;(4Z}$(Ll%kohATOPwz|yexb)Qh1adwBN+&`2Bm1MUzK(zXBX8?eluN2G+ zoDH=^kK>G+nT{)#XqYG9LSs3^PkpSU@n()viPDidg{X^_hB8aFWipw@yDoIl0s^c&^k_gy~CUyn?R?jS4I-3M3(; zZPY1;#u@8N<`l?akXzgk9yZ?mIxL(NsMhz;OR`5G0aWQAAM<9TLW_C7qxAB!01^H|?+NR9NxrV~OS1F7wDZTL!`7?DQ` zD(q@Q!Jou#sDB= z&sOT&o{aMhx793gi(cnumYn*~Y?~_cSS93tLxdWnqfLFHQt!ix%80p{I+_)W_%i<& zqN80>hq*jW0-`^57Di>c;Z9=teqbQ7oJjqi;w}(W)-J(~{L!L@V&CEB)bz;ji5CK! zO!^z&fulD;r|45&=wN0Lqu^HFe3`2f0lj9ZHNytGQzeB;`NbQcvl&1sqoOUeYMt(9 zt?K`Q==IE-9HI+@M{3$*kLTVY+o$d&JPCPdkQ_6NgwIop89kGi`>7&W2iL#^L%YD6 z7rw*fvpgs^)bj8N0~uuXM6|_?<&3uc0X9L5V3dP#xck)#1VORK0rG6aT(BU4;a6N+ z=XXSQen<51mEm-hi)vCnf1DSl4xVE18XcLU|I$%$BGVI4z6xFoDh`5X^Z$AnDPV1q z(J<}XuzJ%cZ^6k=j#y(TyqPMl-+kjUh8A9#CidC?K= zzut$WXXzI3gGfa6Dp48GO!+o_UVe@o%|p>M1p2Oi1sKU zYEqP{p38o^&`sv8nmENE+Bcu+lEh)j1HL8*_Jexis7A=@GfU%O@E zB|@Ais#uUp7)h#u18C2X3S7|z1@yg)d-`3X$PTfnUj+bqpN1nx?S<2Mr}mU*ov-g7 zg6b(Fk0u{$b)7nm1Cg&e8!4JQ*b4Sq-du|rQe?gQjXS5e1Byd-(kR`WQoOB(?9;`c zcKTE4>BiHrU~amz0r`B+Zxoe){3NSHN~SBq$4k1I2~9jhec%v-ypk_O4)8DBrrs;A zIB$(;uZ@dhFQL|I84dt#$x5JWFZoErYLO;aP^tAbO!yA*Wag;%FW=xgPo*|}vmpJ$ zf7sKXm9VMtoHh^4j=S~F)@yZ!S;c~q=6YEZzCBNC0{+wwIDws7nSm0H&dz25SC~A; zyX0`y#D}XcD&5QVIna2B1oFmv*q;r2LR=Zpodro%lj9XBTk~wDLaPa_316$&IVS_ z6hQy)8?i*Nca+8UD*Ib{QbaT}P1H9H^<7lmX`25=MsAVf=5;zoJeZ0P-MHNV#uiIVWw==Q=JVav%U1uK~elyUe|ElNq+i-Thp z)0f!ENk)f?S0<*r9C;(i?>Ir7k#WljAAhR#2JIWShye`g!w2$H=DtsRNnm)Ognkdc%!*L&abBUnYtA%Uk#SzwhXUp;dw~ zaO|)GnJT%bUT4G3;lqOVPV$xuo;UIYuir{>IUSAr`&*f`R&(%5U5Lzg&?+d*Jyr(NWR!7p{FYRzS?M?|j!>CF2` zTzXK42A~3DeQh^LBY=|@K(1oDPZq<`EDyFLiCv!eC=VE3AeVH;Le$^hz=MHAMs9M- z&8uqvmd9zNysrv6o0vDK#bSWJVF7B$xupB7DQjM;UvUQwK!k&S!p0$Eh-a2^7fx_v zvWj7!ZIjrv^cokQbWz1FxIQ1ky`m0K#3^K^F{@vd_Q$$Q6w}erq;)Hh{u-nJlHV%> zn%`#Cun<)$Xkx#t4{7BnJY?p59V^nfN2}_U#)BU8c66XnCXe|$G4EUXYvTnJoOME``Mu01#)JMRLsoN=dnq2iqf5i9X3HwQWyu+RGp<%`;w*(( zH+_Icg*n76=!|Ad1y!11PivsXFDZnw7eoVRdA#Xyo=_Kd7XF)MWF>{Gmq2bw-KH9V z-v=mwQ)^C1;hqp3gJO&GpD9sSUy5i(dFLj8#sbj+pJBhingk{wcyTTE|M&xl!Dc3u zYGTS_oFKAt(A-gAJ|Le&Vt{IEQj5|Qsu^=y()LZk#0S*;g5dnO!a zTeN?Uq;G_;d&y^596GpK>0Q+Qd0OWr|Bqi~u5{>GG10-{8LBbNW_Hi_F)hIy6A!0$ zW|sSO(&S^qK1xU=-N~3dA09v7Ng_oSeA)T}2VR_tPI*M_QnEkq)|5vJ&-8${@t7@u zOL>2>w!&w^I%kGgsbyoQe*NWS>~Q#CC25qm{KbE`=2s|fqo&%C1E~pY&6TWrFi8B2 z_QbdL;qHgR>13-Y2CQ(tn&%rXHuPS%#!*o$rxBS#iB5IItsoR{XJ-(LNFOWgH6r;JbGIh4*}@rQ~-LO>Qu zp4zkb3N53;tc%aTF@@xkI;x5d8!zwC0rw(AvPUESpK2VlQD1>uuwhgr2Dz{zh{D*Q zLnh=)=AvsFQ~PFoS9H8PFx4Fm@S~d@0{=}wBf6(jvQ-dIE8Bt9eWR+GTwK52hmwV% zO%F>OxkeTrmPRbgS2FOK!{wwxO1c6>cfGH+SROduD*1hR%Xz!OwP)%kAC*UF4=2r_ zK*EE(iA@?f$TxrVR1pf)<-@2r*J)7G`wu=@EuO!aB(Z)^gWRWJYI+yTx$(VKn+yRp zR=blPV#u=^r>I#ALS!aryiJL#?-fMMj9mQK$7+DJyyWCW8py@g%K3O>FMs_KOt>N%$pOT z#6?G~-|lIL$N4%yzyGq2~rJs|Z^^qcUFg zCrpVy5O0(RU_mUabK+x{tAcNgj83lb`A`L~9MG_3b&p-vD@5?5-LUu6Gb25N#z--c z0Vv&dtvKen^$6jFgtZ__*vj5I<9F8dYqZ1S6U{L7F9zPZ2frc|s@~fGe8=pAgj2*7 z%wN}DE6Kigt3+a$L(woHo0u1B+fKW|QyxsvLVYC%__$z(we1zQ?WI+4@*xT1pNqvT zOvA-0VH~mwV<~47g5b-$TYKgnE_k+*G@=(-pXf^NA;P|uSSgCNH~XWm&_hAjXEbcL zB0bO0gm(~(Eth;A)Z_WR$1!SM&C7pdrRNU3(XQ%*_yy(5%_9q2)8LhNZQ5J=U$ElP z5VP%>C7p3FjPQh6X%Pz#0Y{Qn@m6*9XqR4R_R`Ml0vEv(bj>eR|EBK%z%M>)g4*hN zsh`q)qi1}|F!$Xr&akrF%x0IR&jH4rIp~8jG_~364nhrvpdh3Y1e30sti^i5W<P(T&?;XaHSsW2KLtT<_Xe=h1GP=cU&ahwZoHNV<@-wSAmbncY%tpRN`n zmXl;msnwsAfY@wgV$zZBaDj?5uBaE~DBymI>r^b$z}T*YY0XgLA4EXxLlL3Wfg=5v z@Axc&l7+_}FT{*}s%KSS7jr%yLJH+A*apVeelYj&n#nwcG@~75^8NjLKY=H9Je?K# z5`)BlThihA&(DdTu(1*p=JDT^n&)b!fm<6a-U+l5`+kKUe?GdB>~=rW3etLp07Z82 z{ZYV2zFhj6&LkzfFIhTo>B;69Oen3PkX@=7|xgnMSxzH zb2CB3DlviM(sx_1MZvLiECyz`?6h|W??ba|17MFr1>0Iwr0?7HJMwPO%P^rHf4%z9 zoHu{t@q9yANW_l%L+Xz8a2y+avARn7n=-*g!o4`L&)Z%^{ZmlH^Z3;V#OVtx+@l1; z<;b$pp1*qOCU1SW>UDv8-3I^>f<^TAC@~SvUn2B*#$COmO2mjV7<9c;mH9gz;b#Jc zS{5;{_jBCmq^N*l_7eUT#3#{cE2?_NH~LIk109lq;K&{>neq0j0-| zrO=%aHg#yNs*#xCTac+3&-g&0(nV&&b7IDM^Xez|;rjVlCSPPjBTbItIcVNT5a5a> zX?U!2pyuDWa_=KF)cc0;g8&3Y=Cos};(OQ>H(rqtZ{RyL|I;+AVa6dh^OjJjWt2K| zUW(z_Um1P|n-xr^JA%vXzGJV8r_O_)0#3m>GM!3?ii{TJT1CE!9U6xhJ1*XFfHQBq z12HW=sb&}`rN%K`eGLYf2gs#A1ky6yeGX_GIjVs_bc;hiTeGwLZCg29$@5JC^7H4I z@w#IvX^-D_)TbG{)qLLD@F(rj4o(k*wO4^G;swJoN#ed$WLE=c7Un&oV8(s`)>${V z$Sn=eKP#;&p^I*w5Tt&sgLnpaT-kZn<7pJ5;p60cXkp`Y8kITIO(|PnDk+`d12g4< zau@=gogD0Chg)^RDsJ42pIyLv*8AAnfb}ZxmtJ2F$2NfMp0(97!`>x!-OJl#yF&+t(MP?g8O3DO{x6E`$4C+V4Wnez4>We6{;%K{YZ5ENcX7lMy1vD`-I2VG8nym|# z7T@o_uRFcpmTOaNHqE9;?be!TdR@OICH+Oh+=^G}kv28tFsGf)Ljq_pd)J&G@y~N- zGgoj*)ke5KkQ8!&hC~7^%3BIRKdF><`>l@ylRx%fdFdUh1si5VA2x<=w9cRGif#m5 zRFh6*4!YP|T1NXYh?|^mi}2F9%^O0x`?PzifS?Ce)g-^sC0pY8AK!#@KPP{K*v2Yz zuVc{&z()>f$|!EF8IUr+oZ)9lGv&yC5r1$OIxro-KjMD@g8-VTVSTv}NurEejKI%I zl!GA@m)ZW6ArgKL{}2YZ58ASjG*=@s^Wb0rNYOAwjt600R#$tNP*DC^^TR@U$Gz*)R&SlO)|pnm`a!I zZXc8bUj^IyHrp(&072_!+3c`^u#%RWc{k-`SF*;QsrN4|aP+u9K3~sBBkA|z+3kYi zO+n@kbIqUwL}yj74PsfFZwaG8ep9W z(4%_lLd%Pc#i%tQ^rdyEST4b$!d*OGwR#%dJwL|hyw7j1?%Cntt|@fYUANg~ZZ$RF zd>Wgaz#aOvsD4!j`21-Au;0p3FR1SFLAC^FWEbr?5ddUEFh9NX(btJZRgzO2YUrLS zpJ(!MVhmhdLwW;`2~TYuU+`Zx+xoJ;Kg>K~tc%ZNaPBbfte)wpc>7PD1 zUeEMQVVAhueM_;pOR)X#pOaW zBy1<{7R`t&r~RV5an8Kg-mhcGhX5Pq+rg$l7fJvO>_gA&kh5lao2&7hyO)pX?3p46 z2d6w#VPhl;`M41_{uL^)yQX8qI=}72~)FyyZJlP3`uRmkUY7?)J$7*_DUVfWrkob%?z!^ZcO^L$cs z?U)mYvzT1HH>e};dsW5Ysp)-DLL z!`Gpj?>_0~^!u=2rBng;-I@iiRPw~BM}Y*>GOWwOx+2gw#A)I#RwLjS}=Ri2E%AX))T%msa;@Gn22$M=ObtmHYz$0e|R-2eZ`b0A5j=u z{BgWo@L`~TrLXRLUn_yzCsa)|n^oKAe=X>K6@zkgp_-=*Oba_p`OByv?VG9z&NY+L z5-4&e9lH5Vc1K1myDDbvpvK4Tej|tf{jDJco)EJ{bJ$v?XAOx;&>5e4+|y=nq|4OJ zUZ8kLPvw%rExwiUVcYf(`};^EgW=EquV40vfy%~l>FHG&8H`D#eUcI$gsc%HI<(Fh z!nVv4)#@>kk=P1XE#FyQ$mJ}AXL{gkH=zFk5MpnQ zRYEyMt)Nzs*sAuf71X9`?>$m#Rn4jq#NM@6Lu-#i>>YJ#*QQ2|&wu#d|M0xm^^W^~ z`HiMU@Q)p+OC(4Z(FuB8w$2Mohilz3=4~z0*`ux%kCnZWQack zq8S1gslYlekr7jwW2JYn_Cxr^ZU-Ao$;@{(c4evMEbEG;N%)c``e)8Jos*v2c+mmp zFHs-dQ+Qoy9`Eo{ktTSb?mkxyoC%YF9o;Kp()_lVUGbGw7sWkIK)}?Glkwid{!Gx_ zcD9)T2Np;}{U-+rrieSrUQv!K!=wsBoBPz0QMT<1?2@e^v_I0e`c+x)@kPJ3&1q3!I@`e=`3Knp= zd2Fl!we^1ya>khlhI$El zSMu#@tXCH=Z!>)?{9`^Hqy9U?4BxYd>3I%<J1Gb3`6Q3{4?6mGk@ZW-oJkFrsL~rn$TeDp1K+n}caOX`FdB<80md z6vDdt(ZA=k4EOkBnYibcrG)#RIkKM;K~~Of02C>Kw9KrAoY-R3EipXq`Ml5VjQ81b zh~}@WrSCm!6o5tbV7=;m_nrw)zp_VwFV)-WN1+77m23oMdh{tO%#%k=19;h;VfdFdIytKbwnkV6t1TY|Rh zwq%c*EE4l3902$mlkd5q3^5J&0EnJ&By|*MREi1B;Pvhg!d(<4z}X=ROAyq1Mo}->9YKe zQ&9MG7w=P^dm&7BX>jw=L~k9R~mBV^H(Y2pDrTyP>tRRWn7pBbaL7SwPf^o`V@ z{dY-8&yn*vZ}fDIk{{K{E+0>F402PG8n)<~Xy~6L8*s)LhvM!%G*MrvrRwc8Ni5H0 zy`hfLg}r!91zY?uYV2|@`Gff-hFns!R^s+92=bM16kt)e1`9GR|H?OdVv}XK&ss?> zbi^roJpD^G>S0V=d{Mh6yQI8s!%mRHJHF9gel++8dUt8Ea^w3WyVkUTQi(wu`*8Kx zFyUz@zz)LVRmD3GupxiVSeS=yQ_^8%3An>f3}^M(%4pBX7giMsO>=qwq}Fb|Aa68w ze7I%QHy*J^)}=;15T560KvixO(Nl!H&1`8b^0~6oH04Hb8cD*ml%86olYnRAPIi-* z)C=z~V~m#0J>(Rtk1eL0Zw$Rgde`Ifu^VZC5zeVB+sXrr|y|0*0?kMV*JF zuVZ!kxQrdS%s&t6Gh=`86(+2E)r8h!e}BA#@)nY7mb{Fy%s!Ad9sT0+j+NWo%8p6r z^ELB)XG!641(JjbICpVi+oR{{IFU@E&RG1~fsk)4M zub0;|AK@s^QlxqbP_LQSR2>BsQQ z4mm?!YyV8A*@u|VhSATc@&PVY(*aLM{W2>g(F>-mGNx3csXSyJ!*2`<3zUXt&JxE~Es2uAu#ZimQ3 zVc^Fc5yohIq*J*r8Aj0keUOtc3s10D@mZ4I8CdVVU&-)0&wqjyhc+q7FC{W4VHD4W z&XkPtzS*IFFY)caHs;Y1oqeqloQE-*I@?exqoWId#f6sy!bl2Po%&BW-!J(GpH#kS zSr%#iXy+QZ&we8yNVEL-*HF*GY6qqio(_)tf%WCFA z151WGpX^AKpvua*n{D7w39xI9eZ3@KJYK7tMr>Me& ziHvW_cDzGVNXB0MgJ)2nqqfl-MWvL=a*ta?Qn0h(8UvWa0o}007u_%E;TwDwBIx9( z9XFRz8iqsz8XAt-RCW{%AzNdC!EZegdXO(h;rsBO8y-R%>AK;s@@h$2^w?|4;(NeE z-^V0DyI++yTo-4}mb)Scy)1X@{J9au4lT;CCW|A&JlR{{LPx8S3LkzjoF9t>j=MgF_ z;vvEdo%=^|YY%^{<&AGG3dFO{KfIqY_IqE|fK1TcphoG)zVMGPe$MPJ(InP+nTxxvxjp!%MpVx&D>S~QI73th~*V`uBk9c zHYD*y`CY<*SEsXcC)^LRxwYRv~W<+)qE}4Ro=@T9tr~_~W z5=QZ!eSivOah7BgxbBp{Hu6j~#)#I6>5w#F(Vu6QC@wTC3XR;_Sl!=9`?Pq!3K>UD zBL0Fu2fkV){ z+I*=;Kuf>H!az_QMuDWeohy9Zpz#|a`{7zdF$#NQVDRqFjar3DHnYKhr|JH>)`JQ4 zdGbr~APqU1c7o`N?MFGmwl(JsUhYEcSUox{I$ zNa4iY17+1D;7-d?s`SL}`1PMSCD6IV2hgb9o(~(=Zz7BK(XSc4{Sp&eRQcV6-_?eX z)8W&5-7-|`Y~|uFZPSSVpkFNRO3RX#T~J#+ zI$s2r>E2tsEM(`${mfHr+>?+?7@;Tpop{R1Pl@K`T2gyhw)y(i1XqI>Yle#;3E>lG zYib3+Zn`)EbYlWP@>nZfU76p;#*n1D4x$*2@7b@2rH4?XfYtmHPaRcJ@vhq@R@*7k zZ?x6?#mK$i5jI%m3hWxRs?7H}N3Kb*V=$YU8LN5+3#S<7tr`NBoD1veiiwSAWx?51 z|}*M(Nemm2r?klV|8ZWpnq^d5nJ*RlQ;1wBrPeC$M{7 zK`$VH`6E{SEw^~AqDJ}L;{Y1hQ)F{s|K@{U34amkI^xe3aoO&W*xU9IUf`gp4&hFRy9XOBRVlh`qIu&OA{Aw{ zZw$ML%YHc}F(4{>pVNh*wEZ`T^lP4^;Y!?m>z^l6Y=8G<#1Ox)3vneV`!q|0!06qa zIfDdE#7=NOV}*_(yfg3(l_X0@Ho@PgV?*^Ijk@3#GYRnir@^g=^#N8uOIvkJY{}~y z(BrhYA#u()gGwyDjHT#L^jO^xcxJnMUd1{p+t4Y)(5aIL+Q1#&spb@G{dQZ%gSi3y z{1VR~GcvXmPt+p)>smp)l`Y;6CcRp&)vv@5lX0rttUPD5%SyO89=8uGpWD)|1wSz;IYxvp;dyoqO0MIb{i zvZ6};9ld(E3bI!uj>l=VP~4L851GBUt&(hgh9r?8KGw9Ejf;?j%*DKywM;vLn|hMT znAS;)l&a8+DwGEPd8T^}g(Onotwa*35-e*Nl3m?cWcDJ7$v&cJ11FFF>&snXfe9Jal!HE_I~ zQfQXQ{lvOjHuNpj%N7iQMgWc$QXPx`Er;aI@mv!PoMX_~p;}KJvsi1Rnf;EOQbk17 zS2$@#c59XMHEG9}E6CeUDsB^?T+8wyV#symKcP_ICT{6Ge8KmsIM$aHF2Pl1l~a zv!qO^owGm;ExjdaZeHnz@7Xgz?O#Z5R1vf3-p*BeE{bst6b>5D}Ud zIq`!i9wG<_7}SdcC#E`7x$r&p9&lo&=rDxJ@qOa#Af_8B5T>wSA7m9ZO;{Z-cbRYF z;pqR+Qwqg?$GMLJr04NWOZjVh4=cfGqVoA|X!D=L*T2l@Xa9J1`EpTB zkO@YuG5ynRhmO9?o?`-HJJZSudgNrgwU@8~8`MIe8B(Q~mY%}8fYGeAP|~RPR=?Sz zhwv!ZnE-@>Pksna*xCNwTrm}poi@%qq+#leWqcd5r~o!J@+9@a_vvkhxV*60w}#rY z<0bP=5h(bwgNA}7#U~S0RXqW0nFS9qwhuNKU-OV)`s-PCN=nOl0y%NkEba}d6?@e` zTCw3|x5`e;cK#uXHbz#Yq!t-8Sq7H#BAw{#s`5|Veug3myT3yeaws_>{W2^0|?_>HkXB$Osa&j8! zZ?v4(ZT1#Yw|I`Niw=5=YrHpL?N;o}|9`auZ^D90^f#^O-(EZ@I{`11wUlZUtRnvh DgZ#1D literal 0 HcmV?d00001 diff --git a/apps/mantis_explorer/templates/base.html b/apps/mantis_explorer/templates/base.html new file mode 100644 index 0000000..2ca56bc --- /dev/null +++ b/apps/mantis_explorer/templates/base.html @@ -0,0 +1,315 @@ + + + + + + {% block title %}Mantis Explorer{% endblock title %} + + + + + + + + + +
+ + + +
+{% block content %}{% endblock content %} +
+ +{# ── Ticket slide panel ───────────────────────── #} +
+ + +{# ── Student slide panel ──────────────────────── #} +
+ + + + +{% block extra_scripts %}{% endblock extra_scripts %} + + + diff --git a/apps/mantis_explorer/templates/index.html b/apps/mantis_explorer/templates/index.html new file mode 100644 index 0000000..9467e65 --- /dev/null +++ b/apps/mantis_explorer/templates/index.html @@ -0,0 +1,155 @@ +{% extends "base.html" %} +{% block title %}Mantis Explorer — Overview{% endblock title %} + +{% block content %} + +{# Hidden filter form — included by all HTMX regions #} +
+ + +
+ +{# ── Filter bar ─────────────────────────────────────────── #} +
+ + + + + +
+ +{# ── Stat row ───────────────────────────────────────────── #} +
+
+
—Loading…
+
+
+ +{# ── Institution table ──────────────────────────────────── #} +
+
+

Institutions

+ + + {# Hidden sort state #} + + +
+
+ + + + + + + + + + + + + + +
InstitutionStudentsTicketsEscalatedNotesDate Range
Loading…
+
+
+ +{# ── Chart grid ─────────────────────────────────────────── #} +
+ + {# Orgs bar #} +
+
Tickets by Institution
+
Loading…
+
+ + {# Timeline #} +
+
Submissions Over Time
+
Loading…
+
+ + {# Escalation rate #} +
+
Escalations by Institution
+
Loading…
+
+ +
+ +{% endblock content %} + +{% block extra_scripts %} + +{% endblock extra_scripts %} diff --git a/apps/mantis_explorer/templates/org.html b/apps/mantis_explorer/templates/org.html new file mode 100644 index 0000000..b823922 --- /dev/null +++ b/apps/mantis_explorer/templates/org.html @@ -0,0 +1,129 @@ +{% extends "base.html" %} +{% block title %}{{ org_name }} — Mantis Explorer{% endblock title %} + +{% block content %} + +{# Hidden filter form — included by all HTMX regions #} +
+ + +
+ +{# Hidden state for student table sort + slug #} + + + + +{# ── Breadcrumb ─────────────────────────────────────────── #} + + +{# ── Filter bar ─────────────────────────────────────────── #} +
+ + + + + +
+ +{# ── Stat row ───────────────────────────────────────────── #} +
+
+
—Loading…
+
+
+ +{# ── Org timeline chart — full width ────────────────────── #} +
+
Submission Timeline
+
Loading…
+
+ +{# ── Student table ──────────────────────────────────────── #} +
+
+

Students

+ + +
+
+ + + + + + + + + + + + + + +
NameTicketsEscalatedNotesTotal ActivityDate Range
Loading…
+
+
+ +{% endblock content %} + +{% block extra_scripts %} + +{% endblock extra_scripts %} diff --git a/apps/mantis_explorer/templates/partials/chart_escalation.html b/apps/mantis_explorer/templates/partials/chart_escalation.html new file mode 100644 index 0000000..2ac6598 --- /dev/null +++ b/apps/mantis_explorer/templates/partials/chart_escalation.html @@ -0,0 +1,59 @@ +
Escalations by Institution
+
+{% if esc.orgs %} +
+ +{% else %} +
No data available.
+{% endif %} +
diff --git a/apps/mantis_explorer/templates/partials/chart_orgs_bar.html b/apps/mantis_explorer/templates/partials/chart_orgs_bar.html new file mode 100644 index 0000000..d67a420 --- /dev/null +++ b/apps/mantis_explorer/templates/partials/chart_orgs_bar.html @@ -0,0 +1,38 @@ +
Tickets by Institution
+
+{% if rows %} +
+ +{% else %} +
No data available.
+{% endif %} +
diff --git a/apps/mantis_explorer/templates/partials/chart_timeline.html b/apps/mantis_explorer/templates/partials/chart_timeline.html new file mode 100644 index 0000000..513da52 --- /dev/null +++ b/apps/mantis_explorer/templates/partials/chart_timeline.html @@ -0,0 +1,45 @@ +
Submissions Over Time
+
+{% if timeline.labels %} +
+ +{% else %} +
No dated tickets in range.
+{% endif %} +
diff --git a/apps/mantis_explorer/templates/partials/org_stat_row.html b/apps/mantis_explorer/templates/partials/org_stat_row.html new file mode 100644 index 0000000..855bafb --- /dev/null +++ b/apps/mantis_explorer/templates/partials/org_stat_row.html @@ -0,0 +1,24 @@ +
+
+ {{ stats.student_count }} + Students +
+
+ {{ stats.ticket_count }} + Tickets +
+
+ {{ stats.escalation_count }} + Escalated +
+
+ {{ stats.note_count }} + Notes +
+ {% if stats.date_range %} +
+ + {{ stats.date_range }} +
+ {% endif %} +
diff --git a/apps/mantis_explorer/templates/partials/org_table.html b/apps/mantis_explorer/templates/partials/org_table.html new file mode 100644 index 0000000..686d6ae --- /dev/null +++ b/apps/mantis_explorer/templates/partials/org_table.html @@ -0,0 +1,18 @@ +{% if rows %} +{% for row in rows %} + + {{ row.name }} + {{ row.student_count }} + {{ row.ticket_count }} + + {{ row.escalation_count }} + + + {{ row.note_count }} + + {{ row.date_range or '—' }} + +{% endfor %} +{% else %} +No institutions found. +{% endif %} diff --git a/apps/mantis_explorer/templates/partials/org_timeline.html b/apps/mantis_explorer/templates/partials/org_timeline.html new file mode 100644 index 0000000..6bf3b26 --- /dev/null +++ b/apps/mantis_explorer/templates/partials/org_timeline.html @@ -0,0 +1,39 @@ +
+ + Submission Timeline — {{ org_name }} +
+
+{% if timeline.labels %} +
+ +{% else %} +
No dated tickets in range.
+{% endif %} +
diff --git a/apps/mantis_explorer/templates/partials/stat_row.html b/apps/mantis_explorer/templates/partials/stat_row.html new file mode 100644 index 0000000..4428582 --- /dev/null +++ b/apps/mantis_explorer/templates/partials/stat_row.html @@ -0,0 +1,28 @@ +
+
+ {{ stats.institution_count }} + Institutions +
+
+ {{ stats.student_count }} + Students +
+
+ {{ stats.ticket_count }} + Tickets +
+
+ {{ stats.escalation_count }} + Escalated +
+
+ {{ stats.note_count }} + Notes +
+ {% if stats.date_range %} +
+ + {{ stats.date_range }} +
+ {% endif %} +
diff --git a/apps/mantis_explorer/templates/partials/student_panel.html b/apps/mantis_explorer/templates/partials/student_panel.html new file mode 100644 index 0000000..aa1888c --- /dev/null +++ b/apps/mantis_explorer/templates/partials/student_panel.html @@ -0,0 +1,110 @@ +{# Student detail panel body — loaded via HTMX into #student-panel-content #} + +{# ── Mini stat row ─────────────────────────────────────── #} +
+
+ {{ student.tickets_created }} + Created +
+
+ {{ student.escalated_tickets }} + Escalated +
+
+ {{ student.notes_written }} + Notes +
+
+ {{ student.total_activity }} + Total +
+
+ +{# ── Tickets created ────────────────────────────────────── #} +
+

+ + Tickets Created + + ({{ student.created_tickets | length }}) + +

+ + {% if student.created_tickets %} + + + + + + + + + + + {% for t in student.created_tickets %} + + + + + + + {% endfor %} + +
IDSummaryStatusCreated
+ #{{ t.id }} + {% if t.is_escalated %} + + {% endif %} + {{ t.summary or '—' }}{{ t.status or '—' }} + {{ (t.created_at or '')[:10] or '—' }} +
+ {% else %} +

No tickets created.

+ {% endif %} +
+ +{# ── Tickets commented on ───────────────────────────────── #} +
+

+ + Tickets Commented On + + ({{ student.noted_tickets | length }}) + +

+ + {% if student.noted_tickets %} + + + + + + + + + + {% for t in student.noted_tickets %} + + + + + + {% endfor %} + +
IDSummaryStatus
#{{ t.id }}{{ t.summary or '—' }}{{ t.status or '—' }}
+ {% else %} +

No other tickets commented on.

+ {% endif %} +
diff --git a/apps/mantis_explorer/templates/partials/student_rows.html b/apps/mantis_explorer/templates/partials/student_rows.html new file mode 100644 index 0000000..db17955 --- /dev/null +++ b/apps/mantis_explorer/templates/partials/student_rows.html @@ -0,0 +1,20 @@ +{% if rows %} +{% for row in rows %} + + {{ row.name }} + {{ row.tickets_created }} + + {{ row.escalated_tickets }} + + + {{ row.notes_written }} + + {{ row.total_activity }} + {{ row.date_range or '—' }} + +{% endfor %} +{% else %} +No students found. +{% endif %} diff --git a/apps/mantis_explorer/templates/partials/ticket_detail.html b/apps/mantis_explorer/templates/partials/ticket_detail.html new file mode 100644 index 0000000..c0fd726 --- /dev/null +++ b/apps/mantis_explorer/templates/partials/ticket_detail.html @@ -0,0 +1,201 @@ +{# Full ticket detail rendered into the slide panel body #} + +{# Meta row — status, severity, project, date #} +
+ {% set status = t.status or '' %} + {% if status in ('resolved', 'closed') %} + {{ status }} + {% elif status in ('new', 'acknowledged') %} + {{ status }} + {% else %} + {{ status or '—' }} + {% endif %} + + {% set sev = (t.severity or '')|lower %} + {% if sev == 'critical' %} + {{ t.severity }} + {% elif sev == 'major' %} + {{ t.severity }} + {% elif sev == 'minor' %} + {{ t.severity }} + {% else %} + {{ t.severity or '—' }} + {% endif %} + + {% if t.priority %} + {{ t.priority }} + {% endif %} + + {% if t.project %} + + {{ t.project }} + + {% endif %} + + {% if t.is_escalated %} + + Escalated + {% if t.escalated_by %} · {{ t.escalated_by }}{% endif %} + + {% endif %} + + {{ (t.created_at or '')[:10] }} + + {% if t.url %} + + Open in Mantis + + {% endif %} +
+ +{# Reporter / handler #} +{% set reporter_name = (t.reporter or {}).get('name') or (t.reporter if t.reporter is string else None) %} +{% set handler_name = (t.handler or {}).get('name') or (t.handler if t.handler is string else None) %} +{% if reporter_name or handler_name %} +
+ {% if reporter_name %} + Reporter: {{ reporter_name }} + {% endif %} + {% if handler_name %} + Handler: {{ handler_name }} + {% endif %} +
+{% endif %} + +{# IP roles — use pre-computed fields when available, fall back to flat list #} +{% set has_roles = t.ip_src is not none or t.ip_dest is not none or t.ip_unknown is not none %} +{% if has_roles %} +
+ {% if t.ip_src %} +
+ + Source + +
+ {% for ip in (t.ip_src or []) %} + {{ ip }} + {% endfor %} +
+
+ {% endif %} + {% if t.ip_dest %} +
+ + Destination + +
+ {% for ip in (t.ip_dest or []) %} + {% set is_private = ip.startswith('10.') or ip.startswith('192.168.') or ip.startswith('172.') %} + {% if is_private %}{% endif %}{{ ip }} + {% endfor %} +
+
+ {% endif %} + {% if t.ip_unknown %} +
+ + Unlabelled + +
+ {% for ip in (t.ip_unknown or []) %} + {% set is_private = ip.startswith('10.') or ip.startswith('192.168.') or ip.startswith('172.') %} + {% if is_private %}{% endif %}{{ ip }} + {% endfor %} +
+
+ {% endif %} + {% if not t.ip_src and not t.ip_dest and not t.ip_unknown %} + No IPs extracted. + {% endif %} +
+{% elif t.ips %} +
+ IPs: + {% for ip in (t.ips or [])[:20] %} + {{ ip }} + {% endfor %} + {% if (t.ips or [])|length > 20 %} + +{{ (t.ips|length) - 20 }} more + {% endif %} +
+{% endif %} + +{# Description #} +{% if t.description and t.description.strip() %} +
+
Description
+
{{ t.description.strip() }}
+
+{% endif %} + +{# Steps to Reproduce #} +{% if t.steps_to_reproduce and t.steps_to_reproduce.strip() %} +
+
Steps to Reproduce
+
{{ t.steps_to_reproduce.strip() }}
+
+{% endif %} + +{# Additional Information #} +{% if t.additional_information and t.additional_information.strip() %} +
+
Additional Information
+
{{ t.additional_information.strip() }}
+
+{% endif %} + +{# External links #} +{% set all_links = ((t.dashboard_links or []) + (t.ti_links or [])) %} +{% if all_links %} + +{% endif %} + +{# Notes / Comments #} +{% set notes = t.notes or [] %} +{% if notes %} +
+
+ + Comments ({{ notes|length }}) +
+ {% for note in notes %} + {% set is_admin = note.get('is_admin_note', False) %} + {% set note_reporter = (note.get('reporter') or {}).get('name') or 'Unknown' %} + {% set note_text = (note.get('text') or '').strip() %} + {% if note_text %} +
+
+ {% if is_admin %} + + Admin + + {% endif %} + {{ note_reporter }} + {% set note_date = note.get('date_submitted') or note.get('created_at') or '' %} + {% if note_date %} + {{ note_date[:10] }} + {% endif %} +
+
{{ note_text }}
+
+ {% endif %} + {% endfor %} +
+{% endif %} + +{% if not t.description and not t.steps_to_reproduce and not t.additional_information and not (t.notes or []) %} +

No additional details available for this ticket.

+{% endif %} From 5b90f0fdaf39a381ee2cd4e2235b7739e1b6df36 Mon Sep 17 00:00:00 2001 From: liamadale Date: Wed, 29 Apr 2026 09:58:27 -0700 Subject: [PATCH 008/109] feat(hub): register mantis-explorer in run_all.py and hub landing page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mount the new mantis-explorer app at /mantis-explorer in the DispatcherMiddleware and add a hub card linking to it with a fa-user-graduate icon. Also bumps locked dependency versions: rich 14→15, ruff 0.15.6→0.15.12, pre-commit 4.5.1→4.6.0, geoip2 >=4.8.0→>=5.2.0, pytest >=9.0.3. --- apps/hub/templates/index.html | 5 +++ run_all.py | 4 ++- uv.lock | 64 +++++++++++++++++------------------ 3 files changed, 40 insertions(+), 33 deletions(-) diff --git a/apps/hub/templates/index.html b/apps/hub/templates/index.html index d75381f..cebb391 100644 --- a/apps/hub/templates/index.html +++ b/apps/hub/templates/index.html @@ -104,6 +104,11 @@

PISCES Security Dashboard

Dashboard
Analytics and visualizations across all data sources.
+ +
+
Mantis Explorer
+
Student activity by institution. Tickets, escalations, and notes with date-range filtering.
+ +{% endif %} From 2ab3d2ad6976ad96cf73a76e60916a4d36a8ed7c Mon Sep 17 00:00:00 2001 From: liamadale Date: Wed, 29 Apr 2026 20:26:09 -0700 Subject: [PATCH 019/109] feat(dashboard): wire sensor filter and date controls into toolbar Register the tickets Blueprint in app.py and add the /api/dashboard/sensors route that serves sensor_summary.html content. In dashboard.html: add the Tickets tab button, sensor-badge button with JS state (_selectedSensors, getSensorParam), date picker row that appears for Mantis and Tickets tabs, and openSensorModal / applySensorSelection helpers. In base.html: add sensor-modal markup (backdrop + panel), CSS for .sensor-badge and #sensor-btn, pill-style sub-tab bar, uniform design tokens (--radius-*, height:28px inputs, rounded toolbar), and dark/light color-scheme for native date inputs. --- apps/dashboard_web/app.py | 22 ++++- apps/dashboard_web/templates/base.html | 99 ++++++++++++++++----- apps/dashboard_web/templates/dashboard.html | 94 ++++++++++++++++++- 3 files changed, 190 insertions(+), 25 deletions(-) diff --git a/apps/dashboard_web/app.py b/apps/dashboard_web/app.py index 23ec604..e8144b1 100644 --- a/apps/dashboard_web/app.py +++ b/apps/dashboard_web/app.py @@ -41,11 +41,31 @@ def api_cache_clear(): dcache.invalidate() return "", 204 + @app.route("/api/dashboard/sensors") + def api_sensor_summary(): + """Sensor browser modal content — terms agg on host.name.""" + from apps.dashboard_web.opensearch.aggregations import agg_opensearch_sensors + + time_range = request.args.get("time_range", "now-24h") + data = agg_opensearch_sensors(time_range) + # Build bucket-like dicts matching the sensor_summary.html template + buckets = [ + {"key": label, "doc_count": count} + for label, count in zip(data["labels"], data["counts"]) + ] + current = [ + s.strip() + for s in request.args.get("sensor", "").split(",") + if s.strip() and s.strip().lower() != "all" + ] + return render_template("sensor_summary.html", buckets=buckets, current_sensors=current) + from apps.dashboard_web.mantis import bp as mantis_bp from apps.dashboard_web.opensearch import bp as opensearch_bp from apps.dashboard_web.overview import bp as overview_bp + from apps.dashboard_web.tickets import bp as tickets_bp - for bp in [overview_bp, opensearch_bp, mantis_bp]: + for bp in [overview_bp, opensearch_bp, mantis_bp, tickets_bp]: app.register_blueprint(bp) return app diff --git a/apps/dashboard_web/templates/base.html b/apps/dashboard_web/templates/base.html index 798a133..0051f6a 100644 --- a/apps/dashboard_web/templates/base.html +++ b/apps/dashboard_web/templates/base.html @@ -24,10 +24,12 @@ .dash-toolbar { display: flex; align-items: center; - gap: 1rem; - padding: 0.75rem 1.25rem; + gap: 0.75rem; + padding: 0.6rem 1rem; + margin: 1rem 1.25rem 0; background: var(--surface-container-low); - border-bottom: 1px solid var(--outline); + border: 1px solid var(--outline); + border-radius: var(--radius-md); flex-wrap: wrap; } .tab-bar { @@ -36,13 +38,13 @@ flex: 1; } .tab-btn { - padding: 0.45rem 1rem; + padding: 0.4rem 0.85rem; border: 1px solid var(--outline); - border-radius: 6px; + border-radius: var(--radius-sm); background: transparent; color: var(--on-surface-dim); cursor: pointer; - font-size: 0.85rem; + font-size: 0.82rem; transition: background 0.15s, color 0.15s, border-color 0.15s; } .tab-btn:hover { @@ -59,41 +61,64 @@ align-items: center; gap: 0.5rem; color: var(--on-surface-dim); - font-size: 0.85rem; + font-size: 0.82rem; } - .dash-controls select { - background: var(--surface-container); + .dash-controls label { + font-size: 0.78rem; + color: var(--on-surface-dim); + white-space: nowrap; + } + .dash-controls select, + .dash-controls input[type="date"] { + background: var(--surface-container-high); color: var(--on-surface); border: 1px solid var(--outline); - border-radius: 4px; - padding: 0.3rem 0.5rem; - font-size: 0.85rem; + border-radius: var(--radius-xs); + padding: 3px 7px; + font-size: 0.82rem; + height: 28px; + outline: none; + transition: border-color 0.15s, box-shadow 0.15s; + color-scheme: dark; + } + [data-theme="light"] .dash-controls select, + [data-theme="light"] .dash-controls input[type="date"] { + color-scheme: light; + } + .dash-controls select:focus, + .dash-controls input[type="date"]:focus { + border-color: var(--primary); + box-shadow: 0 0 0 2px var(--focus-ring); } .tab-panel { - padding: 1.25rem; + padding: 1rem 1.25rem; } .sub-tab-bar { - display: flex; + display: inline-flex; gap: 0.25rem; + background: var(--surface-container-low); + border: 1px solid var(--outline); + border-radius: var(--radius-pill); + padding: 0.2rem; } .sub-tab-btn { - padding: 0.35rem 0.85rem; - border: 1px solid var(--outline); - border-radius: 5px; + padding: 0.3rem 0.9rem; + border: none; + border-radius: var(--radius-pill); background: transparent; color: var(--on-surface-dim); cursor: pointer; - font-size: 0.8rem; - transition: background 0.15s, color 0.15s, border-color 0.15s; + font-size: 0.78rem; + font-weight: 500; + transition: background 0.15s, color 0.15s; } .sub-tab-btn:hover { background: var(--state-hover); color: var(--on-surface); } .sub-tab-btn.active { - background: var(--secondary); - border-color: var(--secondary); - color: #0d0f14; + background: var(--primary); + color: #fff; } .loading-msg { color: var(--on-surface-dim); @@ -194,6 +219,21 @@ .ip-table tr:last-child td { border-bottom: none; } .ip-table tr:hover td { background: var(--state-hover); } .ip-mono { font-family: monospace; } + + /* ── Sensor badge ──────────────────────────────────── */ + .sensor-badge { + position: absolute; + top: -4px; right: -4px; + background: var(--primary); + color: #fff; + font-size: 0.6rem; + font-weight: 700; + min-width: 14px; height: 14px; + border-radius: 7px; + display: flex; align-items: center; justify-content: center; + padding: 0 3px; + } + #sensor-btn { position: relative; overflow: visible; } @@ -216,6 +256,21 @@ {% block content %}{% endblock content %} + + + + + * (the IIFE at the top applies the saved theme before CSS loads) + * + * : + */ + +/* Flash-prevention: apply saved theme immediately */ +(function () { + var t = localStorage.getItem('pisces-theme') || 'dark'; + document.documentElement.setAttribute('data-theme', t); +})(); + +/* Toggle between dark and light */ +window.toggleTheme = function () { + var html = document.documentElement; + var next = html.getAttribute('data-theme') === 'dark' ? 'light' : 'dark'; + html.setAttribute('data-theme', next); + localStorage.setItem('pisces-theme', next); + updateThemeIcon(next); +}; + +/* Sync the icon to match the current theme */ +function updateThemeIcon(theme) { + var icon = document.getElementById('theme-icon'); + if (!icon) return; + icon.className = theme === 'dark' ? 'fa-solid fa-moon' : 'fa-solid fa-sun'; +} + +/* Set the correct icon on first load */ +updateThemeIcon(document.documentElement.getAttribute('data-theme') || 'dark'); diff --git a/apps/shared/static/tokens.css b/apps/shared/static/tokens.css new file mode 100644 index 0000000..3efbaab --- /dev/null +++ b/apps/shared/static/tokens.css @@ -0,0 +1,131 @@ +/* ================================================================ + PISCES Design Tokens — Single source of truth + Imported by all web apps before their own stylesheet. + ================================================================ */ + +/* ── Dark theme (default) ─────────────────────────────── */ + +[data-theme="dark"] { + /* MD3 surfaces */ + --surface: #0d0f14; + --surface-container-low: #161a24; + --surface-container: #1a1f2e; + --surface-container-high: #1e2433; + --surface-container-highest:#232840; + + /* Outline */ + --outline: #2a3045; + --outline-dim: #1e2233; + + /* Text */ + --on-surface: #c8ccd8; + --on-surface-dim: #5a6278; + + /* Primary */ + --primary: #4f8ef7; + --primary-dim: #3a7ae8; + + /* Secondary / Tertiary */ + --secondary: #7ec8e3; + --tertiary: #bc8cff; + + /* Semantic */ + --green: #3fb950; + --yellow: #d29922; + --red: #f85149; + --orange: #e3763c; + --purple: #bc8cff; + + /* State layers */ + --state-hover: rgba(79,142,247,0.08); + --state-press: rgba(79,142,247,0.12); + + /* Badges */ + --badge-green-bg: rgba(63,185,80,0.2); + --badge-red-bg: rgba(248,81,73,0.2); + --badge-yellow-bg: rgba(210,153,34,0.2); + --badge-blue-bg: rgba(79,142,247,0.2); + --badge-gray-bg: rgba(90,98,120,0.3); + + /* Focus ring */ + --focus-ring: rgba(79,142,247,0.25); + + /* Sidebar active link (used by OpenSearch) */ + --primary-container: rgba(79,142,247,0.15); + --on-primary-container: var(--primary); + + /* Elevation / overlay */ + --panel-shadow: rgba(0,0,0,0.45); + --modal-backdrop: rgba(0,0,0,0.55); +} + +/* ── Light theme ──────────────────────────────────────── */ + +[data-theme="light"] { + --surface: #f8f9fc; + --surface-container-low: #f0f1f5; + --surface-container: #e8eaef; + --surface-container-high: #e1e3e9; + --surface-container-highest:#d8dbe2; + + --outline: #c4c8d4; + --outline-dim: #dcdfe6; + + --on-surface: #1a1c24; + --on-surface-dim: #5c6070; + + --primary: #2563eb; + --primary-dim: #1d4fd8; + + --secondary: #0d7490; + --tertiary: #7c3aed; + + --green: #16a34a; + --yellow: #ca8a04; + --red: #dc2626; + --orange: #ea580c; + --purple: #7c3aed; + + --state-hover: rgba(37,99,235,0.06); + --state-press: rgba(37,99,235,0.10); + + --badge-green-bg: rgba(22,163,74,0.12); + --badge-red-bg: rgba(220,38,38,0.12); + --badge-yellow-bg: rgba(202,138,4,0.12); + --badge-blue-bg: rgba(37,99,235,0.10); + --badge-gray-bg: rgba(92,96,112,0.12); + + --focus-ring: rgba(37,99,235,0.25); + + --primary-container: rgba(37,99,235,0.12); + --on-primary-container: var(--primary); + + --panel-shadow: rgba(0,0,0,0.15); + --modal-backdrop: rgba(0,0,0,0.35); +} + +/* ── Legacy aliases + shape + typography ──────────────── */ + +:root { + --bg: var(--surface); + --bg2: var(--surface-container-low); + --bg3: var(--surface-container-high); + --surface-1: var(--surface-container-low); + --surface-2: var(--surface-container-high); + --surface-3: var(--surface-container-highest); + --text: var(--on-surface); + --text-dim: var(--on-surface-dim); + --border: var(--outline); + --accent: var(--primary); + --accent2: var(--secondary); + + /* Shape */ + --radius-xs: 4px; + --radius-sm: 8px; + --radius-md: 12px; + --radius-pill: 28px; + + /* Typography */ + --font: "JetBrains Mono","Fira Mono","Courier New",monospace; + --sans: system-ui,-apple-system,sans-serif; +} From 326b094d1f1f1ff3e0613b3b832bf377f25657d8 Mon Sep 17 00:00:00 2001 From: liamadale Date: Wed, 29 Apr 2026 21:58:15 -0700 Subject: [PATCH 034/109] refactor(web): migrate all apps to shared static assets, remove duplicate files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All five apps (hub, opensearch_web, mantis_explorer, threat_model, dashboard_web) now register make_shared_static_blueprint() so they serve tokens.css, base.css, theme.js, and logos from /shared/static/. Per-app changes: - app.py: register shared_static blueprint - templates: favicon, logo, and theme script tags point to /shared/static/; inline theme-flash-prevention IIFE and toggleTheme() removed (now in theme.js) - CSS: tokens, CSS custom properties, reset, and base component styles stripped — each file now contains only app-specific overrides Ten duplicate logo files deleted (~870 KB saved): apps/{hub,opensearch_web,mantis_explorer,threat_model,dashboard_web}/static/pisces-logo.{ico,png} --- apps/dashboard_web/app.py | 4 + apps/dashboard_web/static/pisces-logo.ico | Bin 152126 -> 0 bytes apps/dashboard_web/static/pisces-logo.png | Bin 21465 -> 0 bytes apps/dashboard_web/static/pisces.css | 386 +------------------- apps/dashboard_web/templates/base.html | 31 +- apps/hub/app.py | 4 + apps/hub/static/pisces-logo.ico | Bin 152126 -> 0 bytes apps/hub/static/pisces-logo.png | Bin 21465 -> 0 bytes apps/hub/static/pisces.css | 386 +------------------- apps/hub/templates/index.html | 28 +- apps/mantis_explorer/app.py | 4 + apps/mantis_explorer/static/me.css | 147 +------- apps/mantis_explorer/static/pisces-logo.ico | Bin 152126 -> 0 bytes apps/mantis_explorer/static/pisces-logo.png | Bin 21465 -> 0 bytes apps/mantis_explorer/templates/base.html | 30 +- apps/opensearch_web/app.py | 2 + apps/opensearch_web/static/pisces-logo.ico | Bin 152126 -> 0 bytes apps/opensearch_web/static/pisces-logo.png | Bin 21465 -> 0 bytes apps/opensearch_web/static/pisces.css | 335 +---------------- apps/opensearch_web/templates/base.html | 38 +- apps/threat_model/app.py | 4 + apps/threat_model/static/pisces-logo.ico | Bin 152126 -> 0 bytes apps/threat_model/static/pisces-logo.png | Bin 21465 -> 0 bytes apps/threat_model/static/tm.css | 136 +------ apps/threat_model/templates/base.html | 30 +- 25 files changed, 99 insertions(+), 1466 deletions(-) delete mode 100644 apps/dashboard_web/static/pisces-logo.ico delete mode 100644 apps/dashboard_web/static/pisces-logo.png delete mode 100644 apps/hub/static/pisces-logo.ico delete mode 100644 apps/hub/static/pisces-logo.png delete mode 100644 apps/mantis_explorer/static/pisces-logo.ico delete mode 100644 apps/mantis_explorer/static/pisces-logo.png delete mode 100644 apps/opensearch_web/static/pisces-logo.ico delete mode 100644 apps/opensearch_web/static/pisces-logo.png delete mode 100644 apps/threat_model/static/pisces-logo.ico delete mode 100644 apps/threat_model/static/pisces-logo.png diff --git a/apps/dashboard_web/app.py b/apps/dashboard_web/app.py index e8144b1..8b77813 100644 --- a/apps/dashboard_web/app.py +++ b/apps/dashboard_web/app.py @@ -68,4 +68,8 @@ def api_sensor_summary(): for bp in [overview_bp, opensearch_bp, mantis_bp, tickets_bp]: app.register_blueprint(bp) + from apps.shared.blueprints import make_shared_static_blueprint + + app.register_blueprint(make_shared_static_blueprint()) + return app diff --git a/apps/dashboard_web/static/pisces-logo.ico b/apps/dashboard_web/static/pisces-logo.ico deleted file mode 100644 index dd66c2316549dbe7302182a4b8cbf60f01d4f830..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 152126 zcmeF41$>;x(Z=^Zv24j~%a)nM%*;%-B!ev3GBb;rB{Q>(f+H~=OR~&PoY+YkYVx&B zlcsH;X*e2hlK99SE_;T1A^?%e?Kz#+&S3rFQ)K@@#1=LqSeFfB4Kz#+&S3rFQ)K@@#1=LqS zeFfB4Kz#+&S3rFQ)K@@#1=LqSeFfB4Kz#+&S3rFQ)K@@#1=LqSeFfB4Kz#+&S3rFQ z)K@@#1=LqSeFfB4Kz#+&S3rFQ)K@@#1=LqSeFfB4Kz#+&S3rFQ)K@@#1=LqSeFfB4 zKz#+&S3rFQ)K@@#1=LqSeFfB4Kz#+&SK!xh1vV||B5M2Zxds(^=b1GNyL2MF#lw8; zWh26UyM{%FwhfL6k{Ha5?9yK{_q=vOyo?h!ql1AFYIA1vlDRW3z%KsJnVS9o9`^hG zEwvWRlv@Y5NHg87<>kIEiUNOk(I$U4>GBXS`Rp(sML>Y7#3saDIyKT?GInODc-WlC zUfsFj7dUgnE;2Ckzhp7Nl|*g7{cDgJ+(yz@onz(%%M2ntWy_q+<*(bD$v<;6mwn=7 zA^)qhrTil|TiG8y?4)ma+Df~99ArX27sOIKuov&1IMt8;RsuV`d+K&M_p zOgm>*u!u>H?qSw1JV%Y(vu@m%^jRBj>tIbV5v&!~K{hPxB?}f_CrcJyBq_gQQ?viu zXP^EXs`!ue=^7d6qBvr2BKyL@O#YLj`8{*MeS0(69S2jzkM^eWZ|zK_UpirYxLL{H z^Rknk^>q}V_H~qG`8kWDf?PT+qdldg7ex0ASf6}O14ix{_)iA^tbzYz@ShC+lfi#7 z_)iA^$>2X3{3nC|U$LqA|L@^HCZ?9-=g~3Q%Td||_HR=5oh%gn43Ft^bWB+E8NF{N zzXSWx!BqYYZa;UmlD_X@BNqGG%eDu($`*xrh&^Kcq~qoVck95&ykH5L7uE(7lle2+ zV0Ex2vS5aYERAo4f&baye-8Mc1O6BMdTl0rhuVJo*MR!}nK5-*{@!h_&Nk8;juu@k z_s*7zpY9sQf-|^h&619de;prm%s7}S?%0`PJebSB!dUp5kDd5@fRnVs&p{Cx;nrn7 zD^N0cR#=+`_-DXB?6=E*9Qaq;Z~q!l{%1#Z>W2AA;v6j%zjrWy0RR2R!UOLAe6CQi z+R4F8hWSDEBXrUiu9ot5d~6j$UkA~eC@+~C_#XxSHP!t8_WAj(`vA&+T(D$dxVLn@ zlcnNAM{}j_x$FBM!22VA#_A^J9KmX(zIg!}=?5os$tM^G3O`3#fxkoYa~{~i3}-;omZbM)Y=KZ ze4CAb!Rdqfe)Qa+;u~vcKOqN1bE8-@fYJ2?RX(+EY$@jiJg=M!esVOGedlZ`dE3iT zu{YF95imcpYjAS3f-^T%!NAC`%@XH!5w-nSTm!AhP?5cxW6OC5b1AFSDaTLCKg< z5AOlq#b3KxNnUri70X?%C9gS}D!z6w>r#$^esfFz7=gwIc4pE$_U6*h-E5>Xe`m?+ z7(e->83A3IOJ?;l8y23YhV|iJhn>nZCjSd|x_Au!&e5c5n>9JJ#gd#ZoIG*NBl6~r7 zCqEtN(m5+4pwT15t8;dsQ%|+CW!EQk9htuNL1Rm~ra)t4*-v(+&_ou}_dV_8yF$IX z+~o1S*Vb&$|Fe{?YlV5}VFR)Ml#^>Ol0sjp6XTU!L{u%JkfPV)3GvJ>A{|xx2 z_8%Ju3*)cp#s*)Ebh5zOml^m4|ET$SAfpfBmOe+>rR!gICX$<8j@_bJk=MNEM_Cyf{vivM5}&_g$Az}?^$Zc)b4O%csra(73?$& zkNcS$sXZM2+_pUVtlr!iU9+96#BVx)f9n6Ag@5X|?MO{G8hIiEeUcERhD*UrDnSv~I@voNNY3Gon*_O=(D!CLUQY;Dma z_*c#Y_wq)4wRgAiz1|LRkQK)Jip_&vI<*3vIvE)GPuc$f{|x>{2Kq-6`d_y?XShLi z(by?f`Nn~~jA4uT*@N@=xdw-M*}4t9Yz=u`j`r1tOucu7>3SatGPOUgP1XF8o2vD_ zAf5ArAcOf)kj~uV!`S;796Z+eUXY>jwID@WBg^qXZUp`dDuvtjK- z!H4`Ty^r{r8lTmrGv5f($d7^ya;H9%{8R@MrjwrpF#gZ&4xWQCxB2Ou@A>K4pYqam z-sh%kz0Awf>EP!M+{w=$wSr$T&X-p@WxTL-{J_S-$y&T3E9Pj08#VH@wsuLceo|cb z9B1nPp*N`hR@S*aT&F)+vnXQ##i%6+J*4N|?oYhihJfp67 zcx+9!{vv*+!B$@801@Ve4}=-acLK^k4CCP^K8%O^^p-xRhOxvNJ3i9-J}-m0!%x@v z0%PHQ%nz4&IfMAZ{GqA!xug7Rb4M6g7mPQkE49>WDtBaH2-73?^Pdb<}dhm&&$=a8>^E8_e zduScc8$F;lZ>%vdZ*#iII~@=2ACb9LeKnLENX&KLZg0a9MB;c8*_2&=YI3q#34Cymzq_N3{618F?u zL>iB{kj7)Kq_NVCwbuP1oYo!g2F=BjTrggW1(|xU3DcP$Fo)co2kz&e&Jho^zA?y- zOI9y&zUHNA{ehpUE9K`7F5+bk53I`{GpfF1l5TzJ6sCU56jEDZO7`X0k%|--QnbOH z`||M+Mm&JfIVn1 z(dQnvvg5(7J+?8v=e3d|&y&Tm=gI1X%VcBn%OrKlb&|g9b&|Q_cO+}&Td=oZm|0IB zLXyCK{wf=CFxQ1tZ}BAcyOrE4nGYbHM}8gFjyg`Q> zN{oF3TP6>#%A3G}F}$s2q;9V*X`t*MbtO&UzUjF8zgXuEH+=p?BYwfCWMR(0tB8*g zBlmqK{oMZ6$AAClSZ&mQT+r{UoD=Y!nVW(%&L@HlovZb^!}bXCMuZBBMi@2hoT_)S z+=^4U!GprIWQY<9va#?Pa1teb6To4r}@EeG@N7sCJ3GW|5KSIskY9 z@!<%sZtGQZ&uhY%&5K?J_c!ixpS|jBlD+Dk7ivjSlgYLe2Xdwy-0$%vt?>07hbixY zM0E7mWo-w%h_Kj%!z-9LgqJtmt}buT1~4K6dmjoiXnsb?9m_rLLt{YOXujW=KS3VMn@&{)%ZW(Xdw$oC#d51Ye*%$1zA`J(jxxsyNr8{Xp?m?PQ zJlHySxHH1C$&9dk(varzF$;xR1Fv#7(|pjqdEjm=&`%TLJ5n7+Q*u|oqLH)u4U)6^E!aD-cYhHR9}6PC z5#u!Pb{SBeJ!UE|d-$r_Og*tML*oOgJyg7__@@l=(#W^mG@Xmw%;6bLrDIc?atHO+ zr)hn{Pap77P3GX`f})Y5nvOYZw4C&0VB}d%ymva&v2)7M`a+|XwW<2QtKH17Y}4<; z@=w|C$G^R+Nb>;HIZoImi>n7=`T{Zp9E`40Ki+p6!+#?<&7WsHFLOV{{Pn6C9M zH+zuaMBb#}EvZi0x$Arx80l}Jp7NmquJVIe|NZhw^7Ai}mC;Y9ZM6lJ82)9_{)&j4$C#Q18GBt(cHS5YI z88?>g!Iljk( zRNggrJa)tLuH@uy7t+4VnrYc;HlikdNP6uitqf7hW76e55EKb^A0bA^GxRAPXH`1`x1G=4Z??YMRKQZ)Zqoo;YMP%$OEb<6n0 z&^+gO8#KQ>yr@{XXX$hmKn!{<>X@j|G@LqnZ#wS7=cjLR=Z}|Q{ z3NsC^Hx!IXs?HxXo?kF#w6JLSr0V=(*4*4tUcAiVA*VL#gq}_v6k1z2KA^V1$WD+k ze0)RR$l*;}rVQY1anan8;>F~H|I{U`!F~zY-$$0rK28?SIyWFTq-Pn}{}Zh_piDiF zU;cOht}5?pJx%t5qlLI9z*7-4JLZyZ`qJMqx$6=4_VJ&&_H~lF;yT#B3ii(_*`Fr{ zbQZ#PQ|{D zF-Z)cKvvG0LQ)r)kb>prWa~OhvNy$=9LcmLCv)NR3*hsM;qy!3^ULAmx5DS|@FuOh zeM$RXKhnO>hqUd*c-U!AP$z@A0d<%q#-w&T;?CW+q-h)SEoBare~w_ww9ySkV`mF9 z^_s!W-}q?+>#p~hVsk3=G5jiib`v`Jt-9xe z55WIP@P8WoSAqYU-Oi-DZ3!bf9jaAdI?0}!GT?OWCgz{`PRiU<-x#IilxmvV%^E)l zG6!F%FB%inuzjk|sV$~Vb%iCTzRXdJmus%GZ`B~3!<#1P)NS|GuHWyaQGd{rsVt5o zJJXht;x(zr(-o183wDsz^A3__bB>e6v#Q9#cph1md{rkt?0hKpslWB88qI$<*H80# zbj;GRYcpN)rKh8U7Z-Zb0{m;1WW7r=HvW#}YAMPuX8~kz}mALb6uBOtRO!a+iNr_izZ&(c%yTeJwI_ zs4kg2b|A4eQTqL8u%8q$k*uCQm835;B}FSO$o36ZWPh3sISQYD3O>I^<@3wnTPwi$ zR`~oK;C;6r>DY&u`al5b*zXPg;rq9mLm!RB7@0__i-wch11_ZXfGcS$_huT4Y;ill>wphw!Z>W$>rLwRc!Pg$!rkQ!-|r3I?@dnQ{z*J{ z96zhXYby7__G2vU$9w?(PwoZ#m^)PLb9ZB|gSGBedxZ@U2ZZZ04qFd_%`3 z&Fj$bulgJ47~rPhyJZ{l1!aiAkNIhwF&P;!?6<{%*1=;Rsne=M-5qtA-@U^T~qvda@$voJN$d!ou5Db_sL99lF-~ zv^5Y)E{InJ^fOJ|1wztJ+oq7 zC+5FaaNY{;+mSb=?90Kvf@OaN_P4A_g60AHsmm{s%+*&(&f05;n_h)o|26T?fPbcA zkHetGeAES`4?N0E)A*43e3l(B&-(kmTb1rX{QF&fw$AzLoDqw#b~=K)W%9V1{K4fl zshrPvDH{K5${AL5_JEJorP?^nOT8<8QT~M(11(!^n5JzO1M0UGah&6RvrwjrA%=7ZUJfWM3^puYda+cR(t@dj+24Kg)Z%qd&*V}BT zNpCw$m%I_?rHG#&dsZL(Gyl!}bNAW}7nT~&t;;eH3sW_}KrDM#=l5ryy$`1NDdhVz zn>gLVlHqfNTP6+bJsL2$zG%Yy>J*)Kc$+xi)nyqfq&v(*Upg77`)>VOrt=WyqF>Pe zQ~vAeJaE8{5mZbyIFUXisVZ&2<=Rw@AJ|+@-`u5|2Y9K>kF^%WkH2Kt9{ zD&Nq!reN3#UY6b!cV;uEXYXoG?EaL2gRclewIn+&y&Ch3 z6x@xR?}e$_Z`bD=E$`T7YH;DW4!`W!t5?A7Q|^UP zarmUS%Jt#C&jY*cxVsyFKbsDtV*q;*ezKV+`fG?+cg5VO^CQ7O^GoO-Ht$1oo_d;O z_+VW!am+wM*M_~3^NIE~Ci5dFk~MRtk<3MAq+~VfBsN)-gXy-UGRK~rM$B7_m{*{R zd9jAvybUogw14{^Khn8BfQS!ao#AjG^uIfFvMu2jjVG0vhG1kAM_4j;A~(k%gO{f9 z8hrlUHKhA}K06lR_j#L8vy-WJ{&fDJgacWIgLkYOz-ilVGP*v0csX~o#y|MTe+sh) z9B3$+VBECRfzyJxwH2D58rA;I`+Z2`ULUOYc_aSyM(q0l?SEJIL;D{Jb(YV2nOQWuhb*1jK~^RBe>HBC8XX`A(|+v(~&sa@W5?@;6*3`Ri|xymdEN{ZC_G zO$-1%9h&dgA>)k(60500h?o5+us@b8h?+#!B}^yTOUy~xS}U?U#fBWtv?C{S9Y|H7 z6Y@7MMA$#(-HsS{C-OardqoEViR5rFksb{plA{5nZI3N!-aegFrVeB((*|;mXBtf6 z<`@~OUDm-lM~?oDqLi}E{$C><${k_f@Q_iFr)(UHfb-;wh=FFM=2?i&# zhQ{*JwOcR-zMy_z^>@htHvx4_VcHok)t<-f;71vU-}aBJIyJz5{pJdk7GUf^G94p5CP z)Sj}Rg<8>Xd>j=wyzIMQb23-lRE>$J8B?m)(%+>z$acEqYfnen$*91y;}*}k%9)Q? zcV%K7S+{^sR?cZ8i=sQxe^)^k$6v(r=MWQjkyVI~*DdHJsmm^r?6ohG{Ee@Y!c8=W zevK5Ye+_k0ud)1dkoeTmqB&1(taS|_7E^{0Pdm!~IFcManXI2@LUNZ`kc#!zWN(@+ z@;z9W%6CM*$C>cq^Xq7yv(M+V`JP?Aq+^di=|s*)b~Kd8D`Ce&iTJ25X{;E>+P*Yh zPGze8_jL5>e(L}4`hL_n3BW!V>{s{C`LOK!f_oV1 zjgI4YPk{fE2e2jp-(LmaPub@|;|O4N@beAeylI~cX@RxDI$)jDFitu!PaMk|&D7+L z&==$m@#bgh9|7+l2vQk#tkbxlHVv^(-iV{Zg0Uk-JN=n6#eU>)?o6^XbtNfUolDXd zZ6|9J4wGf^r;z8Y1?NBa`Sau3@mwbi{g|=dlNi^g8SX81_O_E=#$K3P_F%{nZ2sc> zW1OJ}&`-|hlDEUWJG0P-`T%m7-#!ojbR5v%r}ffyX5u%4JY>r;jcA|Pe*JiW0 z@M_J}B{hZ{7w|^S4QbV$6(XnnGrxBJ`GsRS^`*uWYH|h^*Q9g)Nc}!#{cfE=f4@(k zr@o)&3O91T=4TE$(^xXm^6V)e&g-3vn77*JbK18~vZ~J=+`->W>o2w66BHV+tSdJ$ zke!*$$ict(SK$9C|IOQ>>-RY^7@HbZsF^!eJSw6(OaBOJ@m@!*=eM;fT5nY4ObR?x z?52G<*B!opCVc-&`2Jk@{_XJnhsnYjr^x*1T9OzgB=aI0V9jJ+R4bVm-A)o?I!V$D z`2U$wk{pNpcYM!)C~tYJlbQHkJKD>qPXnoR#f!dW8KwCqGudZe4xQU$eC2LF_KF(B z;K~>TW8+yrQ&lHQb&!L(`29dn@k*TAG+<*w`-FL+tqH+y&7tAGQp0)JM+yF!U$OsZ zcm?B(d09iU!RjBO>6L!JAOH9JeVPa2ZbF_g)j(QTJk?KrCR+FXbEyn(e=s9IW{do$oXXJ*R^jMH@Ww?t47apPx5QyzkZ7c_7j9MRXZ)QH*=Z}zkKSXx}vcO zbvcHoPNy2kDzhgoIF@g&|04Kb6yME>^pQ=(c`m2zOeJ3_*Mpy(!AG;J+%susBERWv zr|60B6M2Stb_IFbDlUWnThFb3`t}~MeI(3e-+S3B>gNPqnzm-Ac(hdGDq81k$ zNn%{c#i7d*E^xp<^Q8KRjeRvZWS}1E9fs)>(@BGghpkfkFN~Q&HYS;p{1uj1f3zX{ zGwetuV%{?kjCqk?LCnjp_3rg2qW!3yJ`hCCo{1)k6QK;Xj{R;%!omsbYBuTo4!L`F zUF(s4AHJVzetw$X|1}hjU)p)VcW{p|iIJesPTgK_j<9Hq8G18_Y0QiLgdYeCj8-?4 zP8xW%)`#gn?n@*`5Qjj+w?WId9zg87ANvRP`J(Q}7qt&QsCPr&2elbzaLcauQtgk| zRbrn_=w0|?Se(Qc$(hlo{iKy+veZ&gwsF7$v zkB0`-{|U-bV*zb{IzJeBCmTk%&4TGX;HcNQWxB!|N9 zAgkx@AxmN_NpegzNs8iQpHe;iKJxsLE%5y~gC)8HzF!32FM;ou!S^c=A9o{8=wg=6 z>D3GKlEpb$h~KyGTMJOds&~)9pgBrvVC-3p`hChbax{0WO(gF}`(N0Q7}!78ms+vwQ0^%4|Q&X zu@{W8Kb|a}F%|tv%}CKIE3$L5E%I>oVG;Hwv>oukXW0&HC^Ak&zVid*JCXA-5H*xewd&aCr_pf$-v!nNYwfcZ z+%{t`TO(?a>QIZs+v%fZAN-$09DH2GKA5jO>Q9ctP8{(gCl8?>0-sj}o>g%#WxoOJ zH-UXN<^}H^2i)*?yP`2tf<TD zEI?iW^&Z>IISt#)wK|S^=!j1EaJHwdBdH53$ckB1^PC~`qj=!Hj?9Z_ghBH}Li0qm zgZ)mhF9!QkurCMuU0}ZlXAbp}WeMlVx`lEk)KfCa+g@@UHI?Yu16EnBj^04fL&`Dm z?EI_Pr?Dw&mp}1wJ-d5h!u2V0gU_4!+AB^XcmEyjA*$s6*~VA@F-dh0odX<9Wk2{h z%D7QMvI&c3^ceWr%QrchNio(WH+-B#YvY4EjMpV!;1s34!IY)HMfR4yN4Dp`LrPQM zRIn~VAn&6?CK($N7c0vC1hO=CI`VB6$aPwiJ!y92XpRH=1)^sY@|*(H zbJlOcekjznwXivUGB%p{FzAorn2e5wJG|&s0;fU z`w!xu)%@VU2IGXAHc-}7G{NV!wj|Bh+U7Ij%4l%sz%&#aPpHcsdIsyfU-LI>eu#R? zw5IZ@gG2{Im=5@Fx@WWXz&&fh{ic2X$kF?u))_qyxBHMY$kU#L?`PSk`5y3Ic`Se& zuM8w7aC`DFa`4FeRDtFD*=O^d@cXRKr_3L8ClYF?-4SE1=tPZB+b--K+wM;4kQ?Kb z`XE;vNRH=3VBI7hwc-m%DfE5r(#@o+CYb4O$}`B|%}k{@kN#c%pLOE)f?dv#3V#y2@zGk?#V-1jF^ zp7|EZTk|@}S^gTyhEeu2QA7Gm^WS^MTeE4)I9pzhVO`Be=3DC1S-$U!bMNYY8t+ow zgV&)p;4i$Kk?UHD%!j-!UBO(aok1x74ZAFLQ41fzOVfUhx0!QOkfDFMrF4Re^q`BD z_%O|P{NjS?ls*9Nu)jc}YLs955UH*xSDfUHz zNk=*cXiQAU0J}E)fsg;p-n64O#7*I`Fy@MGLgcy0&>{P=ulF0uDShqp z@P7aA((lgB0p^mg;Oh_1j_5L49CLn@zhid=&OP|n3S$mGe?8E(XZhNsYeOrt-lqIB z5941?M-w^!fyBWawf(*m$?|wplD*W5Y~5%}4rJPsuz&P>`~-&9z<;@-TS^b68jE~MhFT9Z^eGYPq~{l|4~;odSCOl z>7&lp#4@5|Az14Qz*-j!KEDmO)LKw`+JN~if6C_v3Fkj&(3L*MAhgzY@CV zcn~=OI|co78hQVFzTXj=-I3Kj&HJ25n@aOgpMS^$dJlHc1ODFw+Q%LG%nfzlm@9F; z_+?mQCqh$YCVJR)y=!Z#K;3B9O&4?7OQG(H8S`gd&|A9T zGP7a9HIlaAI>}nx=l@Ylnz8aa89GRpSWO>JB77#2m9x!B!Ad)_JJo?4MVx!O2y0wr zSmUa2LHve)Ps%;^u~7EgcR*wB0`q%N*RVerF&}h4G)MRGDAIe9S}fDB!(ou1VEi)V z?cYQk%hv8cjD0o^relNh&(^(RovUzoVlOvT?{ZT-BRLs}nqW^NIvlN0Uoy^&pRLzL z*N}KATEDL=9v|DX&snb?`?DHQtKW>;=2r0Ditp5MC=k6X0!b5UFocLtSoXJoeb{lt zz;unPvJ&i{2!KKNoD3zWjt7%7&^}e*xCUCEw+nj(b~}K5N4z)KN1b*jxEE9A4|@_B zwL_lZ-xJru1G*pe%h3G|(ET*#tts{=C-cIvW*&<^((_6A+La`aWj`P6Z$WNzFLeG< z==@XA`8D8P0PC0Eq*|YHAJzF#-LrJ=IkJ4-1+p?p$^Dw-D})*y1M8MvWng+Z8*GNN zx#%UEX{tUxN^PQC7wbO`?)hw`uUE#nlsov=I}RrDZ=KDhZ+kmP%4P=i*e;%NQFmTs zmrjsJx3{OALS|3*udwm9iv1^_TU4As?iTf}Ke<@QJ_vD_Z;lU93|KnrlB2JatkKR) zcFPVu+A+?5aIuz%0(_*tGiFG&bCRKbV0p`xv2W&@8zgPjtG^ik*bk^xpEuk=kgnUq zPq|mm_E`Sm`)Ln{`aHwK%{!({Zrf+c^yk0(Xs~|AcB|Rkbe(rm-+Z$%XK+t*k+DV7 z9#4*%{|EC=#{?Y%vqR*=e4Qj~v5xwYjS1EN_k6mlU$inlro5|6ncJkA!_GwU18PeD z=4B&k2yzt9kMWUBm>b!pS&|~xn&Bt5buklhZKp}Tr2VJ+zxUnu?8{@)IQDUTKknyj zA1-tG58n3Ts`!wO@oN-0UW7UHwMq-PY^o5+8%o+kdn%~5nXCR}C?_Bk--e*ULU z7)-*vv9Be;l9a7;M9qmCIfc50no{iJsz6QKR#*5j!w;lCt9q^fL+k8pqF4VH^ z35LH5fe#FY|Mx+Cn?Jc+GnZUE6UAvL88@DnHDHG@mH7wt>8w3S@AN&Vig^(q+~H^G zzFAi?GNJRBw~nkjifMtasN3a81O-l5JDxPMzIgP(npDky)Na=NqOE+=PRSl?qk8ye zA^dJ5-CqXX(*YgUy3Y^2`>7rZLY;y?d_VFp74ZG=$;Y?STqpegDSuLV8ovJ&@-ioa z$?4Nkr0Q58sYVQ2gU{scv_r1c4s~q~$n`iOuY}y@A!r`z^N)BDIkm%H@c&-WKAzBl zsP9AE+lcipA@%^(BIk1|KZG2~no0JgB#{c_HS?FJlCJ4HjhE`oPp2h zl7t8$40)b@`AwSVq4Dm#Xz>GMUh4a4yh}9?_4#WTVqYMP-e1M;6S;)FG8eJ0=Wm?3 zGp!J{4||!&&sC8g?kRSh z7u0FEK2E{ejQ%{CYua_9eZ|4fW|G&er=V7iU29h6eb{l($34zvMchGiP%nUgd`~(i z-7KY7qP!&Gb3&zhGXoXz=+XGPgNagm(VVQkiTF=}j*66}QCCK!FL{G0SpO!;S^qjo zTYH^sUU`jWf5QS>vLh9HS&Kcf9~?DdTO5c0CftTRC-N69+dYUF`9jpD5Yb-b(#!pb zbbl!7T*5H-hoQbF40_jvNGpTL+b!#v?n9miE#;Gwt5db#0?*1`ACF+4&IxQ>0N;-o z;d^fGz~c=&r%!L(W5LwWem`5xN$nOZ(o*iBA{=0h%e zr?IC}fZS%=!9a{d=&@r_qzQVdr#gx>>Ns~ z^$+BHcHp`pN5(Bh?iBgXquDWJZ)zghvVJ8gSdoVPb4Bc$?$Wpe(EG>1eKko4=fmp3 zKF#g7VvQGTx@>-v=6RIdC&ozsCEv4jF3t5^WOWW@e%+#%Sflr;_F0*D3AKu zAMY=ce>$#ceK$1F--11)88HE<35sa7_O%l=Vg5mEqKxgahvt^6x#X2#*RJp-vtQNO zlKnQx+;D?zT>T2!xa{f+=fAhoSF>ZwG&^CIVJBOQ38o+C`&Il?-!I6}dat3ta7oJ^ zbG^1oub=Uco;$kDCDUTLshaO{)1dwH^v`pPhL|1MW5ZGNk3I-b!2j}vH<-mq*YpG3 zByr9boiD>j-a_64{o-gWtoRY@x0ikGJ9c^4wr=uq5XA?%NStH*WD^o1dIu&&ozsX% zK8dowcKOTjp-TSKH=bkCQ?F~ziaKZQ?Ib&DGhK}Fql~Q`+4Y8d{+^Dp7vcZuJjJeQ zo6CRnv6mm66W(JqFQj$2pN)7A`m_H)$2HY->>iqFo!|I5cb%RadBqI;bNchYFV~kG z%Ex*r{9`+OdmGkA`Q@ghZmR>SN32Ut1nb^|Sa(+tVq9<1kmFCzotc4J-w4usJd7w# zg%g|uL3;S{O#5#0;f*C@3Q*7Tu`1X3FwL)eUZs6%vDcBCp(AK68D-mAI+{~+Fp3sevrLbW5iH5Rq#@A0o6T!YveD_eIIHokw$yw+> zVM%UCl1U}zwVxS!eN|#@qO!fuFmF{>hQmThlW-}i#-99|vgyhmEW2!3-`|P4d{*;N zpMT5;IWE{yANYSCXdl!WVQ)Y?*1MW_(!I^-rGh!;Okn^ymJ>zxrOhK-;rk0$rjv|C z#i(Q6iM^_au%3I8%nh$4bHaqUZG`W~nlAD@iHLQRpm&nkwJn=8FDt)zaUnb z>!-e-YI~}4sNY||_+{4UeOCM6XUpea#D3TdWFdV2{OEJYE1g5{nsa1Mh+?>(lQ_lR zLi`tNT2})up(lRyu#xqIc}wO+_;rp8@od(Q_UX`w@kKpd*aealdx;cnyiS%ceuHG8 zjxs;xFC=5_AIYwgkI1H__)haK>iD>HdOMhQijn*Ok+Saqdr_LFe@vj%Y3y3q9ZwsX zXm0db=UD-Ay#Ra38gTysJp;kHFUO1XZ0Ac?=bt+Weu7xb(RHV&t?qc$ee?>!u(9bx_oC$O&pV_q=-sBIfQ z0zJcbSEq5%WAMpz51a2qd`#$41Qv~QgtkM zR$^gN^hM9S&2Q*}|0nYQl>f}-SDB^p=O+ZZ$+p>=h(4wJ7wH-^``#~jE@1h`_l4H~ z+SgHDJ}>&L;rv z^epzQP1Cr@>+4UY(l?L1rTKm~=g-`%%QmPM?{k_eKkCEOR7aEk{GTZ?(W@>PI}5$v zeuupOEpFz3m$><(oLUYAX{h;sdj1zL?nS)a&CChyw)M7GG=PsERXHD=|MHXFl+G{w z>?KE&VlG%^t$AJZN&SD?nkx)UGdi@}1Lvl8*iP;Ik&T7HA=N@JO8-+WMER$_9r5}t zZ#&uPc`-ec5@XH}@pn>ecd=4@OV7cf^N;Fvw$sEvAdYXD9ol2DE~%UI0{uVok`qTm zyX?eXj!#(KLvx;w$@$Qqv+T1vN&8b_?%=G}ZPWD>C&T`Q|Hh(eh7I}SQUqz*A7Iby zH~icowSvMilj{#fF>3ywiGOBpT(@SpmoyHV{~dee3{__aP+sh&qc8as5zccLm4^FE z$2}?k%M!cE%K5!yF?4pQn_`hGYENxV#CUI|7I_icM->OC)+jJ%-S)7Rh-ZaK++zc} z^+H{`H@RALeQbwwK$Q6>fVpu&fnP4TA)7vuj2Q4Z&m!G7UPb)jp$lzP)BTBfJ8Ui zL!RhnA^V*jG!Hwb*|`E^0rLUowyy)7yH3sYJ3ATtGcO?iWtxhn8rJ2GDaAe%oGsL+ z`=7%9ga6xtOoP_e3J0h5ZSEQm;$KiOWTYV9pc?0JeT&%l)4Kc-sSV|(!_@pgWBw;f z%FK$CPV;n>9J4i({^+1O*9ffi*A`eq{Q+m{oelTv315|bS^r7-&s+Hxvp(^ zC>Kllt2QR`+b=o~#A+!zSAl<5E7>2yyyPop_{nu+eWfApma_BEbXbR=I!QUM>0E&4 zZsRQC>%r~{5Ad(~y!`XGS!f7K##st8btUM<_Q)C^mCt8&k7~_}YX1z4@32<9k5^pf zfBd}stL7;>cj0}1^s$p2iudmv9TOoQ=jkjyi9P=~2Z#E9W&Ir;2doA{ZML72s4Cp6 zbK3Lp-?S%;Q&(zg%+EHcfe&ZTp?DbY_dl=VpXN!q>AJ5}WsQkFlRHtfrZVhb_;1~9 zuT@_@!GoKH_yVs-&xns|(g)5sl0HRO&HuCUAL4eF6YO@@%iX4{8~*c_YCj}pMdhEB zd?}ehFWOtEr+g#C_iX&q3*Zur&SbUA7uer*sU3k$b)LmBMhB?5|RCzX^ zN|!%>44~Bb;QBV=`1&ZnF3Sl2P6JuuH~TdC{h#%;kJ<66 z{C;@@m)Aa#%4p6QIDC;Ji&1D~idnlGYEB{?5qL{|A zse=Uhqsnm3(5H0$_DSk|RP(@H{^?pl)_~@g9gfy*`}{c%;{Qx4qg9hn2{`>z8R0GkN278wO9^oz9^Q`>W@Al;gOD(1g(grpOQaQJ&|EEl= z_D z8DWh~bV$=wPY2Nv@bCj&CsNibDEW7xF=HS9R5Pm>vY95iiM=mZVgfFOfqlIv;eY8I z2{SWTq#5SZIm^Ym6M0DqdSu-<)}CVwuuLNdp@RQWPRh61()f7#P;%Df-gt+&q za{hPq4_HPl`=cO7??7G2xS%sJ1@i|rlc=Oj&Wv6TFia>DY@=IHy_1UD|=5_a3!T){Gnx|*LXr;5MwcT=2< z4eOn>e(5WWs{ZX^{(r?>8O|s=$0W!0O!RUPpR_j@Bj2j@%XjCR=i(n@7r9P1Yw7DD z-jdiT?_S*qkMr}KtrWkb^&HBvrsSV~9-9X>ll|cDEPwYI`Dd#0jdZK?2B+~ewLW|r z-J_ZpSpTo&|0`a$Zke!P*nsx^K@a4=xnP)9L+(I7UM8dR|DQIM8mD&ba~P)P|JnTi z@`UqDd`P!Oh?{JNyN&d1=mEA@gEIF6e~)@N%884${Cm_6-|)6ke2D$-w<))_w4TzU z^SucF?t~Q!UK#-Y8Cw7ODE`ZHUO^3!LMOmOvd-B`@*&kybWV9u{L}qjZnm;_!-B>0 zLcH2^{heB8A}92!Bld!+<}TIeDf2*zJ2-Ru+h^gwKHG~U%rZCPWf>k9W@&z}V)|)r zS?vS^WBg!ORNp(U=+B?eA$P>qgCvvNsAolg6dbGY(jukynEL zf8X1qImzEcpzH4{h<34*U2`<2`6lIOs6L0j9(D)y(ceD{|AI^tO+n@)cVW7ImoS65 z^EBM6UQ7Asrm{UwJ_E}+4W(msW&3Q&gY*w~ucrpLVuCd&=a@<-GP$|g+) z|BPDyJbnEG;#cfJxQz9wi$nZfaAvBdlmU9F|F(Ob~o$x&~0i72{?ny++4ypT7t5;GV`A&&NO0R60tpp?GL^U6%GI&&xmJ=f7~X#%$(h zKivPR_@5ngmTX%660Dp@y`l4%ud5OH8t>g;yDM4RqR*O8l{^|RD>i?as zI^T1#<0gjrHR(nNiDD4DT(f8O|Gn7v-aPYc{$F~)lWE&BWq5t=z=OgJjfb99)Zf=X z=)Ow-@H6y3YbzO>+g>(a?_v7qWnmJj&K)zTCVNByFHQ4fHV2x<`Lr?LsHC;bWQ3ak zXXk%*P!EX-WY@x+J**`S=$G(4_P#^=uw4Gt&#i0unuS*e>?(W>XD?mTO$a+5?P@7{ zjqdHjK6mV^l6~*#Aa4!z>YB20&SlP{_+LG@i7ZKI;mnR`o8V<9uCg~*;60ShgxQHI5sx?^F2^2#cOn8rb7lrBbff)y z;;}FC28~nh?gRVT7*O(mBtO95c!8Idu*7Cs^A>wu@XtId|6jkwUZcLu+ym$MNWtNA z@y~Lcfqm>bx@8ST!v~^=$V2(BDY4h8&7a`J&D1%M8i+e|?4joLt=5upfn5h&b=3Sn zJO8tTuxBOy5|bQx-pJ2Rk!LkU^ns13=x>4U($wV%UBlKa=w>MY)aHkEYsL8VxcfMC zUj+Z?4Fvvi)^MP+lo#bI9Si;$y8q=t`iJ`eW%D|)*S|+QF1ULE;_A0mJ&2yq|09k- zUpnOfZv?rEXGHk)qSxK|WLGQsJJ{EWbqyNVD|0~i<`yOY)g?ybxkaN2gt_$mDPvcD z>9m31pYyQ(5B$POn*986@i@2SReCN9_2UoHH;;Qx75@q{G=CK480>G}YC2SMG~%KB zA1iidY6^`<@$v^B#U6mK*xygp_#9_xRB-dgi~|3dyVU(3Xb*@d>i^Ks|5N^F2X&EI zfmb=v-WMjjShVf-b?W5K2|w$avHB$qaDSKooK1gZ=7gUciu(CXtOI;Z_qEVohpv{= zH(~gLP{eU$KWuGtAYnAaY&40?j<(3#t6rz=szYXztsQp(wW=n_jbB<>*TTGefG@u3VYPQ1^jIP z7u7$Me_ol1K6;=mf@b<7%RhPm)u!w9*5;115$tf$Xxrg}-Xwmksr#Rx_uHfTpFDv7 zitIl#Ym#5qj1Q5TCq{PJubq3vaO3=!7#L9*<$qqJR3pT-%ihDL>%5Jr^cIZ+oJaB6t&_cM#Pu%ba`Yxqvj437D{GqNchGbF>`Z@&@0`eU zI`I(~!aW_NRW6pDH(br7e{(YH`VKWt%Fj~94B%h(ZDXO4CC<+hqlV{u)YgB)&CtHi z&l|dxpFi4%S8OzfUphr!wB411{y$8|F;}Lse5_u5&fvnj44p5ZHTOfu0n0yK13>*J zdj2#tmP|Hpsj%d1FS2|f|K~9VxLaK`xdp~{7^j!%xM6Ef(>4DjC@@+sD4%Z7wr#qa z|L5@kvx5|5Z}A(9OhgP~gW2d59?=j-nkW}@C7uYZ=>2bO;tAJZ5Z z{MV!nkP8dUJnM?=G`6SXjHL(of6709moFnInLLu8Z?wNUMe}pE<}{7-ogjN)b6w$x z>22GlYCOdMGw6S%)n`nly8jiS>tCw)SEYX_|FeSRFd0lrW(7;h>`*D08zDmvMg>vz z-C8o|ELl7EoMxD(%pSdVuF(AhY@Xdr`u?oo-j$1HTpai){x>ZZF`+&pJ%2ZGDRMNQ zQ~vuj$y4V+sislZQ_F9AIY@*vgF4M-g>`aZByB|xvvFaU)}lz+kWd%NbYDmDVsE=H z5pu47hy95C4Byq~>N(bD>%3A&{V8~44P0X^+~TG)UvSek{|~(kd+YPZtZm!vHLT&F z(*!}j;mNv8&aJy=nLj4i(_i=e>^{{%Y~QN^FV`1Mp3zWZqPacY6#HNPRsT@_siDuP zZhgt5NSp)w2F4Dnq3|8w6BG5Z%yd`cHw`e|J!r&kNP3>YCmff;icGXKV9+#-6IAc{8gBH z&&l|pi$>rd8tCWwe^vZjlzx#!1WI*6e5Gm5R??5?*m)lQmE#A!pdF<9W(12y%#G<} zV5GlA`LrUo=^PyC+2!o-qS)eM)BTRKmH0burnVD5Q}eZY#xoKmzKCce@aG%xptH!|Hv6iX(Ct<kS4JU3o=Qa59JM5mse!iUYN%_a##%~&P^(&+mV+~shMq3IqwCm{naCf}) z`yCbAPjrj-r(+23|7}f%{?3#6<`g_-D~DV_vk0abv%^+ zaNizn>`_UAetCoDP@l*Dqj$?4Hyi2u*w4E((x+1^4(l3^vW0UlYs{O`Gb}Ew(*ph4 zoY1rC2Ic>LW>tJ6-||^)=D_TR;z|0QI~|>QSqANhTkq>4yL1$9B$29oyeldZwOCj39sy2Fxb z*=9PPmpO0`jqU3&mTOZr{#lbfV0+_^31i#$TQI$qK}`Kl3(|!1`kK%?SFL}ZO#f5= zzZmCk&k2?c3wDvFVGZh2tmo3&(0}^6i1r72bqr37YhoVe|7V0=)d~-|G|Smqex1gy zRL8Sp;9=`r4|rac&&1ff<8CKjJTjS&4kFaqGx|JgVqRr@auMMgy}J1j?tPR(XIwp(*r%EtxOPixa4^X&cpa$o&2WO zf?3z&U9k_H_GebHd~bc~@wE@C-$7pIwwJB6V{Wk6WqyQM<8kcw|9NeOwiEojQvXn2 z4Bn~HvCzjR?N9OvFMCjWb^0*f%5}rtcxhUl&@IYyEvSa+d;PQA($AoE5H;w5Qky$? zJ+EMtHuz^A<^SKTPa-{4bG4c(%w2hzhF5s0oSSS-Tnf2UyIJR@mhvf~qP-3}ybMc2 zUWVZYe!AvARJkh5H8@k7!Fj=h>Ke;UY#X||1e zbo}yoDW^Z*kGn6-(|4%L(YXd4eVg`H;6H$W+cm5jT<2>H+d}o7aG?8e|z5n7uB)-KX>UMqKG9ntn}V{@4X3Fu=if* zT@l3|W4eizWm%dE%CeOO1vRGUrM$en#FzYEx_RcM8DsSSox6JxeGsuFF~8l<=WyAx zvvcRpe&@`YGiT0WPVVy_jy3N_ezrF2hf3#wnhiSNHk6Lof9i4P$wzW0+3>Rmy@>jM zX2&$zOWREI>mh&GziWzzEvw&dF}Pu;wFdv#|GgLd=PZQ(T!?Za&H?O4?w9ZB`AD2Q zc{Mdc8IzUNsoNj^lcMF?F(L9O_~@Lycl^Wehn{b=mwXrODXmP4Rt#F2)q%72ch-Nn zGVnWdYAiN9jdLu&u(3sM!#^vZ^K`zdE*zYDFlVU#(<{upk7Vn#qAaWbJP7>LdjO-? zYx4EwH5Hk#zQ#$cfs5X3{y?|xa{srox`)T}a{eLu7nlGy^8XdyV zV|BHgap_Qowi)YztF;A&!un04oQ`cV=beE5->lL92WmcJiWup4E&o-z4~^p>xdFeQ_hUm9^OE74 zv~=DX;J*v{|G?XmSSpoA{^WTXjF!dv!l-EHTaOcs4>;ga3P%|4Wug^rM2r ziC(bfJ6f>k%Rh?pQLKm$XgBN+|IrZ&o$z3J24ai8L1j3n%MVolr{(FnM#Pl6?&lyC zrN>G=vQnkc{Rf77uH5l0lP5Nojd%w5SNW9n!aqA6fEDa7U8>GEE^OXmZwULlm!Qb> z6h9mCA6R=3_-F4!_3~WaRY9TgzGK_%$DY{bdPn>p-sC{eHOPv1IH59CkgNTDeX()nsa+PvO^>@dRu>pI;_U9t_3wSk ze{P&XFaNbR->6Hlaa?r$7AtM&KMc+Pb-Vqa(|;)b-|8$tepHts4V%W!t|-}O z$y!jq_>T$3{;$7eIrhIkg#GgF{T^TsrTe~KcCs&`d}VowAu@xU#qztxf8CCt!Gf}J zv-x>AR|VWqY;Za~ixXCs@_w$**WYuZe4^=*Dr<}C0wX?RP~G%@xKI3ZIuL8a$9hRx%=c9ObI-zIowI4(Hny(`iw2#m z-#9V|_-8cw|J~bvsQ&Ngc9H?vdOC__E;fp*ZuXLoT^-xjg8IWh6Ye9@5AhTs_kiRx zdX{*gdJk8QmH*iH^0AkmPYDx8XGDv2N>-n~Yy2O3B1xyVWK@VCU+*8l$}O_82Rpgg zTgo?j{#d2!m?K*(#?=%UzEGF<;PFrIC)fa8J>Nuhe2b-TU8M~V@}K!F`A_jLem;Z| zKNqamR52z7?R=#sTeX&_IJg78G*>yf$;o6ElZfjNy`;LO%57@(#xbVw z*WX>Ar*-AN{oi`6HF_{f_pL8T3mz8JxXAGa8ZyfAtE zY@IKtZycRZ==1c8f0Pjx=zm^cW|}EE7_1G_;D3PrpS|rrfu6@1kd3!@iXF$ zd>rJiSR3lu_xg9iOa+q`BOMXq-gXG*W`1GgE8ZpkRX!84k0bpR>*GTd`U|Fs?ppR! z{0pl58TA_Y1?1KGUO;6dhf83G%22x3E66o!sxPy55^Nl6#LpjGhCD>yK8Uq1r;l*s zO?|(sNAe6CUMw2p)cm+N5AvVs?f=tT|EKbw;=i`iio8;v&THB_)mBh6OmK7q^P_5C zL5+XGSiJ>Wmgzm4(tXtDEh5-FW6 z;a6`fdpFi!G&4D*wNLwhdaOhzIYJVS^IF}k?gLxSK+nEV{L}9M|KA6>OP);%Q;wQH z?JP4lvy05h=pwVz?~W+`MLR>a8_LJ|)fF1QUZ2CL>^dA)fE#K*gwM>26H`L@9eui^x*Qt-_?d@RkPxsvN3_jo&n#N0COyPkv_`e7Ir^P6k0C%Yi@@l<` z_=wjK-}AMEu-3SYxK_PB?Eg%BfK)#kd3a&F`519)S=&FiuD#Qh;vca}@g3<~F^E5! z6rkoElXu?BTptbF;1|3Gbx z?C0EAc|ccMCu?`j(fOjT)GWVcuhWQu;{PK>2}7xIo0Wy2!1S4#b-Zt>-)FtLZFH_; z3HK1-%v)X2kTZ=H;{#4U z|AwtLOv6@d59+GlJUR^c*9I}arTG_<*H1M^cO z>ZD_U`Zv*Mqr}gFA6&7?!Q)$}SZVNo@Am)cNv+5?Ax`zSmw)DgGv>jF8#E(MZn1iv zQtNj7Pm5Rb62cX>*ehFuuQT_J|84F}!!~!V zhO)k9{;L4NffI}j`1DP9nBQ?>wY|bxsjHu=7ec`@W z9kfnr!@)V<%XKAYdqsQfCUpzqcmu)zppj?=|GUNpwfI=O`g$N`b^wdWuFOi zf*@b(a&3umUSpMosRsY|j{l^%lfy&(TWKEQZ#?bVui!jDby}#*tYAs^?f(BvdQ7KI zV(6JfoY%Ub(qjg$|4_Xj`qfo`=l0`qfy$|f&!V+(W*50%{L?i6k0GS7%5Hc=@vt2D zLVPVy%Saa6EdJqFRy0T?*lpv%-)L?G-^pC$`~95SSRNMsX}vY@X|CRYcqhd}vs#~W z8Z=P+AK7k41UoF4nr)MY@JmdV@^f|mj`8#>U9(ets226s7aG5Rykc}}^KN?`4gT*P z|FJ>hNg-a+!-y~P1M&x43iT6LMFzDQuK+fC%l~=P8kp3`6UL$LqEb&=>7@bdK6HHy zqB#RxtR??SitJvKm3G1CiHgse2Z{eCgVj=|5EdQ~b9)m4Golj;XI0 z@mO`PQOVH_TK}f!U)X(SdJZPf@M2x*$b~0%SQ=~af6w@5!aZb;0Z#G`S1ZL;#MJpP zE>ylGEly!@JN`2hPw-Nso9+BvP6^>F3cJ55HYcas_jje(SH<`dU-5KoZcd6(*ezXf zLF+-|UsszT&g>-QwHQ){{oQX^JptIKSGpdp$XZ zl-_fDJ!~6rZ~l6%ui+O_)m&vc3HaxMnA`22ec}JD=9!Fgf3Vi^Eml+cxklUh+017` zxnPIA+2N9*Pb2@)kL*5hFWvoi z4Bzj2+6Kt&U-|2GK58r(xl;1D)6jRMs|JF9rg>kSR&9l^Vf99TAi4lZP}0 z*t&6#8tiA)kEI>S zADUBBJa}hQ31S)L>-@yxpU!L0gKE}mf7w(rszSQUc?|H+9NjV*`)U?MxXqR{?RFx~ zPq>rTr@V>y89yR@E&ws+0*QjsbAd$uY#@<79YEUl`H_>4dz0h4JW0bgFYHPCk;CPo z%;>J6WgE}mg8ymJ9lYdlWstX{tOI)duWbJ70oZ-m{Vpryr?-~A5$z*S zOpk2W-&KB*d06;wKb*yYc#S(8ZL14Mym)k-R*yfWY9GwtJwb`t^u}$L`pvtnqxl7f z=lE)$;|I16+{ZZL#sGf<@5h?_A< z-GY4GPjI&2Yv^j1VaKA_@A30v=N7DmYd7e9ASfI%Y@YvdX7{{QJ`bFZ)7O>G57b8F zY$5qN%J1~9X_03}El5@{OQv^{2Z?_MwjXURLJaj8VrIf46CcwiC-!>iHE$UkQ(s`* z&0o)B&jwTbANyFxkoz(!^Y;f4rJCBG3nH>-0*UxZe{yQC z4>_^hlQeGkAi^!)*L20%U)ph-5+TXC+HOQjkjSXC*7x7=&}vItVBGSI&KjtXc3fS-bER zme$btpWrReLfon8-DKgkE;29q44Iv9n#_znO{PbkM$D4aIO}_d_=mU0K1^*V^U~>k zPa}qTC-Rw|BQv6o8AW-CW_sGl|K(z(xaQ|9ktByp?G|Pzv{%jUBe zVQrNy^!tg!g{7k+g+)V-AfM9bh+ldIW!ZBu6yF^F`x;xfjsf0x{9NP1jg^z_x~pTf zCC_>j$@9>8fOq)|!K5A30qUgJ{lP@>Y!Hz=6+l||`jTc~ziEd%soUy7_!WNSKxrg- zDleVvSi6{PTt=~9i5SXHkVR?xf%PLKCA=2YMA9Of@t*2Ct-w7f3D{2-1LsnZ3?wI6 zlz{!LpZzL>n1Z9iJwNoNi{_(z9gwyt8C+)>&Y``@XPG@lR#H4%WScjd*0taBX5YNsk0N8eDQi`p{ka z!`(pn9RBH<%Olx_?>1CTohv&MHmv=ik8Z=(u~XqIosAf{XK^Oyzp(AIx(&C-J7Ap* zg3=-6FDSSst*zm8_ZdBw3ny5OSYS5<-L|E~p9kNBpQL z5t)_g74U;G-$Uiv!dHL?L? zU@hi>RdZe<%kcT43}7P_*iXD&?o;gdf&U2DKO=DdMUDORXbG^7@dxa~mcyW|c^R=C zzW&ZKkrQ;Jf$aBE{3Ev{*1WA>g}aM4Wk$4*&R%qZw-V2`WY&4IaK<_8d7XvNL-#%T zy!G~x<3aXQy=SDUHnEgX8DIV5 zRg>-ON(S%5*-hl&R>vcyWhqhJMoj}6XX@`^zwOuRT>bZI%15O)>~Ymv&6`N}Y*;`lmQ(C+L*A@s$l{D6 zWL{c5i3@HZp<0Qbo0{OZ+$ko)& z*bw2Zu$-UJrL`ynV|vDU$oz9;-UGru9S1tv3^5(7MW&9^BYuv+ey|BynmmjY%pO6u zE*nK!_xK=&prbD0#`+!287!z-uXS0NO|HUE;?%LtQ#=rZ%CKqkqp?d^5TAyP->pJJ(c0c!~_oB~mWF7A( z>;s-?+GgQ&T9~HYw#SQ{Js3#N9tbAg7zZ8uLy7pQKw#g8G*X=h*cWbbfZpRlUM!6! zPvp)el`ARscaT*x_d^e<0d|`p$4?P2hbH3VNM*m6EQT&UH$h6)FKHv0(0?)#C7914 zZxLU7F7zB$=BFz0Jy3E7nUCLV#oSk*Q~Zq-LB1BQ{u_(`Y%2Sq_pE?y9vJpH{422* zXeXJ#erA-MjdeaPs*R*aQ=5|va<-E!gPy%=;RVpk?3!ZbjF-vcIj5K?pVkS8-(TZq z+5Ur@P5Zx-!{j-0QshHQa^GN{E&GV<%>RHXS@yPWQebC{x4rDV<5VeftKJFws(miI z$!$O5Tv%svcvtM|xo;ahUG*Pk*}Rv@qS-GIHYw_&g><72dsbeGafih z*5+t`)l@oSwd4s8Gx;;Fc$nzM!YotVJWix4%SL$BZ<#QZUovzR;@EzOdFX#)_TCNF`z(ul(y>&7c)EvjjnCI^ zoDd}1<)-(b@bBw%jOXui!Z6lfqMsR|&<6e)$baS$lK)J6sCam|k951Iweo8(Yxy?` z{>nYmqdG_AtU60&KT8XyT^JVX+rHY>PW&(A5x(Acd}lTGX>K~2Z`(q0-D#@m<4{*= zVOmJ%=+z7V#;jWS8u>HeA7>HCQ_Bgne}jQ8ztqgPvC!lYja71_XmCc|c85Vtn=Czr zc_vCBVoyD#>Q(NpJs@q?a71jPkM-cREaMyZXO!pYwSD z?rE-hI|*z(ZPy*9wtnqte`-&dXX}J%!AjoJSy<0MWc(ZI^N9HY_ojPHwZEe4=FJ^R*ka|4YZk zU9E3#tNTvvqW6aX2jUUwUufAnKIruGKDy^%1L`^uL1fQ{k>b-aztn!7Q?Mbv|_e-DJh|gJeF|yt9(AubTwhChVDW63^%)1Sma&oMkd!JFz6h zU6d0a+UkkCwMH39?YtF;-?td{&DlvEWM-0r%t`OW_vl&xb|0)gbC$nNDhmEV%Cp}i z1uNeoIj{k(gDq%f)_LTa7|6Ppv;TCcbpNP!GCdmle++Ct(EX=Hi?A-2z~-Zb+&xcL zFM1g{;$Oq{6|ywFi^+`YG>Z@D*zRhf{K|2P>~gS&`0?0a@#wro=a{wg&y%#M4qj5o zX&Zk>>0!soq8~UP=Ueq275-VB2l#if5#Ml_a_Sood(n##f$a|KSG}cOTJ!-^l=pYC zVbxn?#iCcS=1}YY55VTb>HhROe26x2vo<7YAw$UO^xzF+xI zzfbM0Y;2a&p-IhKEVF8|_1=Y@vS)6&Fa7E_#sqtBI`%ebUuq~Bg*jmI@K+AUF`b8^ zNy`%|9?!3Kl#`mZew2rip#6L=j0(!1@;c3$xQY z$=swaCfrLhF2GrG!fCPuelU{HJsiX>zV70cDKQF*xv8DH^Wx7i8PUKcU4z2EVS0j+ z%ueni>z2F;?Ei~AvF&40vi5DV0kUx&)`hF6y&r4B1=QyAKl@x9-fz8TAn!F@_s)oE zCo^J!|5zDGi8x8pV_V^WAqU>ilH9d#lC?`-BROl{#JU^n&y+K|X`vmVzV`B$9Vg4L zB6h>uk$!FAY4L6Pi?hy<<#R7EsZs4?g50HLh-v;Ma>T37w)f%ZKy5%2|1@5%i?!$n zoE<(C9i(twG5-y{C-#29P|Dx%E?EKHe<8*;wfW3}U8^^CAH@GQ6ZKi_XU`l#wyhXV zp3WId4ir(`k0&)7#{=`@3C@%M57lk8W14n(X&rka#I*W}gdtDtovq!pb+Spr#?d7; z>i8q94)hxs8;A@B{(r$bqob~TRMN4HX8f95ogW^CovE+3&@q5mLOA5!GJG+3CEIUM&q#Y+SqJgD28A**-kkkY*)`RV2 zQ6{xdz6zUvyIG9CqRPcW{U z5$k+Lv>ZN2O6*A}NCsqnd`JW495I=jc!q2$eUp?GeMnZ!ewh^JzfI<+bTg^JO0#G$ z#pBLXq?hd{Nxq5n7Vpi7lsuN6)W#I7ew|s6dCn}-SH2Lr;opKT->1EYjq|`_AHEF~ z`-mCs>)h55AFl9Sx9m0j!gYUV9t5rjTF1TOpN@&+$Q9qTH;g&5H<5XK?_B)vsV%IS z;KI)}l;LdT&lErRqoc4gnEId=o63%Dom5;`sQ*UKx&n3gh1mzmWA9JrfNY&lPE<}> z*Sg!u1o&s}7ynD9pEF7h?VROdE&UK@U|D-O;s#!Kww7Fmf0#JHRXQg!P;8c!ent!U zXZ{5EXXYe#8Ki`DB>37(Uv-!wx{C9Q7vckDA**J!>*mf8GppyGH%1)vG%p)*7kuG< zR_$}$Ztqd~d{W;x8Ydd(c)#^`lhh}Mw+F9Uc+Q|;-5b9v_URb8AN=1wFC5$F%QTcv z8Z0atwYnx-`y)H+R-9_w zJZ_TcIe*^$;(z7B(^^Tv66X*X@oAiwy+&;#bibdM)_)&5)$RAV(|g(my511v>b!G&^O$ha zgV?34bA_ZR(LlsiH_B(|yHN*&;z z`GfQyCN)Af7&e);F4k=q?WRb64EAh4mX*-$x-0i>ot1OW7)JYwgFUQTPq|vkeq!zX zx1DKG={zc*9>hDLzRVwc*tgT}vPFY?6iv^NlzJr;im zIeHN%`lcTmaLegK+;t%MyEzViU)fLF)`S0SozGgT%(KOhTN~dm{;3b!ybQ5!T!1vt z-$C{c@>gAVNA5T}53u$?*usIM>+t*i)Xhq?D>3A>#frIa=m7uB@2vmhjMd9}>CtDx zeQYF3hskX}xmdNn9TlidTb%W(@#4%*L)d$Q1Ki}ta4sGDz$*Qww{M#}Yuks+$GE2J z98VkRJE0Ep;xJdy)Tof-+B1^dnfV#re;NGKIiUGS22;P?dW3M}W2M-){haC$+!-;7 zdo~7%dMw-t|9^(hSsf@x>xaft(}Rs0$BY2}nfqD)Qv4UMeu+sB>Uu27UHQ1XmG~Q1 zowl{Njh`FKSkwH9ar}*!o%BSkZ%0~IeAlp9QD>OHGXCippmcKg6oba{;ql0A z`U=jStMbLG@jp=g=g%PjX&dMo=1`9Ad(Y;L&U!k3lHvX0e|jS9j|m;vZ&H|rc}ugA z$LbTR2YFx~K>jS~K`P$>AvrUMI7<@-gcUtke0Jzg`u4%v@^!Zp&iqFY!zo5C&_?u$-4pWirFFFatnWV(Q;RtQx{wiyO2A>%VJ)w z=d0{k@?SAFF8evlJEA;fi;@C5%%{hn)|s8r#em44i5@ooyKB?5ns>Wd3QEWBJBsrI zI7fD)fnr`g4?GAziQmHQgZPXcufV@BPwTsylEM3H%7>ZNRoF2g(wl&P2Kd(o{*8ct zvzP#>d6=KVJuILtGA>M+mKOQa^t6bx>EXVj*f3vlP`FRKL%5&ZJT62sE-_pl;_EJ| zakLWsSG6a=;a{~brgjy277+PPPNfAY!xyD=8mydtn#snPdw}?7BK>LyC59?u{ao5x z9j3JZ*UeJ-&j9E4)qbubH#fW1a`-B|4PEUjotL<^E%jZa?`a!hyZs5Vt={!@lI@8K zkcDQ1%7@JgRqzn|fyRHpd6+*JJxu&_^N0AkFx~pfDPGliCN2DRydN5XB@X}GT<{=p z`q1j=8LvF-MeDa9U(EQrN?Tst7F+m+-=y^m&oV1#caMzqZjT9d6PJW|NCo~bikG~d zr0=;qO8)6)C;Jfbq5kRVAbubD(%<%Xk#z;S%WH!?+IB{Gx2}osl~njR$=`?WfW1@B zM@(%OqI*U(4#*VIx4yQmhf>01uGllu$zFMZc|iDQ62n^t`McFexLS&u9H%NSIZTzk z>uD#;i}6*)c{_;rJ6W~itcwKaccJf7c@JW759}i@+ckG<$)|pf;+AM{@%(U4iFtaW zSbyFuF*9#k8_r>h$UL0woR|3LqPofn< zM0EsK$G4GQgUtL9IXwU4X(Rc}*FpZVucPum@QG*rW;mUiUR8QE_697m9+)b+YhHv={m5PXXUMzbU88Q>pzBhKkW_{v6ydWz>Fe!mi9 z_A>G~z7ygjI~3$0e$mrj{+f%m{6}h^rR`>AzZ&-#6JK~+i@PJ-B&FET_KWv#9-bI{ z3^x6f42V1m^vcl$@;egKRAFgWlQ(23{3<`FQ|H>CI?zMYT`;~LRNZvW;^*rA?P&Sr zq$69L^*}fyeJiE<%b4TfKjxw}7uy-9cqksE4N%lks$lZW@gS#06Sy3mDFH30@7uMn8 z?7lN+Pf(kqDL@M)l9+qoFOw$0Jr zqJSjdw&6=N+ja9-on}DfQ6kYZL5OejyAlui-NzuWMJ~?n{sZS1Z{`txfLMJ{nD1Q= z9jCQtQl$}y<`cq657XXSb9zeL{y~2 zwH71+Ny&zU7R2vq<4sR!Gm7zTwFq?=&j9wDTH+8V+=PHjjE68X-E5s%1>KBb!;-VFV#8T!|wM~cIdlU|5J?i5ae~w|GoHks-wTpS^VeoK5Qs4 zUeZ`P)Br^8M2j;e%-j^IR%)2sDAq^xSX@ABT%@~r2hL5&AoJgKv5{W_zObgGYXfc$ zpksi_f4WXk$$ORlPygrMtHuCy6S^k&!Pi0Za)gg;Q>33PEHR?fGA-`h@Z>0^L5N?i zPD;E)e_DdnG}^aiVzj5&J;GZ&J;=3fzo%{6Kb)=M(?Bs!=OAkZ_MFiM_-48zw=OXM z9en&h^hG|GAXmxrVV=?~e0NHme~a;)M3GMJf_4T(9%XuA<74CxBJCFoO~xP1)6c2P z)&8IX`>QH^^;#F-!`gt>L48IMr|wc+p6+|dQ~A%@T&?eV&dlKR{_uS#KHUQl@cI`&RK7m?V*PVxqH>wjb2 z`J-wa&~+x}0lFrj`Vvbjorzt4Qu~W~9$-JWLLO0T$u~ZZ(pQ2#6-R@;+6x2SrL+B9 zPNf8TNM?li$g`1i=;=U5xg2rWKE`*x$GYTK_~fW$JY9=eL)WpEe~tV8J-|_VI^0#X zKfzzRG$mBw9UCZpEZR?M6c;4Z0wI3UUxS`29ZvorVqPd4sfBZ8<8j6|N0_JgH=OCc zN_A{*Pk@!(1JQ{nrq%KtaXx>k&M|oJXu;53jhn{JZYX_hNo~$he$58m&wyWcjNJ+6 z{e4c`CdkwOrnX|z4&fFXV-Vt=+&wK$ZeixcinLNfr26q;64RteiCv_RBr(KAk{jqO z-tXriZ$~`wH$ChW{{j}j!5aBzS6c~PH$Wz{#Oh6~{RLx$;-Ag~^gU$vHLPK;fcKr3 zz3gjmJISZ;G5x~Bw*6bgX#3ICvi&N`(fG5}-{L!08~JCLYu@#75O+hjsSop#Y>xJq z%uWiHdZq=5M`nac4Hu+#YA?-FGHN2Y{$;3kvo+4{{V^omZp#R(Y_x=%CXYUpJ9shT zN}NRw;wyB2faYl7FirKL{xHe%)%d5r*44Rs|E@0|S6aW>a!k_}OM`~;35GR=Hf~4r zhE^j^))jV4-Rm5H?;#K1r9(xdw;d>%U<@MnOUo80nXG9tt@uERUbvgYC^=9zYDPqt zcXDu7Mv#|eowrL{m5;OZFk;c%5nGINRVnzudyBerJ$Q+lw=qno=J|P&!q=wshD^ zoMr!l(}Vh}xRAUf%(6l18lUbT{UMDm1tbT}(*wEj-b#QEe z^0B1w4%?(?nSXRZ`?L@r#mW%Bw)`+J#in3aMHT#l_Ck+(-q*GD1s|7FFTh7(e}KFA zX+KxVju7wm$`C(Ead@C;U2L!m#@SWJk-CDL1B9`4yXG{P5a;UKAgT4bwEW?U0* zN!AHw^}>^s$o~c6aQFvbnOfwa z+h$dDVz;v~2r-`@9wmgGV3NXHn0X23nXHs9US_IXJ3dsY6Yj6j1D^HagQ*|psW3xGLek62maxc7yPsHvOouR=vasiEeW2N$u1)iB@We zgf}hZ7?TlNPl^0rC96rsBspv}&QVV$6>F@?WfH!e(c3aEl8$wAFk4!}fPM5A5E}=G!#{WJk<}djQI4Oh_o1+A{L1>L zTutluIBPX+9;YR*8b7(I*!aoXTBj70nF$RAB>7k4vj zmz-hNE|)Shrq?nlsYjRvbDEe{3uMgNg`JcT7hOaB^}(M7|J)cmg)z{)!&UoOmHm|3 zqQNT>SFxQBeF)e?92Avps0aUQ-1q23c#k>Y%8`77b4QAYZ4g#YipCk7B;+vIRiCT- z0oJeV`j>vcXYBML=llDN`atswwEiiqFkM);b)*4eV31?*e`wyOp+~h3{%rWC7*i*R1&(Zx1aZ}FM<{P|@ zeYS6C{xEjj0PlBPPxmt>==b^gx_{?ynmCKU#ZF&?|3`n^{~7U5$3We7OT-zOh`ijk zI!#+`toZpua*+$Z9XiZ6unqmno~NX8oFfji>i1MO?)z zuimOIw@9k4bl0of>JItu4*9R4N3{?BtoY~jA>@=}j%{<(s^4roympg$n4o0nj=BP) zH(?|Cf#!JW!9Os{+KMQy>HoLW-QfMUvORSH|AIW-SA=Ee!F82RI>)v;lV*(7liOU$ zscmi=dIb9*IeauJTw+Of=VRTw(TTiRN!PrtSo0#61Af;(10CPt$23;CG1YqSeD5x?MN}@6=r7ti|GfyBlfU zj{J}sdL;YcD1N_3hMvM0*lo+yRk`ZbSK8Xt7n^1w-`C-~eBFz}9Nv$>r%K0%oUYAd z*fYD_oY4Dvw_||AKi1a2V$8KQl#ZMr+Tp?!-qMJ^>JP47(>hh zcJ-yh=GT>&9u(vo{0)0x-(gPZStoOI0T+Xd!!mot+%S-}F!vq!5&ck8IOwpTc<6|> zU49IRXz0=IgBIZbQKnP-Jb5Rda2?cCWodD=Xn5k$V$)r z)&i7z+m3F>fA9H$`=6U9*s^pDovZaFzu06;b*bqP5Yf=1-v^H}_G#OJe+KwRJa$WF z|7Lsr1DmIg7nG0kYbr8ZSywPbP@Sj$ZcPsFKZxazvA|rTdBPe%z3oTbSfJ-Id-fRA za|1gT)NASLJe>~@l#W{QLiu=O5Yf=1-v=$*10etZQpDhSv|1`{hYHKb*c>ey7LD`j zIl|(>hlGVjuQV3ue$kN6T!z2(ParlX5vSkKE7gnB(< zATA5UGKWtt}Iu>$x*XoN0i|Q&TO&076;(>^U9{oP}>terW zTr{)4U~Ir?$x6{G2(Q-)nQl+1vLh93PsI8fzD{*My#+-5ti zr%J{d9WERTy^1?sWru8T!bZK!iB{nGMY9HT~J>-a$SAN$cm@L1I(fk4`_No(*v3w(DZ<&2Q)pP=>bg-XnH`?1DYPt^nj)ZG(Di{0Zk8R zdO*_ynjX;ffTjmDJ)r3UO%G^#K+^-79?fk4`_No(*v3w(DZ<&2Q)pP z=>bg-XnH`?1DYPt^uRs#zzxky(*t+i1DBb;zjB$F^!dR>LIV5x;5YbvR)q4TtdEL* zi?7#D`Mw@OiTX(&IBGlD*#S>T>oyDZ7n-oe(|x6>}B8=fSQJVnT1hsQ5h( zfU@R&@lmYo1}%RPfAHek$v#v0O@5i`15|;&>ctQ2;iCykJn%)rzSMl&7LRff{h;QX ztI8``{$=#Eny-Ff@KNFl{;2wc1219w{uo@tTUy?p4_&56EgMFi1 z5BE^L%;|en%J!7Sy~Xq%HczQ(@#<15wWQ30NPZlJn2 z{Q=mofRD$$*u%d}n11+LEFTTO2$+>|^;}c`_W#Q-V}CPtjNDrOCSU7jJ=aMf$1f+n z_^6-#4Nwn$g>vq;cz71@6uJZC>rvRCJRL*5>(T0m&(Hya^0>W8@ACce2^}DPl<$|X z8r<|}xut#*sATK;4ZeAKPyeIw<`=p0ee%(*z4&-26IFT2zuEsc$|snxKLGt}q73Zm zXpT?iPH(=_Ne?Z=h+z5VJ>@T|`2=?{QTF1m)nd!j@0eV?$-i9On{TqUr=n|=uL|*} zcUez0!s&_B6`qvBs=`B!v-3-ycTtJ+HcmT~+W1oh~#;Oc|V{`gw> zH1O8)eela^JuIKWuW_^dZ}Cl3Cq|)+fsGi*w~K_ijJGdHSuQPeS;ZfuDxhhRO*}-&if~q|CU?z}6w7im{884Msq;IaU zZ7pXX7av?yRYduLc-zBQ-zCnNQ3xd*0Viq+Nv)zzz^sGdJmKO*}PElAt0WUuO{!Y3SG)ge9A ztA80Zk1b!uRj+=p>VuoD#K+~dAo#RLdKFjQkbO+{m+z^F{Uup`4~zYWZAc${wxNCS z*#`E-rw!_hf3Y7vy>VaV=?!{*j^1tUkB=wovpjmHPd;wYH=o69?>ln;U%dJIaV*VM K(*t+xf&UMKOS8WK diff --git a/apps/dashboard_web/static/pisces-logo.png b/apps/dashboard_web/static/pisces-logo.png deleted file mode 100644 index 8980d6ae9a7c435e75e2ed59784cc6e01819fffb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 21465 zcmb?i^K%{E)4un{YV5SJZ8x@YW7{@w?4(I!+qRv?wi`8Stj6EZ`#-!pb9R5&nKNg0 zclONAK2Magq7(`u0U`hZK#`FaSN+fS|KEX!{ZFfI*}(w-YWXtaBI;fn7u#^jIiv2> z|2})bhCU7KY|?1Z1M)Ih5_J61>R(dr#%;%|T6$7f+4-|(N}HTLEdNyZoa0~lc*MIB z!N_UUSDVRbG_y&_PEkmM#2wrMPGi7NwH=^NAR0-~B`iY!mDGy25tLEi;?>SoRi_~g zGRpsh=r>>fo9-)2`}igG{22JKcm48tG1O8OaPV{5_VXzrUhj_iJ6FT2VW7`h=6~)N z&d*mvmT)FQw^zqEgpb6+psIGYcD!B{%s5kJ?2rxwwu2OtYQ$j46Mv-W1`-aDsaq7P zcRC2NSQ<>{g<$|%8t+*a8Yc=1Z0ll3b8DnXPpCZ)bdloZSB6KM1tX+HJG41*n2(6? zPV!c0wzKTG{AI7I$0(lq&q)PBYFq$KIFmO{8Y@T|h?MWWqzTUVZ_3V;UYYskc6ppyXLjYLVWw+q|p4*kp;R<((!`j3JP9s&ah((pn3ci1{| z;7;1Q3Hsz=0}ux|Gi+(qHEz#-Y{xO^2E${+?-z88+KU_Qp`=UVNCLwrX7B`tLE4TqD#qw&prRc57>lyO* z+nO);;d=jY;A}!rgnFXI=nXLd$|Gv<&~vvuzu#(*!(}LA$J(HbcqsSTjGWOc!-^C3+w&YK%hj)>V-iw$`JC&a{?6nWHOqD zd|a&Xf4OC4Vk3}O{$t!d2Yz38i^v)LNa(@1ayD!2{{cLhx*v@L$E5 zN@2VlbJ~7#n028dhSioF*Iq+!k`m6Uviui5$cRMYcICXPhB4~}bWp{1`;W<4%fYMi z&{}|z2;jb2@Fy{*i15eQq4a&wI8~sutXpN!fK`|Or%4~C%xT~_tqsXwW#nLRp@u49 z4Eba`{M*30Aqgo7d;i-$@&uby5o>g64Jk#)p749dy!+=ZvoksgoA8-`63%5wSj@+e z`jjpGprkNDv>JI1JiSKm6kTM?+f3{HiO@FJw_R@DnGd*EsrgBq>bqs;vW!oc`ddJz%E6!U z3#@itcO9Ap=kUb0Z3BGa7I^6qf)*lc{QA)0Xe>TJ75Rr$4h7C^Qzz*}&fHsn3!G?O z#qfvCsdE=~^EDP<$E@P)nv^Dt`m)uj=qi1bJ?76GD8Lv41O!BeuNs9`%6FUN&c((Z zB;NfLjP7nrV$5*Zurvg7(O2?G45aJCS|4!@zM>Rc_1mJ%L^L)h=ZnewrPl!0kE@Gd zMO_bItN!i^JxoFbKp>dhO3{ytc& zXGRzKNXSesng>MWkAEIL-3X9U4HZ<_%T)c|eplVm^y+|}8jyVwpREa~qwh1q#;ph> zYkgUlRP9>jRk)q%u_3utE7?OKDQt-ank0=(qr4of&3*Q6FwHscJeW6{4{)8sOn^~- z*-0{QMs;u0oOxQy8v-cOAW!MN_1^`MY<*O^RAd>i0Zd(H`3j$# z#CzT*i@{vD-~HMaPgVu~(NQXqj3DxTCY8{UkfqiSn^|BJM4lyUa`cCmReuLUqo3Y~ zYrI{NO@4AizR%1%ZohjMcFUF0h%MF;7;W1XcPm!DaXDX7sihKXn*}2jRVKls(!pDhY#5c~M0Xk=ONiF6H;V#9Ekywp?#jJao-wR4jnKze0%6~L(`bFe?nci%g>#EywQ8bqXT*aKMU+* zpx&Ri;QFfVJcIj5Myxeb0F${p{yMs#@?e>#WvJJz)Dz|zbL&H?N57zQhK#PBcKN$h zoz7*m<*Evz?zwh9B=u5(b_Iy>*v*sV+(BJ2L{vnpKuctwG+4dp(AZ%~2(HvIwB?@> z9xPJF-TP_4Xk1)89+}$Y@$1E$G3amIO5+>t;kuIWM2$QaJF6A|8x;*|n66(qE(2^@ zIu48aqa~1eYM0jDYbrkO_+}W^vtk$BE)mBoe%|DgE0rlgllDV1`5zscm;2A9T*`KY z^LQ+9C|#U1_Uvycr5%Oua`xVKofhhC4W6D(8r|~x0Zr(hFhdEVBjs%W#Oq=;{?MbJ z6rakw{h`2Bfx+R8>MM3W5VJ73y)?TE#|L05f`MZ|2CJGT;?v)x8CP5OYktdC&rM^C znoX|uU&xADI%T!NnaARD;n9viu2a-gxzXUKrS?zS5%zq!)M?Z~)SP=R_L zc6Z~B(F9)(KWh;EZ39VGxdyk$NjO#V(4ND7^rn6qXD zu1M!x1n}GeyDuPWqpMle-PWaZ)}6EP*Am$nH&X%K_UW0yoo1qjVu9=j7!43chltG} zttSMrc)waY$NjI!itGvbJzNfxVSBD)w=}2~!OLq^K1w$yUGBQ%c&89p$!gXP}{HDUm=74yqZtc;@{$RLM&%Q@wFCDc6>^__tFu~`^KCw>+ zD~J&*@3-Ewd> zRfbRCjz`Gb%t6Ya{RvstKVz*P7Q_@)JIf}>DnC9XTsCecI~c8&b22^ais!E{cIRJS zCFV8Z%7GK%++?xePye=%&u(#M(7c(($==91;hN_;a5Z8{N;Jwpn~L zZREEI*TMOs%sKZ^J$RzmCnUYU(N^@RB^-0taOz~SihZ&PVL_9VSdNAiP$sbdJ=qnG{XEvz7?5yDBZv!HK>fh z`?wd5gB@W}TZ@ynV+{LjE8JQjUN&>57F0z?EppA0(EJ^3<$SkQ^ce=8W;5>azbKYD)jJh7ig2wV zsB1hQg|aT8wgcW&e<^ohHo91{raT%onM*E(s>c2Xqx?&@)OE9V^AC_WZV}jWJ1yI1 z^92Zpxsn6kZi>s}xjo-(GnpTLP0TEGy)6vB;&r2&!9%0b0)i|1J7kyAN9~)QU+0I1 zsfWfru0&x~PdaX+#jp!VH1*@i`Zn-(^c>s1-PHWymOTm5atG}=2HOD0=AvK$!_Xuv zm0Qr7JH`8>{<(~>t@8~8$vi@7J3f#!1C=6NgO+)nliGzz<`hNGDg_`0pBB%*T%TZT`nC7C zPc~FUjUN{?_dKthvo_ctnAqPP8fI9$%ZW?Ij9e@y=J>3}XB)BCyDs5DrYS3hRoG_?#6EnuMW zZaqijA^CAj=lP?0!+vEDPWM8x^&7C=>#x%fS)Vw1SWkilbXaDkAZ0X?N@b!4J0Yy* z(GMPngbOJQ!L$eWuj?HO!8cLw^M|FV5-qYE72fnUGd?dr3JktpDHJRSwW%Sg1 zmtZh`zc>#}&PuMo0V-5{5F$y~{_}vA+sv*lDa=absLmmO2s)}i%Cid;dm=`@@(=hi zU*KI({rtat>z2=9!mc_Ztl3e3>VQaw@2B8u^*g7i?^LHt#Geu0TSbrQe~;TK>-9Ij zOqHLC5(BGCwzXtD_f{-$hEBoI25Sp=#$pWsgEYk}6A3JEPZ1u@p{P^x#&BjVx_p2s<|6B;tO8{{%GbvxfA zzDibX&zQMP>RZ?>_ARq~q@CcDbN?=w#vR#tZ?wEkpD8=uG!KRG-i)n(Yqj}Bu@PP9VaV6JA^vuxSGG& zkKc0auNLD+*NqNaUZ`@l3QPHZ`YG(pi!QVc0A{#Kls_`5u1Mo^)oSTeny|(H!aghS zX~EdlM&53X2Q60qtwb_zbg2uyjpK8ryUEG4V-BTVkPENlEZPy~CuUy+$wv@#C5Dd; z23oxyB^g!gFy`O4|N6#4An)4e;RTbzK|hP4hYGduB-;bMF)0CfzZEjA7wtp)MKdO5l-$-zt5Lxv<_Evm5x1V#aN09hINQ_F^#I|+K8Ib@i89(4H$ZG z5{wd1bFy!R5x~ds))7}!1XZ2LOmyVRevo8-wJ6Xh3Ds3z@D7`F0yVG{j+$w4ZeW}A zW>Srb6!B%>l`%$l9lWXj7KyhNLL-3O4`ZAT2D>Bgn=-~^k@QO1MJjIBx*V+zlUUKr z=%2=r)xL!I8`p64y|-qW+r3(J(qIZpOAiE<{cbH`_%tumQL=gP#qm*fcX^7*(Ytb! zXS|-68s#J<6Yf9kV@svjMQ_=RUeG5OpAVCPU{lZ;_oKFCj`wW}@V@W>5;z-0f#6F5 z#>rKrEGX5wK;d7KpMr%aF(Ql^4dmpw7xJ=(73K6@b{7`1aI4nfcq_b8=*}`OcL`$1 zi-9BBn$s`MfRNc3+jKMtzPQ^Tcfu(E$F;U|m^!#=VsVrc{&8lU~0J8TM4?=}_<4mo5`caK8C1L=x?r?!;nN_5v&# zdu09P{pj>Rn+oiRK>LhO3zOpY5x)F^{z~aAti>g(z*I>|A?oRupDeB2T2P;6B6$3a zw^J$4F*4>|!9VLc1NthKQGrfsng$WSN-wtEjC+~Q16tZm*a>YT;mdd*6!Qse!TKuT z+%9%GRLjNpqw#Ru5xo9QKF;$866V8rMY>NjWerbIXBqb0drZZGs7jB|?+2S%7HTl> zYCHgpDEKeMwPbfq%11M44i5Q{r28x^^L2KoEAt_Nv*vT^7lmFM`#@*YV(qT3-(bX} zY4-4LG0ETS;ZQxqfln+8HI!z*u~l0`%SL^*PI}r&!>f3OwmD>)a_$?{nrfxEZS43Z zAxRY{^=|(2ct2Gqc=eC6Kb>P7ao8{Ic`QiLrDT4NmDbZ&w} zQb1;-lywcpH?JneC+$*Eh&mFDj<0h$iG z67HBV%AGX~?{&UMtHB-UY}@=8Fi#raACd16A+?$$&~81C2Zkchbyg4l4}Ylndvbo( zG^{foT%dB9-l0Qf1c{0?2PpabXK<5uR^<*GZuQvT`RqnxLkxn0`s6cPRJ*@38M_hZ zhozokE8bI2J}VYweWe26_5BIsYrJ@Q7xBMmCz5cgK>m6hH%{*kV(a(ggv`_!+p|E7|(*<~k zNZM%I^ODz-CoSp6x$tF3ME=&utV+z_LAYch#ZwcYXDDMe;$6o(mm;;EII<%-*tb}F z^TZV&r+0e{U)#n3SY~^G{Vtx;qu!HuPts`DhqIz92jGZ3U&{E-AgV>zD5BdF2#ldH zf(7=o$09_ow>PHWgV)D8d%AFOUt$3Fyb6$~&lsUH9sAHPMEsqmrN@C4$5XR>Luvqy zMmLi9DML=J%R5nSMh!1bm?lhl+)_F*K)&+)8tzDP%g4#Em%X97T>PQLN4rlO7Ft_Q ze$8*RVz;yJg`uuwm4DqmiqYR?nzUb`Auz}}^U|inY1)}W=;=Z`n`lSuCR zo_Bh<{7PTSXhbFKXLkK?6aG#?*Oi>|(v(rQTJ!hF8>rcimgw4>gTw{w4&0Qs4_vvJ zf|>HgyDujVRbHnt08!$h&C_nP_>|V7_8Y)H2n(@mpniBgtyDmz4Q8$!%Sq!jHUhDf-V9=P*pXi*(E&O&%O%- z4?t(vTd$4o9+7{Fm}&wX22=I>23|Ryl&*HiCAh(L@ox{*xw$?qVnCzAL&_y#l~zD6 zW*BR50;e&!w{I6ECWPtMywa;*YltAE-)$xTWF?lM!s!TlFAULo+S6y6R(z`<@oX3o zBwZ9pWD}DC)0b-$1B79r2N4W7$aaWf-gITQ0}+Iy3oOA1zPc!}qE~Ya+ZN&!4zU6R zN#1`Cv0a&>L}KHSeX_1qM-gG{AxJtf8|<6jl`W-R z%Svu^or{FJx(Ig5km;Rqp9@I&fg;};W4;0j6~t+vTCYUk>|}tXu@9H)Oo(r&QQWv_ z8^zw+%~Zcv8TG!65a;u`eg3m?dfAbS{JqG6Xa~EVRs*)DQp4S$87IJN^xyERVcXpq zFnlIOa{8q@s?b-0=D}c8{Vh&4Ma5~zVh}OTFXh((=585Fw-S()CVw_ymD;p_iv0O) z-!|Id(?iyG{&9PC4n___rZ7FA7$Bx|wkdan!Xi)KMfL;gZYNRlPT8(FHRXuYz#GG? zA)5j>I#TPTKm+UjIdwkqs(VkOiw)qDictiRcKqu2wbQ@Rc^1qX=pZ&2h>rh4FZ_gF z!Jy=hBV;vt9~R|t9c9a%fRgZCP>5hjM^NmMl#2^)PH}PPvRVH8X#R^Lj-q0D4doMQ zbnhtF7|HSS9p_n$e&%Cu0R4M%kP<1B-OV24JyY zaeM^(v`)TITl^G7m_yM~59zU|;NAo4b5sD^U^YiYwXfEoG-6mK}{I8aos=s^4xhI2SNeIooHPN;q zn^5&|t~yX~l#Yje>Gfjy?OwNtuBT8N;cCmDyv7!EFN<%Axvf&s0ZRd^%%IkKf%Ul>KQdF9OO35@YJkS1LdsVP0Q_Rc)k+Yh=9u303;F+dWbkTHv63r zZ@T{0+lgoI$axX~3iuMNbLK3zYuv^L$`0<2l`ZC5$aj0L56-}^(21hh$Fcb0SnUEm9d64?niA;C+;&Br z)WTHJdTv`Rm>*4SUcc%s~~R-MS@CqOk(4|$Q%kW-Kuga#yrAhl}O_gBcu9%>T_LX8kNqyGQ9*Z4%jY|7PO(b7!`EPFvbw5~Ts6Dz>L(u>_1zxG( zCrAQ}&-WGIcdUNC`<^gg0Eh(%U~ z^nCHaTciL%%81)4hfAFtOob*lb@wAb3-sw;ISkb8_^#<{=|d@j(81!-Hu5j{2OQsH zMZVFFw2Tf|NMavHz*pNSBpOniKP(KqsgC7ZDInArFWqBoWrI7l*Y6gCGo?xSFkfso zj@`Xd(4g|&Q3K#PjB;93{R7n4z3X$9V2+KR9=PYMwpY}*>0iTMpQp}4S)@rV zFD>b$qX=#6{O3b(na|?7nGMVx5qPanrMr+L(W5J_?Tab*PSvxb(j?y^*@!`V?RVXn zk(cE@r&TH@(3M(@uM@lfSiV-=qDy^9fNFxU0D>3)RG`k{oR;*PXUsBU`3oOm1R^K3 zlN_bWd(RgPI*sovvSUKz1bz0kJdQ|KJ&%G3mk5xOEcMdzhc(ur@eSF!?hV{vCB6Ve z=YplNe$c2}iwGRZgUFqhN$WG9@WC|mF8%1a*z6=a2i5H>tqp|QTyXuy*gQs(+>@E! z;7e7}fA@{O!TE@#9J2dCgHn_ktq^hulk~0-#gw*fFUF6*8Cm=;ibztejznd|Uvsnc zrXHrZMFV8lsQ^Of^2)({;p$Q73R6zE$2NFuUX`q@`Oq*7*xJME085i|PEyl9{%8#F zO{33oPyt){&<~m^5+1gxkM`UZ=o&$~&Q>T^g=lsfWHIbSV=_iCG~lZR9}@TQW-J@$ z;9AQkDL+xSt5To$9i4?4MugnyU*oKm*`XmjzgMcAu3Yk2uX!1!DQJLDUU`yWCKXOU zqIxQ!V*Ue-rLO7BTtR*_g$OO;KXTSGyMtt(DDoUs!@ho=gIV?Eal7IdIM9wy5L=&v%u z7tkdZM4X}bavK)M0NHb=vkbhE&}UI-$Atth9&W!OqLUj`bvJliMY39Er+&G6RnHxAKn?<<49ZC zJ&$sP+4LZJKr@6MHqlE6qa_KVfLwj|k&Va_Wx3)jne^|&2kXTZVIss8>SyW4S#$M{ zezXHDelUjU^_v#nmLYj&czN#4D=QfG&pe7TOGp8jUDLr7CAc$KQaxE4AyI(_@tlVc zQES`;VAUJV2-5g9xi7^zUFRaQMAxaG%{(J+`(+K5gvT@!T?Ap@5BxRDFzK}DPLI!c z&W8z@kJ+Sv3nE8#HL|3Z+VtYm9nHEA8MTn&_xHcZ@2@eT0YOVHQKts*5ZuB+t@Hq( z%hbR_9OnZ{(5>4ebCpNmQT>dlmgGXfGVnXAXC)juJcLkn$m1#Hs)b-PK)yJnJj`=5 zZUkE-g>44&i5|rzp_PqMDS-b7jEyXc2CeR$aQf`c*saN8ieDr46p=4~Z)9@%F_Ui? z7BtdCYc#c=-{(q7zoQ;3H2__11?Y1ub`VPZ6VjFA*;AHqjsVRw>7$7^MxMaLfLtb0 zC*tl^OO?`<)Sb`|i^$}M=BAH@Fk|B8M>OrRiQc3M9;rlw7Den^JPaYU+px^B!<-3m z-;dR-1P=44F=}ATBM#x%LCYdjLO~jGO#xy$tcy^{0Td=aE0Y6wj0>11lC~<>;m^%$ zHS#!Q@X$RPf-f6df-fdziwT`3@sK?cmO0E44J&E3EIhv&?5v*f_xjI1*oUQ-oK?b} ze|XRk8k9n(T-&JM3^=JT(S=09$}IDAN|)+r=<4rOyGnIjrZR_LiDsa>8cnYB@&P&a z6tyIgG$hbuP@uZIM`}bUA>2j*m1wut_>~(w3)qy4`>LF~p0fwgH|?Djad##5FW+Z| zBW$pUOopDcNy6q_0)9&46Mj63Gk(OYEFSNX*MsSC(%{XP$(UE^I8@P=<{+Rc7eTb_ zuvIhz##4geNB|4^nr-ZYqYF_2b_UuEl&?x;wb}Aw5(EJhRwavqY-Q6dQNY{LU#P{e zs^LmPwPAa|{xqD&|H!i)$YAd&_c1Oaqlu6hhA29CZ^i$ihp~T|!h${O`HSq+AJG7FN*-*+wOdAxc;pJXC|z~Q`Zk{$10ziu4g{&r*pU%<};vex6LA= zhDq*j65TyUFzckDv5cbHp^#)Ade;xw!-ojzFUF)z-H%UM-oivB9wk;R9go`Xe$lcoT8gfi1~xbQE*gr_plksjj?j_nyl}P= zALg#B=&wA2(1o#ZK2EF{U-YEckov!VI*lR4!=(0NBff{;EiXTynX;(4Z}$(Ll%kohATOPwz|yexb)Qh1adwBN+&`2Bm1MUzK(zXBX8?eluN2G+ zoDH=^kK>G+nT{)#XqYG9LSs3^PkpSU@n()viPDidg{X^_hB8aFWipw@yDoIl0s^c&^k_gy~CUyn?R?jS4I-3M3(; zZPY1;#u@8N<`l?akXzgk9yZ?mIxL(NsMhz;OR`5G0aWQAAM<9TLW_C7qxAB!01^H|?+NR9NxrV~OS1F7wDZTL!`7?DQ` zD(q@Q!Jou#sDB= z&sOT&o{aMhx793gi(cnumYn*~Y?~_cSS93tLxdWnqfLFHQt!ix%80p{I+_)W_%i<& zqN80>hq*jW0-`^57Di>c;Z9=teqbQ7oJjqi;w}(W)-J(~{L!L@V&CEB)bz;ji5CK! zO!^z&fulD;r|45&=wN0Lqu^HFe3`2f0lj9ZHNytGQzeB;`NbQcvl&1sqoOUeYMt(9 zt?K`Q==IE-9HI+@M{3$*kLTVY+o$d&JPCPdkQ_6NgwIop89kGi`>7&W2iL#^L%YD6 z7rw*fvpgs^)bj8N0~uuXM6|_?<&3uc0X9L5V3dP#xck)#1VORK0rG6aT(BU4;a6N+ z=XXSQen<51mEm-hi)vCnf1DSl4xVE18XcLU|I$%$BGVI4z6xFoDh`5X^Z$AnDPV1q z(J<}XuzJ%cZ^6k=j#y(TyqPMl-+kjUh8A9#CidC?K= zzut$WXXzI3gGfa6Dp48GO!+o_UVe@o%|p>M1p2Oi1sKU zYEqP{p38o^&`sv8nmENE+Bcu+lEh)j1HL8*_Jexis7A=@GfU%O@E zB|@Ais#uUp7)h#u18C2X3S7|z1@yg)d-`3X$PTfnUj+bqpN1nx?S<2Mr}mU*ov-g7 zg6b(Fk0u{$b)7nm1Cg&e8!4JQ*b4Sq-du|rQe?gQjXS5e1Byd-(kR`WQoOB(?9;`c zcKTE4>BiHrU~amz0r`B+Zxoe){3NSHN~SBq$4k1I2~9jhec%v-ypk_O4)8DBrrs;A zIB$(;uZ@dhFQL|I84dt#$x5JWFZoErYLO;aP^tAbO!yA*Wag;%FW=xgPo*|}vmpJ$ zf7sKXm9VMtoHh^4j=S~F)@yZ!S;c~q=6YEZzCBNC0{+wwIDws7nSm0H&dz25SC~A; zyX0`y#D}XcD&5QVIna2B1oFmv*q;r2LR=Zpodro%lj9XBTk~wDLaPa_316$&IVS_ z6hQy)8?i*Nca+8UD*Ib{QbaT}P1H9H^<7lmX`25=MsAVf=5;zoJeZ0P-MHNV#uiIVWw==Q=JVav%U1uK~elyUe|ElNq+i-Thp z)0f!ENk)f?S0<*r9C;(i?>Ir7k#WljAAhR#2JIWShye`g!w2$H=DtsRNnm)Ognkdc%!*L&abBUnYtA%Uk#SzwhXUp;dw~ zaO|)GnJT%bUT4G3;lqOVPV$xuo;UIYuir{>IUSAr`&*f`R&(%5U5Lzg&?+d*Jyr(NWR!7p{FYRzS?M?|j!>CF2` zTzXK42A~3DeQh^LBY=|@K(1oDPZq<`EDyFLiCv!eC=VE3AeVH;Le$^hz=MHAMs9M- z&8uqvmd9zNysrv6o0vDK#bSWJVF7B$xupB7DQjM;UvUQwK!k&S!p0$Eh-a2^7fx_v zvWj7!ZIjrv^cokQbWz1FxIQ1ky`m0K#3^K^F{@vd_Q$$Q6w}erq;)Hh{u-nJlHV%> zn%`#Cun<)$Xkx#t4{7BnJY?p59V^nfN2}_U#)BU8c66XnCXe|$G4EUXYvTnJoOME``Mu01#)JMRLsoN=dnq2iqf5i9X3HwQWyu+RGp<%`;w*(( zH+_Icg*n76=!|Ad1y!11PivsXFDZnw7eoVRdA#Xyo=_Kd7XF)MWF>{Gmq2bw-KH9V z-v=mwQ)^C1;hqp3gJO&GpD9sSUy5i(dFLj8#sbj+pJBhingk{wcyTTE|M&xl!Dc3u zYGTS_oFKAt(A-gAJ|Le&Vt{IEQj5|Qsu^=y()LZk#0S*;g5dnO!a zTeN?Uq;G_;d&y^596GpK>0Q+Qd0OWr|Bqi~u5{>GG10-{8LBbNW_Hi_F)hIy6A!0$ zW|sSO(&S^qK1xU=-N~3dA09v7Ng_oSeA)T}2VR_tPI*M_QnEkq)|5vJ&-8${@t7@u zOL>2>w!&w^I%kGgsbyoQe*NWS>~Q#CC25qm{KbE`=2s|fqo&%C1E~pY&6TWrFi8B2 z_QbdL;qHgR>13-Y2CQ(tn&%rXHuPS%#!*o$rxBS#iB5IItsoR{XJ-(LNFOWgH6r;JbGIh4*}@rQ~-LO>Qu zp4zkb3N53;tc%aTF@@xkI;x5d8!zwC0rw(AvPUESpK2VlQD1>uuwhgr2Dz{zh{D*Q zLnh=)=AvsFQ~PFoS9H8PFx4Fm@S~d@0{=}wBf6(jvQ-dIE8Bt9eWR+GTwK52hmwV% zO%F>OxkeTrmPRbgS2FOK!{wwxO1c6>cfGH+SROduD*1hR%Xz!OwP)%kAC*UF4=2r_ zK*EE(iA@?f$TxrVR1pf)<-@2r*J)7G`wu=@EuO!aB(Z)^gWRWJYI+yTx$(VKn+yRp zR=blPV#u=^r>I#ALS!aryiJL#?-fMMj9mQK$7+DJyyWCW8py@g%K3O>FMs_KOt>N%$pOT z#6?G~-|lIL$N4%yzyGq2~rJs|Z^^qcUFg zCrpVy5O0(RU_mUabK+x{tAcNgj83lb`A`L~9MG_3b&p-vD@5?5-LUu6Gb25N#z--c z0Vv&dtvKen^$6jFgtZ__*vj5I<9F8dYqZ1S6U{L7F9zPZ2frc|s@~fGe8=pAgj2*7 z%wN}DE6Kigt3+a$L(woHo0u1B+fKW|QyxsvLVYC%__$z(we1zQ?WI+4@*xT1pNqvT zOvA-0VH~mwV<~47g5b-$TYKgnE_k+*G@=(-pXf^NA;P|uSSgCNH~XWm&_hAjXEbcL zB0bO0gm(~(Eth;A)Z_WR$1!SM&C7pdrRNU3(XQ%*_yy(5%_9q2)8LhNZQ5J=U$ElP z5VP%>C7p3FjPQh6X%Pz#0Y{Qn@m6*9XqR4R_R`Ml0vEv(bj>eR|EBK%z%M>)g4*hN zsh`q)qi1}|F!$Xr&akrF%x0IR&jH4rIp~8jG_~364nhrvpdh3Y1e30sti^i5W<P(T&?;XaHSsW2KLtT<_Xe=h1GP=cU&ahwZoHNV<@-wSAmbncY%tpRN`n zmXl;msnwsAfY@wgV$zZBaDj?5uBaE~DBymI>r^b$z}T*YY0XgLA4EXxLlL3Wfg=5v z@Axc&l7+_}FT{*}s%KSS7jr%yLJH+A*apVeelYj&n#nwcG@~75^8NjLKY=H9Je?K# z5`)BlThihA&(DdTu(1*p=JDT^n&)b!fm<6a-U+l5`+kKUe?GdB>~=rW3etLp07Z82 z{ZYV2zFhj6&LkzfFIhTo>B;69Oen3PkX@=7|xgnMSxzH zb2CB3DlviM(sx_1MZvLiECyz`?6h|W??ba|17MFr1>0Iwr0?7HJMwPO%P^rHf4%z9 zoHu{t@q9yANW_l%L+Xz8a2y+avARn7n=-*g!o4`L&)Z%^{ZmlH^Z3;V#OVtx+@l1; z<;b$pp1*qOCU1SW>UDv8-3I^>f<^TAC@~SvUn2B*#$COmO2mjV7<9c;mH9gz;b#Jc zS{5;{_jBCmq^N*l_7eUT#3#{cE2?_NH~LIk109lq;K&{>neq0j0-| zrO=%aHg#yNs*#xCTac+3&-g&0(nV&&b7IDM^Xez|;rjVlCSPPjBTbItIcVNT5a5a> zX?U!2pyuDWa_=KF)cc0;g8&3Y=Cos};(OQ>H(rqtZ{RyL|I;+AVa6dh^OjJjWt2K| zUW(z_Um1P|n-xr^JA%vXzGJV8r_O_)0#3m>GM!3?ii{TJT1CE!9U6xhJ1*XFfHQBq z12HW=sb&}`rN%K`eGLYf2gs#A1ky6yeGX_GIjVs_bc;hiTeGwLZCg29$@5JC^7H4I z@w#IvX^-D_)TbG{)qLLD@F(rj4o(k*wO4^G;swJoN#ed$WLE=c7Un&oV8(s`)>${V z$Sn=eKP#;&p^I*w5Tt&sgLnpaT-kZn<7pJ5;p60cXkp`Y8kITIO(|PnDk+`d12g4< zau@=gogD0Chg)^RDsJ42pIyLv*8AAnfb}ZxmtJ2F$2NfMp0(97!`>x!-OJl#yF&+t(MP?g8O3DO{x6E`$4C+V4Wnez4>We6{;%K{YZ5ENcX7lMy1vD`-I2VG8nym|# z7T@o_uRFcpmTOaNHqE9;?be!TdR@OICH+Oh+=^G}kv28tFsGf)Ljq_pd)J&G@y~N- zGgoj*)ke5KkQ8!&hC~7^%3BIRKdF><`>l@ylRx%fdFdUh1si5VA2x<=w9cRGif#m5 zRFh6*4!YP|T1NXYh?|^mi}2F9%^O0x`?PzifS?Ce)g-^sC0pY8AK!#@KPP{K*v2Yz zuVc{&z()>f$|!EF8IUr+oZ)9lGv&yC5r1$OIxro-KjMD@g8-VTVSTv}NurEejKI%I zl!GA@m)ZW6ArgKL{}2YZ58ASjG*=@s^Wb0rNYOAwjt600R#$tNP*DC^^TR@U$Gz*)R&SlO)|pnm`a!I zZXc8bUj^IyHrp(&072_!+3c`^u#%RWc{k-`SF*;QsrN4|aP+u9K3~sBBkA|z+3kYi zO+n@kbIqUwL}yj74PsfFZwaG8ep9W z(4%_lLd%Pc#i%tQ^rdyEST4b$!d*OGwR#%dJwL|hyw7j1?%Cntt|@fYUANg~ZZ$RF zd>Wgaz#aOvsD4!j`21-Au;0p3FR1SFLAC^FWEbr?5ddUEFh9NX(btJZRgzO2YUrLS zpJ(!MVhmhdLwW;`2~TYuU+`Zx+xoJ;Kg>K~tc%ZNaPBbfte)wpc>7PD1 zUeEMQVVAhueM_;pOR)X#pOaW zBy1<{7R`t&r~RV5an8Kg-mhcGhX5Pq+rg$l7fJvO>_gA&kh5lao2&7hyO)pX?3p46 z2d6w#VPhl;`M41_{uL^)yQX8qI=}72~)FyyZJlP3`uRmkUY7?)J$7*_DUVfWrkob%?z!^ZcO^L$cs z?U)mYvzT1HH>e};dsW5Ysp)-DLL z!`Gpj?>_0~^!u=2rBng;-I@iiRPw~BM}Y*>GOWwOx+2gw#A)I#RwLjS}=Ri2E%AX))T%msa;@Gn22$M=ObtmHYz$0e|R-2eZ`b0A5j=u z{BgWo@L`~TrLXRLUn_yzCsa)|n^oKAe=X>K6@zkgp_-=*Oba_p`OByv?VG9z&NY+L z5-4&e9lH5Vc1K1myDDbvpvK4Tej|tf{jDJco)EJ{bJ$v?XAOx;&>5e4+|y=nq|4OJ zUZ8kLPvw%rExwiUVcYf(`};^EgW=EquV40vfy%~l>FHG&8H`D#eUcI$gsc%HI<(Fh z!nVv4)#@>kk=P1XE#FyQ$mJ}AXL{gkH=zFk5MpnQ zRYEyMt)Nzs*sAuf71X9`?>$m#Rn4jq#NM@6Lu-#i>>YJ#*QQ2|&wu#d|M0xm^^W^~ z`HiMU@Q)p+OC(4Z(FuB8w$2Mohilz3=4~z0*`ux%kCnZWQack zq8S1gslYlekr7jwW2JYn_Cxr^ZU-Ao$;@{(c4evMEbEG;N%)c``e)8Jos*v2c+mmp zFHs-dQ+Qoy9`Eo{ktTSb?mkxyoC%YF9o;Kp()_lVUGbGw7sWkIK)}?Glkwid{!Gx_ zcD9)T2Np;}{U-+rrieSrUQv!K!=wsBoBPz0QMT<1?2@e^v_I0e`c+x)@kPJ3&1q3!I@`e=`3Knp= zd2Fl!we^1ya>khlhI$El zSMu#@tXCH=Z!>)?{9`^Hqy9U?4BxYd>3I%<J1Gb3`6Q3{4?6mGk@ZW-oJkFrsL~rn$TeDp1K+n}caOX`FdB<80md z6vDdt(ZA=k4EOkBnYibcrG)#RIkKM;K~~Of02C>Kw9KrAoY-R3EipXq`Ml5VjQ81b zh~}@WrSCm!6o5tbV7=;m_nrw)zp_VwFV)-WN1+77m23oMdh{tO%#%k=19;h;VfdFdIytKbwnkV6t1TY|Rh zwq%c*EE4l3902$mlkd5q3^5J&0EnJ&By|*MREi1B;Pvhg!d(<4z}X=ROAyq1Mo}->9YKe zQ&9MG7w=P^dm&7BX>jw=L~k9R~mBV^H(Y2pDrTyP>tRRWn7pBbaL7SwPf^o`V@ z{dY-8&yn*vZ}fDIk{{K{E+0>F402PG8n)<~Xy~6L8*s)LhvM!%G*MrvrRwc8Ni5H0 zy`hfLg}r!91zY?uYV2|@`Gff-hFns!R^s+92=bM16kt)e1`9GR|H?OdVv}XK&ss?> zbi^roJpD^G>S0V=d{Mh6yQI8s!%mRHJHF9gel++8dUt8Ea^w3WyVkUTQi(wu`*8Kx zFyUz@zz)LVRmD3GupxiVSeS=yQ_^8%3An>f3}^M(%4pBX7giMsO>=qwq}Fb|Aa68w ze7I%QHy*J^)}=;15T560KvixO(Nl!H&1`8b^0~6oH04Hb8cD*ml%86olYnRAPIi-* z)C=z~V~m#0J>(Rtk1eL0Zw$Rgde`Ifu^VZC5zeVB+sXrr|y|0*0?kMV*JF zuVZ!kxQrdS%s&t6Gh=`86(+2E)r8h!e}BA#@)nY7mb{Fy%s!Ad9sT0+j+NWo%8p6r z^ELB)XG!641(JjbICpVi+oR{{IFU@E&RG1~fsk)4M zub0;|AK@s^QlxqbP_LQSR2>BsQQ z4mm?!YyV8A*@u|VhSATc@&PVY(*aLM{W2>g(F>-mGNx3csXSyJ!*2`<3zUXt&JxE~Es2uAu#ZimQ3 zVc^Fc5yohIq*J*r8Aj0keUOtc3s10D@mZ4I8CdVVU&-)0&wqjyhc+q7FC{W4VHD4W z&XkPtzS*IFFY)caHs;Y1oqeqloQE-*I@?exqoWId#f6sy!bl2Po%&BW-!J(GpH#kS zSr%#iXy+QZ&we8yNVEL-*HF*GY6qqio(_)tf%WCFA z151WGpX^AKpvua*n{D7w39xI9eZ3@KJYK7tMr>Me& ziHvW_cDzGVNXB0MgJ)2nqqfl-MWvL=a*ta?Qn0h(8UvWa0o}007u_%E;TwDwBIx9( z9XFRz8iqsz8XAt-RCW{%AzNdC!EZegdXO(h;rsBO8y-R%>AK;s@@h$2^w?|4;(NeE z-^V0DyI++yTo-4}mb)Scy)1X@{J9au4lT;CCW|A&JlR{{LPx8S3LkzjoF9t>j=MgF_ z;vvEdo%=^|YY%^{<&AGG3dFO{KfIqY_IqE|fK1TcphoG)zVMGPe$MPJ(InP+nTxxvxjp!%MpVx&D>S~QI73th~*V`uBk9c zHYD*y`CY<*SEsXcC)^LRxwYRv~W<+)qE}4Ro=@T9tr~_~W z5=QZ!eSivOah7BgxbBp{Hu6j~#)#I6>5w#F(Vu6QC@wTC3XR;_Sl!=9`?Pq!3K>UD zBL0Fu2fkV){ z+I*=;Kuf>H!az_QMuDWeohy9Zpz#|a`{7zdF$#NQVDRqFjar3DHnYKhr|JH>)`JQ4 zdGbr~APqU1c7o`N?MFGmwl(JsUhYEcSUox{I$ zNa4iY17+1D;7-d?s`SL}`1PMSCD6IV2hgb9o(~(=Zz7BK(XSc4{Sp&eRQcV6-_?eX z)8W&5-7-|`Y~|uFZPSSVpkFNRO3RX#T~J#+ zI$s2r>E2tsEM(`${mfHr+>?+?7@;Tpop{R1Pl@K`T2gyhw)y(i1XqI>Yle#;3E>lG zYib3+Zn`)EbYlWP@>nZfU76p;#*n1D4x$*2@7b@2rH4?XfYtmHPaRcJ@vhq@R@*7k zZ?x6?#mK$i5jI%m3hWxRs?7H}N3Kb*V=$YU8LN5+3#S<7tr`NBoD1veiiwSAWx?51 z|}*M(Nemm2r?klV|8ZWpnq^d5nJ*RlQ;1wBrPeC$M{7 zK`$VH`6E{SEw^~AqDJ}L;{Y1hQ)F{s|K@{U34amkI^xe3aoO&W*xU9IUf`gp4&hFRy9XOBRVlh`qIu&OA{Aw{ zZw$ML%YHc}F(4{>pVNh*wEZ`T^lP4^;Y!?m>z^l6Y=8G<#1Ox)3vneV`!q|0!06qa zIfDdE#7=NOV}*_(yfg3(l_X0@Ho@PgV?*^Ijk@3#GYRnir@^g=^#N8uOIvkJY{}~y z(BrhYA#u()gGwyDjHT#L^jO^xcxJnMUd1{p+t4Y)(5aIL+Q1#&spb@G{dQZ%gSi3y z{1VR~GcvXmPt+p)>smp)l`Y;6CcRp&)vv@5lX0rttUPD5%SyO89=8uGpWD)|1wSz;IYxvp;dyoqO0MIb{i zvZ6};9ld(E3bI!uj>l=VP~4L851GBUt&(hgh9r?8KGw9Ejf;?j%*DKywM;vLn|hMT znAS;)l&a8+DwGEPd8T^}g(Onotwa*35-e*Nl3m?cWcDJ7$v&cJ11FFF>&snXfe9Jal!HE_I~ zQfQXQ{lvOjHuNpj%N7iQMgWc$QXPx`Er;aI@mv!PoMX_~p;}KJvsi1Rnf;EOQbk17 zS2$@#c59XMHEG9}E6CeUDsB^?T+8wyV#symKcP_ICT{6Ge8KmsIM$aHF2Pl1l~a zv!qO^owGm;ExjdaZeHnz@7Xgz?O#Z5R1vf3-p*BeE{bst6b>5D}Ud zIq`!i9wG<_7}SdcC#E`7x$r&p9&lo&=rDxJ@qOa#Af_8B5T>wSA7m9ZO;{Z-cbRYF z;pqR+Qwqg?$GMLJr04NWOZjVh4=cfGqVoA|X!D=L*T2l@Xa9J1`EpTB zkO@YuG5ynRhmO9?o?`-HJJZSudgNrgwU@8~8`MIe8B(Q~mY%}8fYGeAP|~RPR=?Sz zhwv!ZnE-@>Pksna*xCNwTrm}poi@%qq+#leWqcd5r~o!J@+9@a_vvkhxV*60w}#rY z<0bP=5h(bwgNA}7#U~S0RXqW0nFS9qwhuNKU-OV)`s-PCN=nOl0y%NkEba}d6?@e` zTCw3|x5`e;cK#uXHbz#Yq!t-8Sq7H#BAw{#s`5|Veug3myT3yeaws_>{W2^0|?_>HkXB$Osa&j8! zZ?v4(ZT1#Yw|I`Niw=5=YrHpL?N;o}|9`auZ^D90^f#^O-(EZ@I{`11wUlZUtRnvh DgZ#1D diff --git a/apps/dashboard_web/static/pisces.css b/apps/dashboard_web/static/pisces.css index a03c15f..c0b215c 100644 --- a/apps/dashboard_web/static/pisces.css +++ b/apps/dashboard_web/static/pisces.css @@ -1,167 +1,17 @@ /* ================================================================ - PISCES Web UI — Material Design 3 Dual-Theme SOC Analyst Console + PISCES Hub & Dashboard — App-specific styles + Requires: /shared/static/tokens.css, /shared/static/base.css ================================================================ */ -/* ── 1. Design Tokens ─────────────────────────────────── */ - -/* Dark theme (default) */ -[data-theme="dark"] { - /* MD3 surfaces (tonal elevation via colour, not shadow) */ - --surface: #0d0f14; - --surface-container-low: #161a24; - --surface-container: #1a1f2e; - --surface-container-high: #1e2433; - --surface-container-highest:#232840; - - /* Outline */ - --outline: #2a3045; - --outline-dim: #1e2233; - - /* Text */ - --on-surface: #c8ccd8; - --on-surface-dim: #5a6278; - - /* Primary */ - --primary: #4f8ef7; - --primary-dim: #3a7ae8; - - /* Secondary / Tertiary */ - --secondary: #7ec8e3; - --tertiary: #bc8cff; - - /* Semantic */ - --green: #3fb950; --yellow: #d29922; - --red: #f85149; --orange: #e3763c; --purple: #bc8cff; - - /* State layers */ - --state-hover: rgba(79,142,247,0.08); - --state-press: rgba(79,142,247,0.12); - - /* Badges (theme-aware) */ - --badge-green-bg: rgba(63,185,80,0.2); - --badge-red-bg: rgba(248,81,73,0.2); - --badge-yellow-bg: rgba(210,153,34,0.2); - --badge-blue-bg: rgba(79,142,247,0.2); - --badge-gray-bg: rgba(90,98,120,0.3); - - /* Focus ring */ - --focus-ring: rgba(79,142,247,0.25); - - /* Detail panel shadow */ - --panel-shadow: rgba(0,0,0,0.45); - --modal-backdrop: rgba(0,0,0,0.55); -} - -/* Light theme */ -[data-theme="light"] { - --surface: #f8f9fc; - --surface-container-low: #f0f1f5; - --surface-container: #e8eaef; - --surface-container-high: #e1e3e9; - --surface-container-highest:#d8dbe2; - - --outline: #c4c8d4; - --outline-dim: #dcdfe6; - - --on-surface: #1a1c24; - --on-surface-dim: #5c6070; - - --primary: #2563eb; - --primary-dim: #1d4fd8; - - --secondary: #0d7490; - --tertiary: #7c3aed; - - --green: #16a34a; --yellow: #ca8a04; - --red: #dc2626; --orange: #ea580c; --purple: #7c3aed; - - --state-hover: rgba(37,99,235,0.06); - --state-press: rgba(37,99,235,0.10); - - --badge-green-bg: rgba(22,163,74,0.12); - --badge-red-bg: rgba(220,38,38,0.12); - --badge-yellow-bg: rgba(202,138,4,0.12); - --badge-blue-bg: rgba(37,99,235,0.10); - --badge-gray-bg: rgba(92,96,112,0.12); - - --focus-ring: rgba(37,99,235,0.25); - - --panel-shadow: rgba(0,0,0,0.15); - --modal-backdrop: rgba(0,0,0,0.35); -} - -/* Legacy aliases — zero-breakage bridge from old var names */ -:root { - --bg: var(--surface); - --bg2: var(--surface-container-low); - --bg3: var(--surface-container-high); - --surface-1: var(--surface-container-low); - --surface-2: var(--surface-container-high); - --surface-3: var(--surface-container-highest); - --text: var(--on-surface); - --text-dim: var(--on-surface-dim); - --border: var(--outline); - --accent: var(--primary); - --accent2: var(--secondary); - - /* Shape */ - --radius-xs: 4px; - --radius-sm: 8px; - --radius-md: 12px; - --radius-pill: 28px; - - /* Typography */ - --font: "JetBrains Mono","Fira Mono","Courier New",monospace; - --sans: system-ui,-apple-system,sans-serif; -} - - -/* ── 2. Reset & Base ──────────────────────────────────── */ - -*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; } - -html { font-size: 14px; } +/* ── Layout ───────────────────────────────────────────── */ body { - background: var(--surface); - color: var(--on-surface); - font-family: var(--font); height: 100vh; overflow: hidden; display: flex; flex-direction: column; - line-height: 1.5; -} - -h1 { - font-family: var(--sans); - font-size: 1.2rem; - font-weight: 600; - color: var(--secondary); - margin-bottom: 0.3rem; } -h2 { - font-family: var(--sans); - font-size: 1rem; - font-weight: 600; - color: var(--on-surface); - margin-bottom: 0.5rem; -} - -.subtitle { - font-size: 0.8rem; - color: var(--on-surface-dim); - margin-bottom: 1rem; -} - -a { color: var(--primary); text-decoration: none; } -a:hover { text-decoration: underline; } -.ip-link { color: var(--secondary); font-weight: 600; } - - -/* ── 3. Layout ────────────────────────────────────────── */ - /* Nav */ nav { flex-shrink: 0; @@ -283,7 +133,6 @@ main { padding: 1.5rem; } -/* Table-fill mode: header/subtitle stay pinned, only the table scrolls */ main.table-fill { overflow-y: hidden; display: flex; @@ -296,125 +145,7 @@ main.table-fill > .table-wrap { } -/* ── 4. Buttons ───────────────────────────────────────── */ - -.btn { - position: relative; - font-family: var(--sans); - background: var(--surface-container-high); - border: 1px solid var(--outline); - color: var(--on-surface); - padding: 0 14px; - border-radius: var(--radius-pill); - cursor: pointer; - font-size: 0.82rem; - height: 32px; - text-decoration: none; - display: inline-flex; - align-items: center; - gap: 6px; - overflow: hidden; - transition: border-color 0.15s, color 0.15s; -} - -.btn::before { - content: ''; - position: absolute; - inset: 0; - background: transparent; - transition: background 0.15s; - pointer-events: none; -} - -.btn:hover::before { background: var(--state-hover); } -.btn:active::before { background: var(--state-press); } -.btn:hover { border-color: var(--primary); color: var(--primary); } - -.btn:disabled { - opacity: 0.4; - cursor: not-allowed; - pointer-events: none; -} - -.btn-primary { - background: var(--primary); - border-color: var(--primary); - color: #fff; -} -.btn-primary:hover { background: var(--primary-dim); border-color: var(--primary-dim); color: #fff; } -.btn-primary:hover::before { background: transparent; } - -.btn-sm { padding: 0 10px; font-size: 0.78rem; height: 26px; } - -/* Icon-only circle button */ -.btn-icon { - position: relative; - background: transparent; - border: none; - color: var(--on-surface-dim); - width: 28px; - height: 28px; - border-radius: 50%; - cursor: pointer; - display: inline-flex; - align-items: center; - justify-content: center; - padding: 0; - overflow: hidden; - transition: color 0.15s; -} - -.btn-icon::before { - content: ''; - position: absolute; - inset: 0; - border-radius: 50%; - background: transparent; - transition: background 0.15s; - pointer-events: none; -} - -.btn-icon:hover::before { background: var(--state-hover); } -.btn-icon:active::before { background: var(--state-press); } -.btn-icon:hover { color: var(--primary); } - -/* Chevron rotates when expand button is active */ -.btn-icon.active { color: var(--primary); } -.btn-icon .fa-chevron-down { transition: transform 0.2s ease; display: block; } -.btn-icon.active .fa-chevron-down { transform: rotate(180deg); } - -/* Theme toggle button */ -.btn-theme { - position: relative; - background: transparent; - border: none; - color: var(--on-surface-dim); - width: 36px; - height: 36px; - border-radius: 50%; - cursor: pointer; - display: inline-flex; - align-items: center; - justify-content: center; - padding: 0; - overflow: hidden; - transition: color 0.15s; - font-size: 1rem; -} -.btn-theme::before { - content: ''; - position: absolute; - inset: 0; - border-radius: 50%; - background: transparent; - transition: background 0.15s; - pointer-events: none; -} -.btn-theme:hover::before { background: var(--state-hover); } -.btn-theme:hover { color: var(--primary); } - - -/* ── 5. Tables ────────────────────────────────────────── */ +/* ── Tables ───────────────────────────────────────────── */ .table-wrap { overflow-x: auto; @@ -432,7 +163,6 @@ table { thead th:last-child, tbody td:last-child { border-right: 1px solid var(--outline); } -/* Column width classes — min-width hints, auto layout sizes to content */ .col-num { min-width: 42px; } .col-time { min-width: 120px; } .col-sensor { min-width: 80px; } @@ -444,7 +174,6 @@ tbody td:last-child { border-right: 1px solid var(--outline); } .col-freq { min-width: 52px; } .col-detail { min-width: 44px; } -/* Risk bar visual (3-block tier indicator) */ .risk-bar { display: inline-flex; gap: 2px; align-items: center; } .rb { width: 8px; height: 12px; border-radius: 2px; } .rb-low { background: var(--green); } @@ -467,7 +196,6 @@ thead th { user-select: none; } -/* Column resize handle */ .resize-handle { position: absolute; right: 0; @@ -524,7 +252,7 @@ td.num, th.num { text-align: right; font-variant-numeric: tabular-nums; } td.zero { color: var(--on-surface-dim); } -/* ── 6. Detail Panel ──────────────────────────────────── */ +/* ── Detail Panel ─────────────────────────────────────── */ tr.detail-row td { background: var(--surface-container-low); @@ -543,7 +271,6 @@ tr.detail-row td { .detail-grid .dk { color: var(--on-surface-dim); } .detail-grid .dv { color: var(--on-surface); word-break: break-all; } -/* Slide-in detail panel */ .detail-panel-slide { position: fixed; top: 0; right: 0; @@ -601,7 +328,6 @@ tr.detail-row td { gap: 12px; } -/* Propagate flex layout through the HTMX swap target div */ #detail-panel-content { display: flex; flex-direction: column; @@ -617,12 +343,10 @@ tr.detail-row td { margin-bottom: 12px; } -/* Stack enrich-cols vertically inside the narrow panel */ .detail-panel-slide .enrich-cols { grid-template-columns: 1fr; } -/* Ensure enrich columns stretch full width inside panel */ .detail-panel-slide .enrich-col { display: flex; flex-direction: column; @@ -633,9 +357,8 @@ tr.detail-row td { } -/* ── 7. Enrichment ────────────────────────────────────── */ +/* ── Enrichment ───────────────────────────────────────── */ -/* Side-by-side enrichment grid */ .enrich-cols { display: grid; grid-template-columns: 1fr 1fr; @@ -657,10 +380,8 @@ tr.detail-row td { gap: 6px; } -/* Legacy enrich triggers */ .enrich-triggers { display: flex; flex-wrap: wrap; gap: 0.4rem; margin-top: 0.6rem; } -/* Enrich card */ .enrich-card { border: 2px solid var(--outline); border-radius: var(--radius-md); @@ -722,22 +443,8 @@ tr.detail-row td { .cve-list { color: var(--red); } -.badge { - display: inline-block; - padding: 1px 7px; - border-radius: 10px; - font-family: var(--sans); - font-size: 0.75rem; - font-weight: 600; -} -.badge-green { background: var(--badge-green-bg); color: var(--green); } -.badge-red { background: var(--badge-red-bg); color: var(--red); } -.badge-yellow { background: var(--badge-yellow-bg); color: var(--yellow); } -.badge-blue { background: var(--badge-blue-bg); color: var(--primary); } -.badge-gray { background: var(--badge-gray-bg); color: var(--on-surface-dim); } - -/* ── 7b. Mantis Ticket Card ───────────────────────────── */ +/* ── Mantis Ticket Card ───────────────────────────────── */ .mantis-lookup-btns { display: flex; @@ -823,7 +530,7 @@ tr.detail-row td { } -/* ── 8. Filter Forms ──────────────────────────────────── */ +/* ── Filter Forms ─────────────────────────────────────── */ .filter-form { background: var(--surface-container-low); @@ -860,7 +567,6 @@ tr.detail-row td { box-shadow: 0 0 0 2px var(--focus-ring); } -/* Filter creation form (detail panel) */ .filter-action-row { padding-top: 8px; margin-top: 6px; @@ -911,13 +617,12 @@ tr.detail-row td { margin-top: 4px; } -/* Filter zone fills panel width */ [id^="filter-zone-"] { width: 100%; } -/* ── 9. Pagination ────────────────────────────────────── */ +/* ── Pagination ───────────────────────────────────────── */ .pagination-bar { display: flex; @@ -948,7 +653,7 @@ tr.detail-row td { } -/* ── 10. Modals ───────────────────────────────────────── */ +/* ── Modals ───────────────────────────────────────────── */ .modal-backdrop { display: none; @@ -1001,7 +706,6 @@ tr.detail-row td { padding: 6px 0; } -/* Notice summary rows */ .ns-meta { font-size: 0.75rem; color: var(--on-surface-dim); @@ -1055,7 +759,7 @@ tr.detail-row td { .ns-row:hover .ns-bar-fill { opacity: 1; } -/* ── 10b. Sensor Summary Modal ────────────────────────── */ +/* ── Sensor Summary Modal ─────────────────────────────── */ .ss-meta { font-family: var(--sans); font-size: 0.78rem; @@ -1098,17 +802,8 @@ tr.detail-row td { } -/* ── 11. HTMX Indicators ─────────────────────────────── */ +/* ── Spinner ──────────────────────────────────────────── */ -.htmx-indicator { - display: none; - font-size: 0.8rem; - color: var(--on-surface-dim); - padding: 4px 8px; -} -.htmx-request .htmx-indicator { display: inline; } - -/* Spinner in table body */ .loading-row td { text-align: center; color: var(--on-surface-dim); @@ -1116,40 +811,8 @@ tr.detail-row td { font-style: italic; } -/* Global HTMX progress bar */ -#htmx-bar { - position: fixed; - top: 0; left: 0; right: 0; - height: 3px; - z-index: 9999; - pointer-events: none; - overflow: hidden; -} -#htmx-bar::after { - content: ''; - display: block; - height: 100%; - width: 45%; - background: var(--primary); - border-radius: 0 2px 2px 0; - transform: translateX(-120%); -} -#htmx-bar.htmx-bar-active::after { - animation: htmx-sweep 1.4s ease-in-out infinite; -} -#htmx-bar.htmx-bar-done::after { - animation: none; - transform: translateX(280%); - transition: transform 0.25s ease, opacity 0.25s ease; - opacity: 0; -} -@keyframes htmx-sweep { - 0% { transform: translateX(-120%); } - 100% { transform: translateX(280%); } -} - -/* ── 12. IP Pivot ─────────────────────────────────────── */ +/* ── IP Pivot ─────────────────────────────────────────── */ .pivot-section { margin-bottom: 2rem; @@ -1175,10 +838,8 @@ tr.detail-row td { letter-spacing: 0.05em; } -.empty-note { color: var(--on-surface-dim); font-size: 0.82rem; font-style: italic; } - -/* ── 13. Utilities ────────────────────────────────────── */ +/* ── Utilities ────────────────────────────────────────── */ .page-header { display: flex; @@ -1193,7 +854,6 @@ tr.detail-row td { color: var(--on-surface-dim); } -/* Checkbox */ .check-label { display: flex; align-items: center; @@ -1202,21 +862,3 @@ tr.detail-row td { cursor: pointer; color: var(--on-surface); } - -/* Org identity icons */ -.org-icon { - font-size: 0.72rem; - margin-right: 3px; - opacity: 0.85; - vertical-align: middle; -} -.org-icon.org-cdn { color: var(--primary); } -.org-icon.org-cloud { color: var(--secondary); } -.org-icon.org-scanner { color: var(--yellow); } -.org-icon.org-research { color: var(--purple); } -.org-icon.org-private { color: var(--on-surface-dim); } - -/* Links */ -.mt-1 { margin-top: 0.5rem; } -.mt-2 { margin-top: 1rem; } -.mb-1 { margin-bottom: 0.5rem; } diff --git a/apps/dashboard_web/templates/base.html b/apps/dashboard_web/templates/base.html index 0051f6a..79f9888 100644 --- a/apps/dashboard_web/templates/base.html +++ b/apps/dashboard_web/templates/base.html @@ -4,14 +4,10 @@ {% block title %}PISCES · Dashboard{% endblock title %} - - + + + +
diff --git a/apps/opensearch_web/templates/base.html b/apps/opensearch_web/templates/base.html index cb0ba8b..2b8a943 100644 --- a/apps/opensearch_web/templates/base.html +++ b/apps/opensearch_web/templates/base.html @@ -59,10 +59,6 @@
-
From a5676a875cecbee7ea2b53d26d43660a4ccd4405 Mon Sep 17 00:00:00 2001 From: liamadale Date: Wed, 29 Apr 2026 22:09:13 -0700 Subject: [PATCH 036/109] feat: add Hub settings page with theme dropdown Add a /settings route to the Hub with a dropdown to select dark or light theme. The choice is persisted in localStorage and applies across all PISCES apps. The Hub nav bar now shows a gear icon linking to the settings page instead of the inline theme toggle button. --- apps/hub/app.py | 4 + apps/hub/templates/index.html | 6 +- apps/hub/templates/settings.html | 139 +++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 3 deletions(-) create mode 100644 apps/hub/templates/settings.html diff --git a/apps/hub/app.py b/apps/hub/app.py index 039d76d..cf4eac4 100644 --- a/apps/hub/app.py +++ b/apps/hub/app.py @@ -116,4 +116,8 @@ def create_app() -> Flask: def index(): return render_template("index.html", stats=_gather_stats(), version=_VERSION, git=_GIT_INFO) + @app.route("/settings") + def settings(): + return render_template("settings.html") + return app diff --git a/apps/hub/templates/index.html b/apps/hub/templates/index.html index 82c4cf2..9160f31 100644 --- a/apps/hub/templates/index.html +++ b/apps/hub/templates/index.html @@ -123,9 +123,9 @@
diff --git a/apps/hub/templates/settings.html b/apps/hub/templates/settings.html new file mode 100644 index 0000000..4aaeb38 --- /dev/null +++ b/apps/hub/templates/settings.html @@ -0,0 +1,139 @@ + + + + + + PISCES · Settings + + + + + + + + + + + +
+

Settings

+

Preferences are saved locally and apply across all PISCES apps

+
+ +
+
+
+
+
Theme
+
Controls the colour scheme for all apps
+
+ +
+
+ + + + + From afac74038f4bdf8ade5f49906a7de6ae1ccec098 Mon Sep 17 00:00:00 2001 From: liamadale Date: Wed, 29 Apr 2026 22:14:05 -0700 Subject: [PATCH 037/109] fix: use CSS variables for ECharts colors so charts work on light theme MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All dashboard ECharts (tickets, mantis, opensearch, overview) had hardcoded dark-theme hex colors for axis labels, tooltips, grid lines, and legends. On light theme these were illegible — dark text on a light background. Replace all hardcoded values with getComputedStyle reads of the design tokens (--on-surface, --on-surface-dim, --outline, --surface-container, --surface) so charts adapt to whichever theme is active. --- .../mantis/templates/mantis/section.html | 25 ++++++++++++------- .../templates/opensearch/malcolm_grid.html | 14 ++++++++--- .../templates/opensearch/section.html | 20 +++++++++------ .../overview/templates/overview/section.html | 15 +++++++---- .../tickets/templates/tickets/section.html | 24 +++++++++++------- 5 files changed, 64 insertions(+), 34 deletions(-) diff --git a/apps/dashboard_web/mantis/templates/mantis/section.html b/apps/dashboard_web/mantis/templates/mantis/section.html index 7e553f8..7cdf13c 100644 --- a/apps/dashboard_web/mantis/templates/mantis/section.html +++ b/apps/dashboard_web/mantis/templates/mantis/section.html @@ -72,8 +72,15 @@ var ATTACK_TYPES = {{ data.attack_types | tojson }}; var TIMELINE = {{ data.timeline | tojson }}; - var AXIS_DIM = { color: '#5a6278', fontSize: 11 }; - var GRID_LINE = { lineStyle: { color: '#2a3045' } }; + var S = getComputedStyle(document.documentElement); + var C_TEXT = S.getPropertyValue('--on-surface').trim(); + var C_TEXT_DIM = S.getPropertyValue('--on-surface-dim').trim(); + var C_OUTLINE = S.getPropertyValue('--outline').trim(); + var C_SURFACE = S.getPropertyValue('--surface-container').trim(); + var C_BG = S.getPropertyValue('--surface').trim(); + + var AXIS_DIM = { color: C_TEXT_DIM, fontSize: 11 }; + var GRID_LINE = { lineStyle: { color: C_OUTLINE } }; /* Attack type donut */ if (ATTACK_TYPES.length) { @@ -83,13 +90,13 @@ backgroundColor: 'transparent', tooltip: { trigger: 'item', - backgroundColor: '#1a1f2e', borderColor: '#2a3045', - textStyle: { color: '#c8ccd8' }, + backgroundColor: C_SURFACE, borderColor: C_OUTLINE, + textStyle: { color: C_TEXT }, formatter: '{b}: {c} ({d}%)' }, legend: { type: 'scroll', orient: 'vertical', right: '2%', top: 'middle', - textStyle: { color: '#c8ccd8', fontSize: 11 } + textStyle: { color: C_TEXT, fontSize: 11 } }, series: [{ type: 'pie', @@ -98,7 +105,7 @@ data: ATTACK_TYPES.map(function(d, i) { return { name: d.name, value: d.value, itemStyle: { color: DONUT_COLORS[i % DONUT_COLORS.length] } }; }), - itemStyle: { borderColor: '#0d1117', borderWidth: 2 }, + itemStyle: { borderColor: C_BG, borderWidth: 2 }, label: { show: false } }] }); @@ -110,12 +117,12 @@ var c3 = echarts.init(document.getElementById('chart-timeline')); c3.setOption({ backgroundColor: 'transparent', - tooltip: { trigger: 'axis', backgroundColor: '#1a1f2e', borderColor: '#2a3045', textStyle: { color: '#c8ccd8' } }, + tooltip: { trigger: 'axis', backgroundColor: C_SURFACE, borderColor: C_OUTLINE, textStyle: { color: C_TEXT } }, grid: { left: 10, right: 20, top: 10, bottom: 10, containLabel: true }, xAxis: { type: 'category', data: TIMELINE.months, - axisLabel: { color: '#5a6278', rotate: 35, fontSize: 11 }, - axisLine: { lineStyle: { color: '#2a3045' } } + axisLabel: { color: C_TEXT_DIM, rotate: 35, fontSize: 11 }, + axisLine: { lineStyle: { color: C_OUTLINE } } }, yAxis: { type: 'value', axisLabel: AXIS_DIM, splitLine: GRID_LINE }, series: [{ diff --git a/apps/dashboard_web/opensearch/templates/opensearch/malcolm_grid.html b/apps/dashboard_web/opensearch/templates/opensearch/malcolm_grid.html index 06a6fc0..ea8a694 100644 --- a/apps/dashboard_web/opensearch/templates/opensearch/malcolm_grid.html +++ b/apps/dashboard_web/opensearch/templates/opensearch/malcolm_grid.html @@ -110,11 +110,17 @@ (function() { var PANELS = {{ panels | tojson }}; - var ADM = { color: '#5a6278', fontSize: 11 }; - var GL = { lineStyle: { color: '#2a3045' } }; + var S = getComputedStyle(document.documentElement); + var C_TEXT = S.getPropertyValue('--on-surface').trim(); + var C_TEXT_DIM = S.getPropertyValue('--on-surface-dim').trim(); + var C_OUTLINE = S.getPropertyValue('--outline').trim(); + var C_SURFACE = S.getPropertyValue('--surface-container').trim(); + + var ADM = { color: C_TEXT_DIM, fontSize: 11 }; + var GL = { lineStyle: { color: C_OUTLINE } }; var TT = { trigger: 'axis', axisPointer: { type: 'shadow' }, - backgroundColor: '#1a1f2e', borderColor: '#2a3045', textStyle: { color: '#c8ccd8' } + backgroundColor: C_SURFACE, borderColor: C_OUTLINE, textStyle: { color: C_TEXT } }; function hbar(id, labels, counts, color) { @@ -126,7 +132,7 @@ grid: { left: 10, right: 20, top: 10, bottom: 10, containLabel: true }, xAxis: { type: 'value', axisLabel: ADM, splitLine: GL }, yAxis: { type: 'category', data: labels.slice().reverse(), - axisLabel: { color: '#c8ccd8', fontSize: 11, overflow: 'truncate', width: 280 } }, + axisLabel: { color: C_TEXT, fontSize: 11, overflow: 'truncate', width: 280 } }, series: [{ type: 'bar', data: counts.slice().reverse(), itemStyle: { color: color }, barMaxWidth: 24 }] }); diff --git a/apps/dashboard_web/opensearch/templates/opensearch/section.html b/apps/dashboard_web/opensearch/templates/opensearch/section.html index 8880827..c2bd447 100644 --- a/apps/dashboard_web/opensearch/templates/opensearch/section.html +++ b/apps/dashboard_web/opensearch/templates/opensearch/section.html @@ -70,13 +70,19 @@ var SENSORS = {{ data.sensors | tojson }}; var TOP_IPS = {{ data.top_ips | tojson }}; - var AXIS_LABEL = { color: '#c8ccd8', fontSize: 12 }; - var AXIS_DIM = { color: '#5a6278', fontSize: 11 }; - var GRID_LINE = { lineStyle: { color: '#2a3045' } }; + var S = getComputedStyle(document.documentElement); + var C_TEXT = S.getPropertyValue('--on-surface').trim(); + var C_TEXT_DIM = S.getPropertyValue('--on-surface-dim').trim(); + var C_OUTLINE = S.getPropertyValue('--outline').trim(); + var C_SURFACE = S.getPropertyValue('--surface-container').trim(); + + var AXIS_LABEL = { color: C_TEXT, fontSize: 12 }; + var AXIS_DIM = { color: C_TEXT_DIM, fontSize: 11 }; + var GRID_LINE = { lineStyle: { color: C_OUTLINE } }; var TOOLTIP_STYLE = { trigger: 'axis', axisPointer: { type: 'shadow' }, - backgroundColor: '#1a1f2e', borderColor: '#2a3045', - textStyle: { color: '#c8ccd8' } + backgroundColor: C_SURFACE, borderColor: C_OUTLINE, + textStyle: { color: C_TEXT } }; function tsLabel(v) { @@ -90,9 +96,9 @@ var c = echarts.init(el); c.setOption({ backgroundColor: 'transparent', - tooltip: { trigger: 'axis', backgroundColor: '#1a1f2e', borderColor: '#2a3045', textStyle: { color: '#c8ccd8' } }, + tooltip: { trigger: 'axis', backgroundColor: C_SURFACE, borderColor: C_OUTLINE, textStyle: { color: C_TEXT } }, grid: { left: 10, right: 20, top: 10, bottom: 10, containLabel: true }, - xAxis: { type: 'category', data: ts, axisLabel: { color: '#5a6278', fontSize: 11, rotate: 35, formatter: tsLabel }, axisLine: { lineStyle: { color: '#2a3045' } } }, + xAxis: { type: 'category', data: ts, axisLabel: { color: C_TEXT_DIM, fontSize: 11, rotate: 35, formatter: tsLabel }, axisLine: { lineStyle: { color: C_OUTLINE } } }, yAxis: { type: 'value', axisLabel: AXIS_DIM, splitLine: GRID_LINE }, series: [{ type: 'line', data: counts, smooth: true, lineStyle: { color: color, width: 2 }, itemStyle: { color: color }, areaStyle: { color: { type: 'linear', x:0, y:0, x2:0, y2:1, colorStops: [{ offset:0, color: color.replace(')', ',0.3)').replace('rgb','rgba') }, { offset:1, color: color.replace(')', ',0.02)').replace('rgb','rgba') }] } } diff --git a/apps/dashboard_web/overview/templates/overview/section.html b/apps/dashboard_web/overview/templates/overview/section.html index 5a927d4..07354f5 100644 --- a/apps/dashboard_web/overview/templates/overview/section.html +++ b/apps/dashboard_web/overview/templates/overview/section.html @@ -188,20 +188,25 @@ var NOTICE = {{ data.notice_over_time | tojson }}; var SURICATA = {{ data.suricata_over_time | tojson }}; var TS = NOTICE.timestamps.length >= SURICATA.timestamps.length ? NOTICE.timestamps : SURICATA.timestamps; + var S = getComputedStyle(document.documentElement); + var C_TEXT = S.getPropertyValue('--on-surface').trim(); + var C_TEXT_DIM = S.getPropertyValue('--on-surface-dim').trim(); + var C_OUTLINE = S.getPropertyValue('--outline').trim(); + var C_SURFACE = S.getPropertyValue('--surface-container').trim(); var chart = echarts.init(document.getElementById('chart-alerts-trend')); chart.setOption({ backgroundColor: 'transparent', - tooltip: { trigger: 'axis', backgroundColor: '#1a1f2e', borderColor: '#2a3045', textStyle: { color: '#c8ccd8' } }, - legend: { data: ['Zeek Notices', 'Suricata Alerts'], textStyle: { color: '#c8ccd8', fontSize: 11 }, top: 0 }, + tooltip: { trigger: 'axis', backgroundColor: C_SURFACE, borderColor: C_OUTLINE, textStyle: { color: C_TEXT } }, + legend: { data: ['Zeek Notices', 'Suricata Alerts'], textStyle: { color: C_TEXT, fontSize: 11 }, top: 0 }, grid: { left: 10, right: 20, top: 30, bottom: 10, containLabel: true }, xAxis: { type: 'category', data: TS, - axisLabel: { color: '#5a6278', fontSize: 11, rotate: 35, + axisLabel: { color: C_TEXT_DIM, fontSize: 11, rotate: 35, formatter: function(v) { var d = new Date(v); return d.toLocaleString([], {month:'short',day:'numeric',hour:'2-digit',minute:'2-digit'}); } }, - axisLine: { lineStyle: { color: '#2a3045' } } + axisLine: { lineStyle: { color: C_OUTLINE } } }, - yAxis: { type: 'value', axisLabel: { color: '#5a6278', fontSize: 11 }, splitLine: { lineStyle: { color: '#2a3045' } } }, + yAxis: { type: 'value', axisLabel: { color: C_TEXT_DIM, fontSize: 11 }, splitLine: { lineStyle: { color: C_OUTLINE } } }, series: [ { name: 'Zeek Notices', type: 'line', data: NOTICE.counts, smooth: true, lineStyle: { color: '#f85149', width: 2 }, itemStyle: { color: '#f85149' }, diff --git a/apps/dashboard_web/tickets/templates/tickets/section.html b/apps/dashboard_web/tickets/templates/tickets/section.html index 6461107..f6b4ecd 100644 --- a/apps/dashboard_web/tickets/templates/tickets/section.html +++ b/apps/dashboard_web/tickets/templates/tickets/section.html @@ -59,11 +59,17 @@ var ESC = {{ data.escalation | tojson }}; var TL = {{ data.timeline | tojson }}; - var ADM = { color: '#5a6278', fontSize: 11 }; - var GL = { lineStyle: { color: '#2a3045' } }; + var S = getComputedStyle(document.documentElement); + var C_TEXT = S.getPropertyValue('--on-surface').trim(); + var C_TEXT_DIM = S.getPropertyValue('--on-surface-dim').trim(); + var C_OUTLINE = S.getPropertyValue('--outline').trim(); + var C_SURFACE = S.getPropertyValue('--surface-container').trim(); + + var ADM = { color: C_TEXT_DIM, fontSize: 11 }; + var GL = { lineStyle: { color: C_OUTLINE } }; var TT = { trigger: 'axis', axisPointer: { type: 'shadow' }, - backgroundColor: '#1a1f2e', borderColor: '#2a3045', - textStyle: { color: '#c8ccd8' } }; + backgroundColor: C_SURFACE, borderColor: C_OUTLINE, + textStyle: { color: C_TEXT } }; /* Escalation grouped bar */ if (ESC.orgs.length) { @@ -71,11 +77,11 @@ c1.setOption({ backgroundColor: 'transparent', tooltip: TT, - legend: { data: ['Total', 'Escalated'], textStyle: { color: '#c8ccd8', fontSize: 11 }, top: 0 }, + legend: { data: ['Total', 'Escalated'], textStyle: { color: C_TEXT, fontSize: 11 }, top: 0 }, grid: { left: 10, right: 20, top: 30, bottom: 10, containLabel: true }, xAxis: { type: 'value', axisLabel: ADM, splitLine: GL }, yAxis: { type: 'category', data: ESC.orgs.slice().reverse(), - axisLabel: { color: '#c8ccd8', fontSize: 11, overflow: 'truncate', width: 200 } }, + axisLabel: { color: C_TEXT, fontSize: 11, overflow: 'truncate', width: 200 } }, series: [ { name: 'Total', type: 'bar', data: ESC.totals.slice().reverse(), itemStyle: { color: '#4f8ef7' }, barMaxWidth: 20 }, @@ -91,11 +97,11 @@ var c2 = echarts.init(document.getElementById('chart-tk-timeline')); c2.setOption({ backgroundColor: 'transparent', - tooltip: { trigger: 'axis', backgroundColor: '#1a1f2e', borderColor: '#2a3045', textStyle: { color: '#c8ccd8' } }, + tooltip: { trigger: 'axis', backgroundColor: C_SURFACE, borderColor: C_OUTLINE, textStyle: { color: C_TEXT } }, grid: { left: 10, right: 20, top: 10, bottom: 10, containLabel: true }, xAxis: { type: 'category', data: TL.labels, - axisLabel: { color: '#5a6278', fontSize: 11, rotate: 35 }, - axisLine: { lineStyle: { color: '#2a3045' } } }, + axisLabel: { color: C_TEXT_DIM, fontSize: 11, rotate: 35 }, + axisLine: { lineStyle: { color: C_OUTLINE } } }, yAxis: { type: 'value', axisLabel: ADM, splitLine: GL }, series: [{ type: 'line', data: TL.counts, smooth: true, lineStyle: { color: '#4f8ef7', width: 2 }, itemStyle: { color: '#4f8ef7' }, From 8c4cf0a4c718642c3958772baf899dca5c406a88 Mon Sep 17 00:00:00 2001 From: liamadale Date: Wed, 29 Apr 2026 22:22:42 -0700 Subject: [PATCH 038/109] feat: add 8 community themes (Gruvbox, Tokyo Night, Catppuccin) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename existing themes to PISCES Dark and PISCES Light, then add: - Gruvbox Dark & Light (morhetz/gruvbox palette) - Tokyo Night Dark & Light (enkia/tokyo-night-vscode-theme palette) - Catppuccin Latte, Frappé, Macchiato, Mocha (catppuccin/catppuccin) All 10 themes define the full set of design tokens in tokens.css. The settings dropdown groups themes by family using optgroups. theme.js auto-migrates users who had the old 'dark'/'light' values in localStorage to the new 'pisces-dark'/'pisces-light' names. Also updates data-theme defaults in all base templates and fixes color-scheme CSS selectors to cover all light theme variants. --- apps/dashboard_web/templates/base.html | 12 +- apps/hub/templates/index.html | 2 +- apps/hub/templates/settings.html | 22 +- apps/mantis_explorer/static/me.css | 10 +- apps/mantis_explorer/templates/base.html | 2 +- apps/opensearch_web/templates/base.html | 2 +- apps/shared/static/theme.js | 27 +- apps/shared/static/tokens.css | 376 ++++++++++++++++++++++- apps/threat_model/templates/base.html | 2 +- 9 files changed, 420 insertions(+), 35 deletions(-) diff --git a/apps/dashboard_web/templates/base.html b/apps/dashboard_web/templates/base.html index befc7db..57d8054 100644 --- a/apps/dashboard_web/templates/base.html +++ b/apps/dashboard_web/templates/base.html @@ -1,5 +1,5 @@ - + @@ -77,8 +77,14 @@ transition: border-color 0.15s, box-shadow 0.15s; color-scheme: dark; } - [data-theme="light"] .dash-controls select, - [data-theme="light"] .dash-controls input[type="date"] { + [data-theme="pisces-light"] .dash-controls select, + [data-theme="pisces-light"] .dash-controls input[type="date"], + [data-theme="gruvbox-light"] .dash-controls select, + [data-theme="gruvbox-light"] .dash-controls input[type="date"], + [data-theme="tokyonight-light"] .dash-controls select, + [data-theme="tokyonight-light"] .dash-controls input[type="date"], + [data-theme="catppuccin-latte"] .dash-controls select, + [data-theme="catppuccin-latte"] .dash-controls input[type="date"] { color-scheme: light; } .dash-controls select:focus, diff --git a/apps/hub/templates/index.html b/apps/hub/templates/index.html index 9160f31..7c20ee4 100644 --- a/apps/hub/templates/index.html +++ b/apps/hub/templates/index.html @@ -1,5 +1,5 @@ - + diff --git a/apps/hub/templates/settings.html b/apps/hub/templates/settings.html index 4aaeb38..47ddcc1 100644 --- a/apps/hub/templates/settings.html +++ b/apps/hub/templates/settings.html @@ -1,5 +1,5 @@ - + @@ -117,8 +117,24 @@

Settings

Controls the colour scheme for all apps
diff --git a/apps/mantis_explorer/static/me.css b/apps/mantis_explorer/static/me.css index 216118d..91820fa 100644 --- a/apps/mantis_explorer/static/me.css +++ b/apps/mantis_explorer/static/me.css @@ -184,8 +184,14 @@ main { transition: border-color 0.15s, box-shadow 0.15s; color-scheme: dark; } -[data-theme="light"] .me-filter-bar input[type="date"], -[data-theme="light"] .me-filter-bar input[type="text"] { +[data-theme="pisces-light"] .me-filter-bar input[type="date"], +[data-theme="pisces-light"] .me-filter-bar input[type="text"], +[data-theme="gruvbox-light"] .me-filter-bar input[type="date"], +[data-theme="gruvbox-light"] .me-filter-bar input[type="text"], +[data-theme="tokyonight-light"] .me-filter-bar input[type="date"], +[data-theme="tokyonight-light"] .me-filter-bar input[type="text"], +[data-theme="catppuccin-latte"] .me-filter-bar input[type="date"], +[data-theme="catppuccin-latte"] .me-filter-bar input[type="text"] { color-scheme: light; } diff --git a/apps/mantis_explorer/templates/base.html b/apps/mantis_explorer/templates/base.html index d602dfb..e873ac7 100644 --- a/apps/mantis_explorer/templates/base.html +++ b/apps/mantis_explorer/templates/base.html @@ -1,5 +1,5 @@ - + diff --git a/apps/opensearch_web/templates/base.html b/apps/opensearch_web/templates/base.html index 2b8a943..4eea1d0 100644 --- a/apps/opensearch_web/templates/base.html +++ b/apps/opensearch_web/templates/base.html @@ -1,5 +1,5 @@ - + diff --git a/apps/shared/static/theme.js b/apps/shared/static/theme.js index 5d3b6cf..2e4d94b 100644 --- a/apps/shared/static/theme.js +++ b/apps/shared/static/theme.js @@ -5,30 +5,19 @@ * : * (the IIFE at the top applies the saved theme before CSS loads) * - * : + * Settings page: - {% for tr in TIME_RANGES %} - - {% endfor %} - - -
- - - - - -
- - - - -
- - - - -
- - - - -
+
+
+ + +
- +
+ + + +
-
+
+ + +
- - +
+ + +
-
+
+ + +
- +
+ +
-
+
+ + +
+
- +
+ - @@ -201,15 +196,6 @@ ) would raise a ValueError whose str(exc) was passed to the template via data['error']. While Jinja2 autoescape mitigates this, defence in depth requires rejecting invalid input at the boundary. Add safe_date_param() to apps.dashboard_web and apply it in both the tickets and mantis dashboard routes. Invalid values are silently replaced with empty strings (no date filter applied). --- apps/dashboard_web/__init__.py | 19 ++++++++++++++++++ apps/dashboard_web/mantis/__init__.py | 5 +++-- apps/dashboard_web/tickets/__init__.py | 5 +++-- tests/test_dashboard_date_param.py | 27 ++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 tests/test_dashboard_date_param.py diff --git a/apps/dashboard_web/__init__.py b/apps/dashboard_web/__init__.py index e69de29..b37057d 100644 --- a/apps/dashboard_web/__init__.py +++ b/apps/dashboard_web/__init__.py @@ -0,0 +1,19 @@ +"""PISCES Dashboard web application package.""" + +from datetime import date + + +def safe_date_param(value: str) -> str: + """Validate a date query parameter, returning '' for invalid values. + + Prevents user-supplied strings from reaching exception messages + (reflected XSS via error rendering). + """ + v = value.strip() + if not v: + return "" + try: + date.fromisoformat(v) + return v + except ValueError: + return "" diff --git a/apps/dashboard_web/mantis/__init__.py b/apps/dashboard_web/mantis/__init__.py index 9ed6b4b..fc229d8 100644 --- a/apps/dashboard_web/mantis/__init__.py +++ b/apps/dashboard_web/mantis/__init__.py @@ -1,6 +1,7 @@ from flask import Blueprint, render_template, request from apps.dashboard_web import cache as dcache +from apps.dashboard_web import safe_date_param from apps.dashboard_web.mantis.aggregations import ( agg_mantis_attack_types, agg_mantis_infra_count, @@ -13,8 +14,8 @@ @bp.route("/api/dashboard/mantis") def section(): - since = request.args.get("since", "") - until = request.args.get("until", "") + since = safe_date_param(request.args.get("since", "")) + until = safe_date_param(request.args.get("until", "")) cache_key = {"since": since, "until": until} cached = dcache.get("mantis", cache_key) if cached is not None: diff --git a/apps/dashboard_web/tickets/__init__.py b/apps/dashboard_web/tickets/__init__.py index 38f15bd..62b8669 100644 --- a/apps/dashboard_web/tickets/__init__.py +++ b/apps/dashboard_web/tickets/__init__.py @@ -3,6 +3,7 @@ from flask import Blueprint, render_template, request from apps.dashboard_web import cache as dcache +from apps.dashboard_web import safe_date_param from apps.dashboard_web.tickets.aggregations import agg_tickets bp = Blueprint("tickets", __name__, template_folder="templates") @@ -10,8 +11,8 @@ @bp.route("/api/dashboard/tickets") def section() -> str: - since = request.args.get("since", "") - until = request.args.get("until", "") + since = safe_date_param(request.args.get("since", "")) + until = safe_date_param(request.args.get("until", "")) cache_key = {"since": since, "until": until} cached = dcache.get("tickets", cache_key) if cached is not None: diff --git a/tests/test_dashboard_date_param.py b/tests/test_dashboard_date_param.py new file mode 100644 index 0000000..363de3c --- /dev/null +++ b/tests/test_dashboard_date_param.py @@ -0,0 +1,27 @@ +"""Regression tests for dashboard date parameter sanitisation (reflected XSS).""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import pytest + +from apps.dashboard_web import safe_date_param + + +@pytest.mark.parametrize( + "raw, expected", + [ + ("2026-01-15", "2026-01-15"), + (" 2026-01-15 ", "2026-01-15"), + ("", ""), + (" ", ""), + ("", ""), + ("2026-99-99", ""), + ("not-a-date", ""), + ("2026-01-15; DROP TABLE", ""), + ], +) +def test_safe_date_param(raw: str, expected: str) -> None: + assert safe_date_param(raw) == expected From 8cd887abfbc33a6790c82e37f48f33c48c83100b Mon Sep 17 00:00:00 2001 From: liamadale Date: Wed, 29 Apr 2026 23:52:36 -0700 Subject: [PATCH 049/109] fix(dashboard): break CodeQL taint chain by returning parsed date isoformat Return date.fromisoformat(v).isoformat() instead of the original user string. This produces a new string from the parsed date object, which CodeQL can verify is not tainted by user input. --- apps/dashboard_web/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/dashboard_web/__init__.py b/apps/dashboard_web/__init__.py index b37057d..1a14eb6 100644 --- a/apps/dashboard_web/__init__.py +++ b/apps/dashboard_web/__init__.py @@ -13,7 +13,6 @@ def safe_date_param(value: str) -> str: if not v: return "" try: - date.fromisoformat(v) - return v + return date.fromisoformat(v).isoformat() except ValueError: return "" From 30d9789e5a40663273bb208727e5fecda5426809 Mon Sep 17 00:00:00 2001 From: liamadale Date: Thu, 30 Apr 2026 13:02:24 -0700 Subject: [PATCH 050/109] feat(correlator): add Phase 1 incident-context orchestrator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces src/correlator/incident_context.py — a parallel investigation engine that assembles a full IncidentContext for a src/dest IP pair: - Device profiling (private IPs only, via device_profiler) - Kerberos + NTLM auth history between the pair - ATTACK:: notice chain for the source IP - Related Mantis tickets for both IPs - Threat-intel enrichment for public IPs All five tracks run concurrently (ThreadPoolExecutor, max_workers=5). Track failures are isolated — each error is stored in ctx.errors and investigate() always returns a populated IncidentContext even under partial failure. build_timeline() merges kerberos, ntlm, and notice events into a single chronological list. Tests cover timeline ordering, type tagging, routing logic for private/public IP combinations, track-failure isolation, and full-failure resilience (13 cases total). --- src/correlator/__init__.py | 1 + src/correlator/incident_context.py | 191 +++++++++++++++++++++ tests/test_correlator.py | 265 +++++++++++++++++++++++++++++ 3 files changed, 457 insertions(+) create mode 100644 src/correlator/__init__.py create mode 100644 src/correlator/incident_context.py create mode 100644 tests/test_correlator.py diff --git a/src/correlator/__init__.py b/src/correlator/__init__.py new file mode 100644 index 0000000..555157a --- /dev/null +++ b/src/correlator/__init__.py @@ -0,0 +1 @@ +"""Correlation engine — incident context builder.""" diff --git a/src/correlator/incident_context.py b/src/correlator/incident_context.py new file mode 100644 index 0000000..2ebdde7 --- /dev/null +++ b/src/correlator/incident_context.py @@ -0,0 +1,191 @@ +"""Incident context builder — correlates device profiles, auth history, attack chains, +related tickets, and threat intel enrichment into a unified investigation view. +""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from src.profiler.device_profiler import DeviceProfile + + +@dataclass +class IncidentContext: + """All context gathered for an incident investigation.""" + + # Trigger + trigger_type: str # "notice", "ip_pair", "ticket" + trigger: dict + src_ip: str + dest_ip: str + sensor: str + time_range: str + + # Device profiles (None if IP is public or profiling failed) + src_profile: DeviceProfile | None = None + dest_profile: DeviceProfile | None = None + + # Auth history between src ↔ dest + kerberos_history: list[dict] = field(default_factory=list) + ntlm_history: list[dict] = field(default_factory=list) + + # Attack chain (ATTACK::* notices for src_ip) + attack_chain: list[dict] = field(default_factory=list) + + # Related Mantis tickets + src_tickets: list[dict] = field(default_factory=list) + dest_tickets: list[dict] = field(default_factory=list) + + # Threat intel enrichment (None if IP is private) + src_enrichment: dict | None = None + dest_enrichment: dict | None = None + + # Merged chronological timeline + timeline: list[dict] = field(default_factory=list) + + # Errors from individual tracks (track_name → error message) + errors: dict[str, str] = field(default_factory=dict) + + +# --------------------------------------------------------------------------- +# Module-level imports — placed here so unit tests can patch them cleanly +# --------------------------------------------------------------------------- + +from src.enricher.threat_intel import enrich_ip # noqa: E402 +from src.mantis.mantis_search import search as search_tickets # noqa: E402 +from src.profiler.device_profiler import profile_device # noqa: E402 +from src.querier.zeek_modules import MODULES # noqa: E402 +from src.querier.zeek_modules.base import ( # noqa: E402 + INDEX, + is_private, + query_opensearch, + run_query, +) + + +def _query_auth_history( + src_ip: str, + dest_ip: str, + sensor: str, + time_range: str, +) -> tuple[list[dict], list[dict]]: + """Fetch Kerberos + NTLM records between src and dest IPs.""" + sp = { + "src_ip": src_ip, + "dest_ip": dest_ip, + "sensor": sensor, + "time_range": time_range, + "limit": 200, + "no_filters": False, + "public_only": False, + "raise_on_error": False, + } + krb = run_query(MODULES["kerberos"], dict(sp)) + ntlm = run_query(MODULES["ntlm"], dict(sp)) + return krb, ntlm + + +def build_timeline(ctx: IncidentContext) -> list[dict]: + """Merge kerberos, ntlm, and attack chain events into chronological order.""" + events: list[dict] = [] + + for rec in ctx.kerberos_history: + events.append({"type": "kerberos", "timestamp": rec.get("timestamp", ""), **rec}) + for rec in ctx.ntlm_history: + events.append({"type": "ntlm", "timestamp": rec.get("timestamp", ""), **rec}) + for rec in ctx.attack_chain: + events.append({"type": "notice", "timestamp": rec.get("timestamp", ""), **rec}) + + events.sort(key=lambda e: e.get("timestamp", "")) + return events + + +def investigate( + src_ip: str, + dest_ip: str, + sensor: str, + time_range: str = "now-24h", + *, + trigger_type: str = "ip_pair", + trigger: dict | None = None, +) -> IncidentContext: + """Build full incident context for an IP pair. + + Runs 5 parallel tracks: src profile, dest profile, auth history, + attack chain, and context gather (tickets + enrichment). + Each track failure is caught and stored in ctx.errors — the context is + always returned even with partial data. + """ + ctx = IncidentContext( + trigger_type=trigger_type, + trigger=trigger or {"src_ip": src_ip, "dest_ip": dest_ip}, + src_ip=src_ip, + dest_ip=dest_ip, + sensor=sensor, + time_range=time_range, + ) + + def _profile_src() -> None: + if is_private(src_ip): + ctx.src_profile = profile_device(src_ip, time_range=time_range, sensor=sensor) + + def _profile_dest() -> None: + if is_private(dest_ip): + ctx.dest_profile = profile_device(dest_ip, time_range=time_range, sensor=sensor) + + def _auth_history() -> None: + ctx.kerberos_history, ctx.ntlm_history = _query_auth_history( + src_ip, dest_ip, sensor, time_range + ) + + def _attack_chain() -> None: + notice_mod = MODULES["notice"] + must: list = [ + {"range": {"@timestamp": {"gte": time_range, "lte": "now"}}}, + {"terms": {"event.dataset": notice_mod.DATASETS}}, + {"term": {"source.ip": src_ip}}, + {"prefix": {"zeek.notice.note": "ATTACK::"}}, + ] + if sensor != "all": + must.append({"terms": {"host.name": [s.strip() for s in sensor.split(",")]}}) + body = { + "size": 500, + "query": {"bool": {"must": must}}, + "sort": [{"@timestamp": {"order": "asc"}}], + "_source": notice_mod.SOURCE_FIELDS, + } + raw = query_opensearch(body, {"path": f"{INDEX}/_search", "method": "POST"}) + if raw: + hits = raw.get("hits", {}).get("hits", []) + ctx.attack_chain = [notice_mod.parse_hit(h["_source"]) for h in hits] + + def _context_gather() -> None: + ctx.src_tickets = search_tickets(src_ip) + ctx.dest_tickets = search_tickets(dest_ip) + if not is_private(src_ip): + ctx.src_enrichment = enrich_ip(src_ip, offer_fp=False) + if not is_private(dest_ip): + ctx.dest_enrichment = enrich_ip(dest_ip, offer_fp=False) + + tracks = { + "src_profile": _profile_src, + "dest_profile": _profile_dest, + "auth_history": _auth_history, + "attack_chain": _attack_chain, + "context_gather": _context_gather, + } + + with ThreadPoolExecutor(max_workers=5) as pool: + futures = {pool.submit(fn): name for name, fn in tracks.items()} + for future in as_completed(futures): + name = futures[future] + try: + future.result() + except Exception as exc: + ctx.errors[name] = str(exc) + + ctx.timeline = build_timeline(ctx) + return ctx diff --git a/tests/test_correlator.py b/tests/test_correlator.py new file mode 100644 index 0000000..60374a6 --- /dev/null +++ b/tests/test_correlator.py @@ -0,0 +1,265 @@ +"""Tests for the correlation engine — Phase 1: core orchestrator.""" + +from __future__ import annotations + +import os +import sys +from unittest.mock import MagicMock, patch + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from src.correlator.incident_context import IncidentContext, build_timeline, investigate + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +PRIVATE_SRC = "10.200.50.15" +PRIVATE_DEST = "10.100.50.15" +PUBLIC_SRC = "8.8.8.8" +PUBLIC_DEST = "1.1.1.1" +SENSOR = "hedgehog-east-wenatchee" +TIME_RANGE = "now-24h" + +KRB_RECORD = {"timestamp": "2024-06-01T10:00:00.000Z", "src_ip": PRIVATE_SRC} +NTLM_RECORD = {"timestamp": "2024-06-01T09:00:00.000Z", "src_ip": PRIVATE_SRC} +NOTICE_RECORD = { + "timestamp": "2024-06-01T11:00:00.000Z", + "notice_note": "ATTACK::Credential_Access", +} + +MOCK_PROFILE = MagicMock() +MOCK_ENRICHMENT = {"ip": PUBLIC_SRC, "greynoise": {"classification": "malicious"}} + +MOD = "src.correlator.incident_context" + + +def _base_patches( + *, + profile_side_effect=None, + profile_return=None, + krb: list | None = None, + ntlm: list | None = None, + chain_hits: list | None = None, + tickets: list | None = None, + enrichment: dict | None = None, +) -> list: + """Return a stack of patch context managers covering all external calls.""" + if profile_side_effect is not None: + profile_patch = patch(f"{MOD}.profile_device", side_effect=profile_side_effect) + elif profile_return is not None: + profile_patch = patch(f"{MOD}.profile_device", return_value=profile_return) + else: + profile_patch = patch(f"{MOD}.profile_device", return_value=MOCK_PROFILE) + + return [ + profile_patch, + patch(f"{MOD}._query_auth_history", return_value=(krb or [], ntlm or [])), + patch(f"{MOD}.query_opensearch", return_value={"hits": {"hits": chain_hits or []}}), + patch(f"{MOD}.search_tickets", return_value=tickets or []), + patch(f"{MOD}.enrich_ip", return_value=enrichment or {}), + ] + + +# --------------------------------------------------------------------------- +# build_timeline +# --------------------------------------------------------------------------- + + +def test_timeline_chronological_ordering() -> None: + ctx = IncidentContext( + trigger_type="ip_pair", + trigger={}, + src_ip=PRIVATE_SRC, + dest_ip=PRIVATE_DEST, + sensor=SENSOR, + time_range=TIME_RANGE, + kerberos_history=[KRB_RECORD], + ntlm_history=[NTLM_RECORD], + attack_chain=[NOTICE_RECORD], + ) + timeline = build_timeline(ctx) + timestamps = [e["timestamp"] for e in timeline] + assert timestamps == sorted(timestamps) + + +def test_timeline_type_tags() -> None: + ctx = IncidentContext( + trigger_type="ip_pair", + trigger={}, + src_ip=PRIVATE_SRC, + dest_ip=PRIVATE_DEST, + sensor=SENSOR, + time_range=TIME_RANGE, + kerberos_history=[KRB_RECORD], + ntlm_history=[NTLM_RECORD], + attack_chain=[NOTICE_RECORD], + ) + timeline = build_timeline(ctx) + assert {e["type"] for e in timeline} == {"kerberos", "ntlm", "notice"} + + +def test_timeline_empty() -> None: + ctx = IncidentContext( + trigger_type="ip_pair", + trigger={}, + src_ip=PRIVATE_SRC, + dest_ip=PRIVATE_DEST, + sensor=SENSOR, + time_range=TIME_RANGE, + ) + assert build_timeline(ctx) == [] + + +def test_timeline_missing_timestamp_sorts_first() -> None: + """Records without a timestamp key get empty string, which sorts before any ISO date.""" + ctx = IncidentContext( + trigger_type="ip_pair", + trigger={}, + src_ip=PRIVATE_SRC, + dest_ip=PRIVATE_DEST, + sensor=SENSOR, + time_range=TIME_RANGE, + kerberos_history=[{"src_ip": PRIVATE_SRC}], # no timestamp + attack_chain=[NOTICE_RECORD], + ) + timeline = build_timeline(ctx) + assert timeline[0]["type"] == "kerberos" + + +def test_timeline_does_not_mutate_source_records() -> None: + original = {"timestamp": "2024-06-01T10:00:00.000Z", "src_ip": PRIVATE_SRC} + ctx = IncidentContext( + trigger_type="ip_pair", + trigger={}, + src_ip=PRIVATE_SRC, + dest_ip=PRIVATE_DEST, + sensor=SENSOR, + time_range=TIME_RANGE, + kerberos_history=[original], + ) + build_timeline(ctx) + assert "type" not in original + + +# --------------------------------------------------------------------------- +# investigate() — IP routing logic +# --------------------------------------------------------------------------- + + +def test_investigate_both_private() -> None: + patches = _base_patches(krb=[KRB_RECORD], ntlm=[NTLM_RECORD]) + with patches[0], patches[1], patches[2], patches[3], patches[4]: + ctx = investigate(PRIVATE_SRC, PRIVATE_DEST, SENSOR, TIME_RANGE) + + assert ctx.src_profile is not None + assert ctx.dest_profile is not None + assert len(ctx.kerberos_history) == 1 + assert len(ctx.ntlm_history) == 1 + assert ctx.src_enrichment is None + assert ctx.dest_enrichment is None + assert ctx.errors == {} + + +def test_investigate_one_public() -> None: + """Private src → profiled; public dest → enriched, not profiled.""" + patches = _base_patches(enrichment=MOCK_ENRICHMENT) + with patches[0], patches[1], patches[2], patches[3], patches[4]: + ctx = investigate(PRIVATE_SRC, PUBLIC_DEST, SENSOR, TIME_RANGE) + + assert ctx.src_profile is not None + assert ctx.dest_profile is None + assert ctx.dest_enrichment is not None + assert ctx.src_enrichment is None + + +def test_investigate_both_public() -> None: + """Both public → no profiling, both enrichments populated.""" + patches = _base_patches(enrichment=MOCK_ENRICHMENT) + with patches[0], patches[1], patches[2], patches[3], patches[4]: + ctx = investigate(PUBLIC_SRC, PUBLIC_DEST, SENSOR, TIME_RANGE) + + assert ctx.src_profile is None + assert ctx.dest_profile is None + assert ctx.src_enrichment is not None + assert ctx.dest_enrichment is not None + + +def test_investigate_track_failure_isolated() -> None: + """Failure in src_profile track stores error; other tracks succeed.""" + patches = _base_patches(profile_side_effect=ValueError("sensor mismatch")) + with patches[0], patches[1], patches[2], patches[3], patches[4]: + ctx = investigate(PRIVATE_SRC, PRIVATE_DEST, SENSOR, TIME_RANGE) + + failed = ctx.errors.get("src_profile", "") + ctx.errors.get("dest_profile", "") + assert "sensor mismatch" in failed + assert "auth_history" not in ctx.errors + assert "attack_chain" not in ctx.errors + assert "context_gather" not in ctx.errors + + +def test_investigate_no_auth_history() -> None: + patches = _base_patches(krb=[], ntlm=[]) + with patches[0], patches[1], patches[2], patches[3], patches[4]: + ctx = investigate(PRIVATE_SRC, PRIVATE_DEST, SENSOR, TIME_RANGE) + + assert ctx.kerberos_history == [] + assert ctx.ntlm_history == [] + assert "auth_history" not in ctx.errors + + +def test_investigate_returns_context_on_full_failure() -> None: + """investigate() always returns IncidentContext even when all tracks fail.""" + with ( + patch(f"{MOD}.profile_device", side_effect=RuntimeError("ES down")), + patch(f"{MOD}._query_auth_history", side_effect=RuntimeError("ES down")), + patch(f"{MOD}.query_opensearch", side_effect=RuntimeError("ES down")), + patch(f"{MOD}.search_tickets", side_effect=RuntimeError("Mantis down")), + patch(f"{MOD}.enrich_ip", return_value={}), + ): + ctx = investigate(PRIVATE_SRC, PRIVATE_DEST, SENSOR, TIME_RANGE) + + assert isinstance(ctx, IncidentContext) + assert len(ctx.errors) > 0 + assert ctx.timeline == [] + + +# --------------------------------------------------------------------------- +# investigate() — metadata +# --------------------------------------------------------------------------- + + +def test_investigate_default_trigger_and_time_range() -> None: + patches = _base_patches() + with patches[0], patches[1], patches[2], patches[3], patches[4]: + ctx = investigate(PRIVATE_SRC, PRIVATE_DEST, SENSOR) + + assert ctx.trigger == {"src_ip": PRIVATE_SRC, "dest_ip": PRIVATE_DEST} + assert ctx.trigger_type == "ip_pair" + assert ctx.time_range == "now-24h" + + +def test_investigate_custom_trigger() -> None: + custom_trigger = {"ticket_id": 1234} + patches = _base_patches() + with patches[0], patches[1], patches[2], patches[3], patches[4]: + ctx = investigate( + PRIVATE_SRC, + PRIVATE_DEST, + SENSOR, + trigger_type="ticket", + trigger=custom_trigger, + ) + + assert ctx.trigger == custom_trigger + assert ctx.trigger_type == "ticket" + + +def test_investigate_timeline_populated() -> None: + patches = _base_patches(krb=[KRB_RECORD], ntlm=[NTLM_RECORD]) + with patches[0], patches[1], patches[2], patches[3], patches[4]: + ctx = investigate(PRIVATE_SRC, PRIVATE_DEST, SENSOR, TIME_RANGE) + + assert len(ctx.timeline) == 2 + timestamps = [e["timestamp"] for e in ctx.timeline] + assert timestamps == sorted(timestamps) From a7db77bfd91997f2cf055fb459258451e7bb729d Mon Sep 17 00:00:00 2001 From: liamadale Date: Thu, 30 Apr 2026 13:02:31 -0700 Subject: [PATCH 051/109] feat(mcp/opensearch): add investigate tool wrapping Phase 1 orchestrator Exposes the correlator's investigate() function as an MCP tool so AI assistants can trigger a full incident investigation for an IP pair directly from a tool call. The tool trims raw DeviceProfile dicts down to the fields most useful for LLM context (ip, hostname, role, confidence, os_family, software, users, inbound_services) before serialising the result, keeping token usage manageable without losing investigative signal. Accepts src_ip, dest_ip, sensor (default: all), and time_range (default: now-24h). Returns a serialised IncidentContext bundle or an error envelope on failure. --- mcp/opensearch/server.py | 50 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/mcp/opensearch/server.py b/mcp/opensearch/server.py index 46055bf..c0e0f5b 100644 --- a/mcp/opensearch/server.py +++ b/mcp/opensearch/server.py @@ -1340,6 +1340,56 @@ def build_share_urls( return _err(str(exc)) +# --------------------------------------------------------------------------- +# Incident correlator +# --------------------------------------------------------------------------- + + +@mcp.tool() +def investigate( + src_ip: str, + dest_ip: str, + sensor: str = "all", + time_range: str = "now-24h", +) -> str: + """Build full incident context for a source/destination IP pair. + + Runs device profiling, auth history, attack chain, Mantis ticket search, + and threat intel enrichment in parallel. Returns a unified context bundle + for incident investigation. + + Args: + src_ip: Source IP address (the actor / attacker). + dest_ip: Destination IP address (the target / victim). + sensor: Sensor hostname — required for private IP profiling. + time_range: ES date-math range (default: now-24h). + """ + try: + from dataclasses import asdict + + from src.correlator.incident_context import investigate as _investigate + + ctx = _investigate(src_ip, dest_ip, sensor, time_range) + data = asdict(ctx) + # Trim raw profile dicts to a compact summary for LLM context + for key in ("src_profile", "dest_profile"): + p = data.get(key) + if p is not None: + data[key] = { + "ip": p["ip"], + "hostname": p.get("hostname"), + "role": p["role"], + "confidence": p["confidence"], + "os_family": p.get("os_family"), + "software": p.get("software", []), + "users": p.get("users", []), + "inbound_services": p.get("inbound_services", []), + } + return _ok(data) + except Exception as exc: + return _err(str(exc)) + + # --------------------------------------------------------------------------- # Entrypoint # --------------------------------------------------------------------------- From 7cc8439f26c37cc8ffbb0b2507882ef2bdfdcaba Mon Sep 17 00:00:00 2001 From: liamadale Date: Thu, 30 Apr 2026 13:09:32 -0700 Subject: [PATCH 052/109] refactor(correlator): promote auth history and attack chain queries to public API Extract _query_auth_history() and the inline attack-chain query closure from investigate() into two public module-level functions: query_auth_history() and query_attack_chain(). investigate() now delegates to both. Exposes these queries as a stable surface for the web UI to call independently via HTMX partial routes, without going through the full investigate() orchestrator. --- src/correlator/incident_context.py | 48 +++++++++++++++++------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/src/correlator/incident_context.py b/src/correlator/incident_context.py index 2ebdde7..b2cf121 100644 --- a/src/correlator/incident_context.py +++ b/src/correlator/incident_context.py @@ -66,7 +66,7 @@ class IncidentContext: ) -def _query_auth_history( +def query_auth_history( src_ip: str, dest_ip: str, sensor: str, @@ -88,6 +88,30 @@ def _query_auth_history( return krb, ntlm +def query_attack_chain(src_ip: str, sensor: str, time_range: str) -> list[dict]: + """Fetch ATTACK::* notices originating from src_ip in the time window.""" + notice_mod = MODULES["notice"] + must: list = [ + {"range": {"@timestamp": {"gte": time_range, "lte": "now"}}}, + {"terms": {"event.dataset": notice_mod.DATASETS}}, + {"term": {"source.ip": src_ip}}, + {"prefix": {"zeek.notice.note": "ATTACK::"}}, + ] + if sensor != "all": + must.append({"terms": {"host.name": [s.strip() for s in sensor.split(",")]}}) + body = { + "size": 500, + "query": {"bool": {"must": must}}, + "sort": [{"@timestamp": {"order": "asc"}}], + "_source": notice_mod.SOURCE_FIELDS, + } + raw = query_opensearch(body, {"path": f"{INDEX}/_search", "method": "POST"}) + if not raw: + return [] + hits = raw.get("hits", {}).get("hits", []) + return [notice_mod.parse_hit(h["_source"]) for h in hits] + + def build_timeline(ctx: IncidentContext) -> list[dict]: """Merge kerberos, ntlm, and attack chain events into chronological order.""" events: list[dict] = [] @@ -137,30 +161,12 @@ def _profile_dest() -> None: ctx.dest_profile = profile_device(dest_ip, time_range=time_range, sensor=sensor) def _auth_history() -> None: - ctx.kerberos_history, ctx.ntlm_history = _query_auth_history( + ctx.kerberos_history, ctx.ntlm_history = query_auth_history( src_ip, dest_ip, sensor, time_range ) def _attack_chain() -> None: - notice_mod = MODULES["notice"] - must: list = [ - {"range": {"@timestamp": {"gte": time_range, "lte": "now"}}}, - {"terms": {"event.dataset": notice_mod.DATASETS}}, - {"term": {"source.ip": src_ip}}, - {"prefix": {"zeek.notice.note": "ATTACK::"}}, - ] - if sensor != "all": - must.append({"terms": {"host.name": [s.strip() for s in sensor.split(",")]}}) - body = { - "size": 500, - "query": {"bool": {"must": must}}, - "sort": [{"@timestamp": {"order": "asc"}}], - "_source": notice_mod.SOURCE_FIELDS, - } - raw = query_opensearch(body, {"path": f"{INDEX}/_search", "method": "POST"}) - if raw: - hits = raw.get("hits", {}).get("hits", []) - ctx.attack_chain = [notice_mod.parse_hit(h["_source"]) for h in hits] + ctx.attack_chain = query_attack_chain(src_ip, sensor, time_range) def _context_gather() -> None: ctx.src_tickets = search_tickets(src_ip) From d584dba98c255e5ad9c72850d8235199f880e677 Mon Sep 17 00:00:00 2001 From: liamadale Date: Thu, 30 Apr 2026 13:09:43 -0700 Subject: [PATCH 053/109] feat(opensearch_web): add investigate page for IP pair incident context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add /investigate// and five HTMX API partials that together render a one-click incident investigation view: - /api/investigate/profiles — side-by-side device profile cards for src and dest (private IPs only, skips public addresses) - /api/investigate/auth — Kerberos and NTLM auth history tables - /api/investigate/chain — ATTACK::* notice chain for the source IP - /api/investigate/tickets — Mantis tickets for both IPs - /api/investigate/timeline — merged chronological event list built via build_timeline(), fetching auth and chain concurrently Each section loads independently via HTMX with skeleton loaders while in flight, so slow queries don't block the visible page. The timeline endpoint re-issues auth + chain queries in a ThreadPoolExecutor to avoid a sequential round-trip. CSS adds investigate-specific layout (two-column profile/ticket grids, skeleton pulse animation, attack-note badge, responsive breakpoints at 900 px). --- apps/opensearch_web/app.py | 196 ++++++++++++++++++ apps/opensearch_web/static/pisces.css | 142 +++++++++++++ .../opensearch_web/templates/investigate.html | 103 +++++++++ .../templates/partials/investigate_auth.html | 106 ++++++++++ .../templates/partials/investigate_chain.html | 50 +++++ .../partials/investigate_profiles.html | 39 ++++ .../partials/investigate_tickets.html | 74 +++++++ .../partials/investigate_timeline.html | 72 +++++++ 8 files changed, 782 insertions(+) create mode 100644 apps/opensearch_web/templates/investigate.html create mode 100644 apps/opensearch_web/templates/partials/investigate_auth.html create mode 100644 apps/opensearch_web/templates/partials/investigate_chain.html create mode 100644 apps/opensearch_web/templates/partials/investigate_profiles.html create mode 100644 apps/opensearch_web/templates/partials/investigate_tickets.html create mode 100644 apps/opensearch_web/templates/partials/investigate_timeline.html diff --git a/apps/opensearch_web/app.py b/apps/opensearch_web/app.py index 4321fbe..fb4311f 100644 --- a/apps/opensearch_web/app.py +++ b/apps/opensearch_web/app.py @@ -605,4 +605,200 @@ def api_profile(ip: str): profile = profile_device(ip, time_range=time_range, sensor=sensor) return render_template("partials/device_card.html", profile=profile, compact=compact) + # ------------------------------------------------------------------ + # GET /investigate// — one-click incident context page + # ------------------------------------------------------------------ + @app.route("/investigate//") + def investigate_view(src_ip: str, dest_ip: str): + search_params = build_search_params_from_request(request) + return render_template( + "investigate.html", + src_ip=src_ip, + dest_ip=dest_ip, + search_params=search_params, + ) + + # ------------------------------------------------------------------ + # GET /api/investigate/profiles — HTMX: device profiles for src + dest + # ------------------------------------------------------------------ + @app.route("/api/investigate/profiles") + def api_investigate_profiles(): + from src.profiler.device_profiler import profile_device + from src.querier.zeek_modules.base import is_private + + src_ip = request.args.get("src_ip", "") + dest_ip = request.args.get("dest_ip", "") + sensor = request.args.get("sensor", "all") + time_range = request.args.get("time_range", "now-24h") + + src_profile = None + dest_profile = None + src_error = None + dest_error = None + + if is_private(src_ip): + try: + src_profile = profile_device(src_ip, time_range=time_range, sensor=sensor) + except Exception as exc: + src_error = str(exc) + + if is_private(dest_ip): + try: + dest_profile = profile_device(dest_ip, time_range=time_range, sensor=sensor) + except Exception as exc: + dest_error = str(exc) + + return render_template( + "partials/investigate_profiles.html", + src_ip=src_ip, + dest_ip=dest_ip, + src_profile=src_profile, + dest_profile=dest_profile, + src_error=src_error, + dest_error=dest_error, + ) + + # ------------------------------------------------------------------ + # GET /api/investigate/auth — HTMX: Kerberos + NTLM auth history + # ------------------------------------------------------------------ + @app.route("/api/investigate/auth") + def api_investigate_auth(): + from src.correlator.incident_context import query_auth_history + + src_ip = request.args.get("src_ip", "") + dest_ip = request.args.get("dest_ip", "") + sensor = request.args.get("sensor", "all") + time_range = request.args.get("time_range", "now-24h") + + try: + kerberos, ntlm = query_auth_history(src_ip, dest_ip, sensor, time_range) + error = None + except Exception as exc: + kerberos, ntlm, error = [], [], str(exc) + + return render_template( + "partials/investigate_auth.html", + kerberos_history=kerberos, + ntlm_history=ntlm, + src_ip=src_ip, + dest_ip=dest_ip, + error=error, + ) + + # ------------------------------------------------------------------ + # GET /api/investigate/chain — HTMX: ATTACK::* notice chain for src_ip + # ------------------------------------------------------------------ + @app.route("/api/investigate/chain") + def api_investigate_chain(): + from src.correlator.incident_context import query_attack_chain + + src_ip = request.args.get("src_ip", "") + sensor = request.args.get("sensor", "all") + time_range = request.args.get("time_range", "now-24h") + + try: + chain = query_attack_chain(src_ip, sensor, time_range) + error = None + except Exception as exc: + chain, error = [], str(exc) + + return render_template( + "partials/investigate_chain.html", + attack_chain=chain, + src_ip=src_ip, + error=error, + ) + + # ------------------------------------------------------------------ + # GET /api/investigate/tickets — HTMX: Mantis tickets for src + dest + # ------------------------------------------------------------------ + @app.route("/api/investigate/tickets") + def api_investigate_tickets(): + from src.mantis.mantis_search import search as search_tickets + + src_ip = request.args.get("src_ip", "") + dest_ip = request.args.get("dest_ip", "") + + try: + src_tickets = search_tickets(src_ip) if src_ip else [] + dest_tickets = search_tickets(dest_ip) if dest_ip else [] + error = None + except Exception as exc: + src_tickets, dest_tickets, error = [], [], str(exc) + + return render_template( + "partials/investigate_tickets.html", + src_ip=src_ip, + dest_ip=dest_ip, + src_tickets=src_tickets, + dest_tickets=dest_tickets, + error=error, + ) + + # ------------------------------------------------------------------ + # GET /api/investigate/timeline — HTMX: merged chronological event list + # ------------------------------------------------------------------ + @app.route("/api/investigate/timeline") + def api_investigate_timeline(): + from concurrent.futures import ThreadPoolExecutor + from concurrent.futures import as_completed as _as_completed + + from src.correlator.incident_context import ( + IncidentContext, + build_timeline, + query_attack_chain, + query_auth_history, + ) + + src_ip = request.args.get("src_ip", "") + dest_ip = request.args.get("dest_ip", "") + sensor = request.args.get("sensor", "all") + time_range = request.args.get("time_range", "now-24h") + + kerberos: list[dict] = [] + ntlm: list[dict] = [] + chain: list[dict] = [] + errors: dict[str, str] = {} + + def _auth() -> tuple[list[dict], list[dict]]: + return query_auth_history(src_ip, dest_ip, sensor, time_range) + + def _chain() -> list[dict]: + return query_attack_chain(src_ip, sensor, time_range) + + with ThreadPoolExecutor(max_workers=2) as pool: + f_auth = pool.submit(_auth) + f_chain = pool.submit(_chain) + for fut in _as_completed([f_auth, f_chain]): + if fut is f_auth: + try: + kerberos, ntlm = fut.result() + except Exception as exc: + errors["auth"] = str(exc) + else: + try: + chain = fut.result() + except Exception as exc: + errors["chain"] = str(exc) + + ctx = IncidentContext( + trigger_type="ip_pair", + trigger={}, + src_ip=src_ip, + dest_ip=dest_ip, + sensor=sensor, + time_range=time_range, + kerberos_history=kerberos, + ntlm_history=ntlm, + attack_chain=chain, + errors=errors, + ) + timeline = build_timeline(ctx) + + return render_template( + "partials/investigate_timeline.html", + timeline=timeline, + errors=errors, + ) + return app diff --git a/apps/opensearch_web/static/pisces.css b/apps/opensearch_web/static/pisces.css index a9d33d2..994482b 100644 --- a/apps/opensearch_web/static/pisces.css +++ b/apps/opensearch_web/static/pisces.css @@ -1460,3 +1460,145 @@ tr.detail-row td { .device-card-compact .device-card-grid { grid-template-columns: 1fr; } .device-card-compact .device-card-header { flex-direction: column; align-items: flex-start; } .device-card-compact .device-card-identity { font-size: .95rem; } + +/* ── Investigate page ─────────────────────────────────────────────── */ +.investigate-arrow { color: var(--on-surface-dim); margin: 0 .25rem; } + +.investigate-section { + margin-bottom: 2rem; +} + +.investigate-section-header { + display: flex; + align-items: baseline; + gap: .75rem; + margin-bottom: .75rem; +} + +.investigate-section-header h2 { + font-size: 1rem; + font-weight: 600; + margin: 0; + color: var(--on-surface); +} + +.investigate-section-sub { + font-size: .8rem; + color: var(--on-surface-dim); +} + +/* Side-by-side device profile columns */ +.investigate-profiles-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 1rem; +} + +.investigate-profile-col { + min-width: 0; +} + +.investigate-profile-label { + font-size: .8rem; + font-weight: 600; + color: var(--on-surface-dim); + margin-bottom: .5rem; + display: flex; + align-items: center; + gap: .4rem; +} + +/* Auth history protocol sub-headers */ +.investigate-auth-protocol { + margin-bottom: 1.25rem; +} + +.investigate-auth-proto-label { + font-size: .875rem; + font-weight: 600; + margin: 0 0 .5rem; + color: var(--on-surface); + display: flex; + align-items: center; + gap: .5rem; +} + +/* Two-column ticket grid */ +.investigate-tickets-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 1rem; + align-items: start; +} + +.investigate-ticket-col { + min-width: 0; +} + +/* Attack chain notice badge */ +.attack-note-badge { + font-size: .75rem; + font-family: var(--font-mono, monospace); + background: color-mix(in srgb, var(--red) 12%, transparent); + color: var(--red); + padding: 2px 6px; + border-radius: 3px; + white-space: nowrap; +} + +/* Error state */ +.investigate-error { + color: var(--red); + font-size: .85rem; + display: flex; + align-items: center; + gap: .4rem; +} + +.investigate-error-list { + display: flex; + flex-direction: column; + gap: .25rem; + margin-bottom: .75rem; +} + +/* Skeleton loaders */ +.investigate-skeleton { opacity: .5; } + +.skeleton-card { + height: 140px; + background: var(--surface-dim); + border-radius: 6px; + animation: skeleton-pulse 1.4s ease-in-out infinite; +} + +.skeleton-table { + height: 80px; + background: var(--surface-dim); + border-radius: 4px; + animation: skeleton-pulse 1.4s ease-in-out infinite; +} + +.skeleton-list { + height: 60px; + background: var(--surface-dim); + border-radius: 4px; + animation: skeleton-pulse 1.4s ease-in-out infinite; +} + +.investigate-profiles-skeleton { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 1rem; +} + +@keyframes skeleton-pulse { + 0%, 100% { opacity: .5; } + 50% { opacity: .25; } +} + +@media (max-width: 900px) { + .investigate-profiles-grid, + .investigate-tickets-grid { grid-template-columns: 1fr; } + .investigate-profiles-skeleton { grid-template-columns: 1fr; } +} diff --git a/apps/opensearch_web/templates/investigate.html b/apps/opensearch_web/templates/investigate.html new file mode 100644 index 0000000..d7c647e --- /dev/null +++ b/apps/opensearch_web/templates/investigate.html @@ -0,0 +1,103 @@ +{% extends "base.html" %} +{% block title %}PISCES · Investigate — {{ src_ip }} → {{ dest_ip }}{% endblock title %} + +{% block content %} +
+

+ + {{ src_ip }} + → + {{ dest_ip }} +

+ + + {{ fmt_time_window(search_params.get('time_range', 'now-24h')) }} + + + Src Pivot + + + Dst Pivot + +
+ +{# Lazy-load query string shared across all sections #} +{% set qs = "src_ip=" ~ src_ip ~ "&dest_ip=" ~ dest_ip ~ "&sensor=" ~ search_params.get('sensor', 'all') ~ "&time_range=" ~ search_params.get('time_range', 'now-24h') %} + +{# ── Device Profiles ──────────────────────────────────────────────── #} +
+
+

Device Profiles

+
+
+
+
+
+
+
+
+ +{# ── Authentication History ───────────────────────────────────────── #} +
+
+

Authentication History

+ Kerberos + NTLM between {{ src_ip }} ↔ {{ dest_ip }} +
+
+
+
+
+
+
+ +{# ── Attack Chain ─────────────────────────────────────────────────── #} +
+
+

Attack Chain

+ ATTACK::* notices from {{ src_ip }} +
+
+
+
+
+
+
+ +{# ── Related Tickets ──────────────────────────────────────────────── #} +
+
+

Related Tickets

+ Mantis tickets mentioning either IP +
+
+
+
+
+
+
+ +{# ── Timeline ─────────────────────────────────────────────────────── #} +
+
+

Timeline

+ Merged chronological events +
+
+
+
+
+
+
+{% endblock content %} diff --git a/apps/opensearch_web/templates/partials/investigate_auth.html b/apps/opensearch_web/templates/partials/investigate_auth.html new file mode 100644 index 0000000..3059aba --- /dev/null +++ b/apps/opensearch_web/templates/partials/investigate_auth.html @@ -0,0 +1,106 @@ +{# Investigate — authentication history partial. + Receives: kerberos_history, ntlm_history, src_ip, dest_ip, error #} + +{% if error %} +

{{ error }}

+{% else %} + +{# Kerberos #} +
+

+ Kerberos + {{ kerberos_history | length }} record(s) +

+ {% if kerberos_history %} +
+ + + + + + + + + + + + + + {% for rec in kerberos_history %} + + + + + + + + + + {% endfor %} + +
TimeSensorClientServiceTypeAuthError
+ {{ rec.get('timestamp') | fmt_ts(full=True) }} + {{ rec.get('sensor', '—') }}{{ rec.get('client', '—') or '—' }}{{ rec.get('service', '—') or '—' }}{{ rec.get('request_type', '—') or '—' }} + {% if rec.get('success') == True %} + ✓ OK + {% elif rec.get('success') == False %} + ✗ Fail + {% else %} + — + {% endif %} + {{ rec.get('error_msg', '—') or '—' }}
+
+ {% else %} +

No Kerberos records between {{ src_ip }} ↔ {{ dest_ip }} in this window.

+ {% endif %} +
+ +{# NTLM #} +
+

+ NTLM + {{ ntlm_history | length }} record(s) +

+ {% if ntlm_history %} +
+ + + + + + + + + + + + + {% for rec in ntlm_history %} + + + + + + + + + {% endfor %} + +
TimeSensorUsernameClient HostDomainAuth
+ {{ rec.get('timestamp') | fmt_ts(full=True) }} + {{ rec.get('sensor', '—') }}{{ rec.get('username', '—') or '—' }}{{ rec.get('client_hostname', '—') or '—' }}{{ rec.get('domain', '—') or '—' }} + {% if rec.get('success') == True %} + ✓ OK + {% elif rec.get('success') == False %} + ✗ Fail + {% else %} + — + {% endif %} +
+
+ {% else %} +

No NTLM records between {{ src_ip }} ↔ {{ dest_ip }} in this window.

+ {% endif %} +
+ +{% endif %} diff --git a/apps/opensearch_web/templates/partials/investigate_chain.html b/apps/opensearch_web/templates/partials/investigate_chain.html new file mode 100644 index 0000000..97cef27 --- /dev/null +++ b/apps/opensearch_web/templates/partials/investigate_chain.html @@ -0,0 +1,50 @@ +{# Investigate — attack chain partial. + Receives: attack_chain (list[dict]), src_ip, error #} + +{% if error %} +

{{ error }}

+{% elif not attack_chain %} +

No ATTACK::* notices detected for {{ src_ip }} in this window.

+{% else %} +
+ + + + + + + + + + + + + {% for rec in attack_chain %} + + + + + + + + + {% endfor %} + +
#TimeSensorDst IPNoticeMessage
{{ loop.index }} + {{ rec.get('timestamp') | fmt_ts(full=True) }} + {{ rec.get('sensor', '—') }} + {% if rec.get('dest_ip') %} + + {{ rec.dest_ip }} + + {% else %}—{% endif %} + + {{ rec.get('notice_note', '—') }} + + {{ (rec.get('notice_msg', '') or '')[:100] or '—' }} +
+
+

+ {{ attack_chain | length }} notice{{ 's' if attack_chain | length != 1 else '' }} +

+{% endif %} diff --git a/apps/opensearch_web/templates/partials/investigate_profiles.html b/apps/opensearch_web/templates/partials/investigate_profiles.html new file mode 100644 index 0000000..3987449 --- /dev/null +++ b/apps/opensearch_web/templates/partials/investigate_profiles.html @@ -0,0 +1,39 @@ +{# Investigate — device profiles partial. + Receives: src_ip, dest_ip, src_profile, dest_profile, src_error, dest_error #} +
+ + {# Source IP #} +
+
+ Source + {{ src_ip }} +
+ {% if src_error %} +

{{ src_error }}

+ {% elif src_profile %} + {% with profile=src_profile, compact=False %} + {% include "partials/device_card.html" %} + {% endwith %} + {% else %} +

Public IP — device profiling not available.

+ {% endif %} +
+ + {# Destination IP #} +
+
+ Destination + {{ dest_ip }} +
+ {% if dest_error %} +

{{ dest_error }}

+ {% elif dest_profile %} + {% with profile=dest_profile, compact=False %} + {% include "partials/device_card.html" %} + {% endwith %} + {% else %} +

Public IP — device profiling not available.

+ {% endif %} +
+ +
diff --git a/apps/opensearch_web/templates/partials/investigate_tickets.html b/apps/opensearch_web/templates/partials/investigate_tickets.html new file mode 100644 index 0000000..15220bb --- /dev/null +++ b/apps/opensearch_web/templates/partials/investigate_tickets.html @@ -0,0 +1,74 @@ +{# Investigate — related Mantis tickets partial. + Receives: src_ip, dest_ip, src_tickets, dest_tickets, error #} + +{% if error %} +

Mantis unavailable: {{ error }}

+{% else %} +
+ + {# Source IP tickets #} +
+

+ {{ src_ip }} + {{ src_tickets | length }} ticket(s) +

+ {% if src_tickets %} +
+ {% for t in src_tickets %} +
+
+ #{{ t.id }} + {{ t.summary }} + {{ t.status or '—' }} + {% if t.severity %}{{ t.severity }}{% endif %} +
+
+ {% if t.handler %} {{ t.handler.name }}{% endif %} + {% if t.project %}{{ t.project }}{% endif %} + {{ t.last_updated or t.updated_at or '—' }} +
+ {% for n in t.notes %}{% if n.is_admin_note %} +
★ {{ n.reporter.name }}: {{ n.text[:200] }}{% if n.text|length > 200 %}…{% endif %}
+ {% endif %}{% endfor %} +
+ {% endfor %} +
+ {% else %} +

No tickets found for {{ src_ip }}.

+ {% endif %} +
+ + {# Destination IP tickets #} +
+

+ {{ dest_ip }} + {{ dest_tickets | length }} ticket(s) +

+ {% if dest_tickets %} +
+ {% for t in dest_tickets %} +
+
+ #{{ t.id }} + {{ t.summary }} + {{ t.status or '—' }} + {% if t.severity %}{{ t.severity }}{% endif %} +
+
+ {% if t.handler %} {{ t.handler.name }}{% endif %} + {% if t.project %}{{ t.project }}{% endif %} + {{ t.last_updated or t.updated_at or '—' }} +
+ {% for n in t.notes %}{% if n.is_admin_note %} +
★ {{ n.reporter.name }}: {{ n.text[:200] }}{% if n.text|length > 200 %}…{% endif %}
+ {% endif %}{% endfor %} +
+ {% endfor %} +
+ {% else %} +

No tickets found for {{ dest_ip }}.

+ {% endif %} +
+ +
+{% endif %} diff --git a/apps/opensearch_web/templates/partials/investigate_timeline.html b/apps/opensearch_web/templates/partials/investigate_timeline.html new file mode 100644 index 0000000..038dc8b --- /dev/null +++ b/apps/opensearch_web/templates/partials/investigate_timeline.html @@ -0,0 +1,72 @@ +{# Investigate — merged chronological timeline partial. + Receives: timeline (list[dict]), errors (dict) #} + +{% if errors %} +
+ {% for track, msg in errors.items() %} +

{{ track }}: {{ msg }}

+ {% endfor %} +
+{% endif %} + +{% if not timeline %} +

No timeline events found in this window.

+{% else %} +
+ + + + + + + + + + + + + + {% for evt in timeline %} + + + + + + + + + + {% endfor %} + +
#TimeTypeSensorSrc IPDst IPDetail
{{ loop.index }} + {{ evt.get('timestamp') | fmt_ts(full=True) }} + + {% set etype = evt.get('type', '') %} + {% if etype == 'kerberos' %} + Kerberos + {% elif etype == 'ntlm' %} + NTLM + {% elif etype == 'notice' %} + Notice + {% else %} + {{ etype or '—' }} + {% endif %} + {{ evt.get('sensor', '—') }}{{ evt.get('src_ip', '—') }}{{ evt.get('dest_ip', '—') }} + {% set etype = evt.get('type', '') %} + {% if etype == 'kerberos' %} + {{ evt.get('client', '') or '' }}{% if evt.get('client') and evt.get('request_type') %} · {% endif %}{{ evt.get('request_type', '') or '' }} + {% if evt.get('success') == True %}✓{% elif evt.get('success') == False %}✗{% endif %} + {% elif etype == 'ntlm' %} + {{ evt.get('domain', '') or '' }}{% if evt.get('domain') and evt.get('username') %}\{% endif %}{{ evt.get('username', '') or '' }} + {% if evt.get('success') == True %}✓{% elif evt.get('success') == False %}✗{% endif %} + {% elif etype == 'notice' %} + {{ evt.get('notice_note', '') or '' }} + {% else %} + — + {% endif %} +
+
+

+ {{ timeline | length }} event{{ 's' if timeline | length != 1 else '' }} +

+{% endif %} From fae7dab309742a6e88347afc27d166f7cb05ab06 Mon Sep 17 00:00:00 2001 From: liamadale Date: Thu, 30 Apr 2026 13:22:48 -0700 Subject: [PATCH 054/109] fix(opensearch_web): remove doubled script_name prefix from investigate HTMX paths {{ script_name }} was being prepended to every hx-get URL in investigate.html, causing 404s when the app is served through the hub dispatcher (which already sets the SCRIPT_NAME WSGI variable). Replaced with root-relative paths so the browser resolves them correctly under any mount point. --- apps/opensearch_web/templates/investigate.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/opensearch_web/templates/investigate.html b/apps/opensearch_web/templates/investigate.html index d7c647e..690b494 100644 --- a/apps/opensearch_web/templates/investigate.html +++ b/apps/opensearch_web/templates/investigate.html @@ -31,7 +31,7 @@

Device Profiles

-
@@ -47,7 +47,7 @@

Device Profiles

Authentication History

Kerberos + NTLM between {{ src_ip }} ↔ {{ dest_ip }}
-
@@ -62,7 +62,7 @@

Authentication History

Attack Chain

ATTACK::* notices from {{ src_ip }}
-
@@ -77,7 +77,7 @@

Attack Chain

Related Tickets

Mantis tickets mentioning either IP
-
@@ -92,7 +92,7 @@

Related Tickets

Timeline

Merged chronological events
-
From 69b27fefd675058512be2421086a44f380b872c4 Mon Sep 17 00:00:00 2001 From: liamadale Date: Thu, 30 Apr 2026 13:22:53 -0700 Subject: [PATCH 055/109] feat(opensearch_web): add Investigate entry-points to IP pivot and notice records - ip_pivot.html: adds an "Investigate..." button to the page header that prompts for a destination IP then navigates to the investigate page - record_detail.html: adds an "Investigate" link on expanded notice records that have both src_ip and dest_ip, linking directly to the pair view Both entry-points wire into the existing /investigate// route added in the Phase 4 UI work. --- apps/opensearch_web/templates/ip_pivot.html | 12 ++++++++++++ .../templates/partials/record_detail.html | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/apps/opensearch_web/templates/ip_pivot.html b/apps/opensearch_web/templates/ip_pivot.html index 75be338..c85e423 100644 --- a/apps/opensearch_web/templates/ip_pivot.html +++ b/apps/opensearch_web/templates/ip_pivot.html @@ -25,6 +25,9 @@

IP Pivot: {{ ip }}

Enrich {% endif %} +
@@ -96,4 +99,13 @@

IP Pivot: {{ ip }}

{% endif %}
{% endfor %} + + {% endblock content %} diff --git a/apps/opensearch_web/templates/partials/record_detail.html b/apps/opensearch_web/templates/partials/record_detail.html index 09a0773..135b105 100644 --- a/apps/opensearch_web/templates/partials/record_detail.html +++ b/apps/opensearch_web/templates/partials/record_detail.html @@ -30,6 +30,12 @@ hx-swap="innerHTML"> Create FP Filter + {% if log_type == "notice" and record.get('src_ip') and record.get('dest_ip') %} + + Investigate + + {% endif %}
{% endif %}
From 28b6931fc641b4a00882af09b9c046fddeb91249 Mon Sep 17 00:00:00 2001 From: liamadale Date: Thu, 30 Apr 2026 13:29:39 -0700 Subject: [PATCH 056/109] test(correlator): expand test suite to 29 tests and add fixtures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add integration tests, sub-function tests (query_auth_history, query_attack_chain), and MCP tool tests covering JSON serialization, profile trimming, null profile handling, and error propagation. Fix a pre-existing bug in _base_patches: the patch target was _query_auth_history (non-existent private name) instead of the public query_auth_history — causing all full-failure and integration tests to patch the wrong symbol. Add tests/fixtures/correlator/ with kerberos_hit.json, ntlm_hit.json, and notice_attack_chain_hit.json to ground fixture-based assertions in realistic ES response shapes. --- tests/fixtures/correlator/kerberos_hit.json | 18 + .../correlator/notice_attack_chain_hit.json | 16 + tests/fixtures/correlator/ntlm_hit.json | 17 + tests/test_correlator.py | 314 +++++++++++++++++- 4 files changed, 361 insertions(+), 4 deletions(-) create mode 100644 tests/fixtures/correlator/kerberos_hit.json create mode 100644 tests/fixtures/correlator/notice_attack_chain_hit.json create mode 100644 tests/fixtures/correlator/ntlm_hit.json diff --git a/tests/fixtures/correlator/kerberos_hit.json b/tests/fixtures/correlator/kerberos_hit.json new file mode 100644 index 0000000..4f25f84 --- /dev/null +++ b/tests/fixtures/correlator/kerberos_hit.json @@ -0,0 +1,18 @@ +{ + "_source": { + "@timestamp": "2024-06-01T10:00:00.000Z", + "source": {"ip": "10.200.50.15", "port": 49200}, + "destination": {"ip": "10.100.50.15", "port": 88}, + "zeek": { + "kerberos": { + "request_type": "TGS", + "service": "host/dc01.cyberrangepoulsbo.com", + "success": true, + "client": "ALICE$", + "cipher": "aes256-cts-hmac-sha1-96" + } + }, + "host": {"name": "hedgehog-east-wenatchee"}, + "event": {"dataset": "zeek.kerberos"} + } +} diff --git a/tests/fixtures/correlator/notice_attack_chain_hit.json b/tests/fixtures/correlator/notice_attack_chain_hit.json new file mode 100644 index 0000000..3da219d --- /dev/null +++ b/tests/fixtures/correlator/notice_attack_chain_hit.json @@ -0,0 +1,16 @@ +{ + "_source": { + "@timestamp": "2024-06-01T11:00:00.000Z", + "source": {"ip": "10.200.50.15"}, + "destination": {"ip": "10.100.50.15"}, + "zeek": { + "notice": { + "note": "ATTACK::Credential_Access", + "msg": "DCSync detected from 10.200.50.15 to 10.100.50.15", + "actions": ["Notice::ACTION_LOG"] + } + }, + "host": {"name": "hedgehog-east-wenatchee"}, + "event": {"dataset": "zeek.notice"} + } +} diff --git a/tests/fixtures/correlator/ntlm_hit.json b/tests/fixtures/correlator/ntlm_hit.json new file mode 100644 index 0000000..e32f9da --- /dev/null +++ b/tests/fixtures/correlator/ntlm_hit.json @@ -0,0 +1,17 @@ +{ + "_source": { + "@timestamp": "2024-06-01T09:00:00.000Z", + "source": {"ip": "10.200.50.15", "port": 49199}, + "destination": {"ip": "10.100.50.15", "port": 445}, + "zeek": { + "ntlm": { + "username": "ALICE", + "domainname": "CYBERRANGE", + "hostname": "WORKSTATION01", + "status": "SUCCESS" + } + }, + "host": {"name": "hedgehog-east-wenatchee"}, + "event": {"dataset": "zeek.ntlm"} + } +} diff --git a/tests/test_correlator.py b/tests/test_correlator.py index 60374a6..42e8673 100644 --- a/tests/test_correlator.py +++ b/tests/test_correlator.py @@ -1,14 +1,39 @@ -"""Tests for the correlation engine — Phase 1: core orchestrator.""" +"""Tests for the correlation engine — Phases 1 & 5: core orchestrator + integration.""" from __future__ import annotations +import json import os import sys +from pathlib import Path from unittest.mock import MagicMock, patch +import pytest + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from src.correlator.incident_context import IncidentContext, build_timeline, investigate +from src.correlator.incident_context import ( + IncidentContext, + build_timeline, + investigate, + query_attack_chain, + query_auth_history, +) + +# --------------------------------------------------------------------------- +# Fixtures (file-based) +# --------------------------------------------------------------------------- + +FIXTURE_DIR = Path(__file__).parent / "fixtures" / "correlator" + + +def _load_fixture(name: str) -> dict: + return json.loads((FIXTURE_DIR / name).read_text()) + + +KRB_HIT = _load_fixture("kerberos_hit.json") +NTLM_HIT = _load_fixture("ntlm_hit.json") +NOTICE_HIT = _load_fixture("notice_attack_chain_hit.json") # --------------------------------------------------------------------------- # Test data @@ -54,7 +79,7 @@ def _base_patches( return [ profile_patch, - patch(f"{MOD}._query_auth_history", return_value=(krb or [], ntlm or [])), + patch(f"{MOD}.query_auth_history", return_value=(krb or [], ntlm or [])), patch(f"{MOD}.query_opensearch", return_value={"hits": {"hits": chain_hits or []}}), patch(f"{MOD}.search_tickets", return_value=tickets or []), patch(f"{MOD}.enrich_ip", return_value=enrichment or {}), @@ -212,7 +237,7 @@ def test_investigate_returns_context_on_full_failure() -> None: """investigate() always returns IncidentContext even when all tracks fail.""" with ( patch(f"{MOD}.profile_device", side_effect=RuntimeError("ES down")), - patch(f"{MOD}._query_auth_history", side_effect=RuntimeError("ES down")), + patch(f"{MOD}.query_auth_history", side_effect=RuntimeError("ES down")), patch(f"{MOD}.query_opensearch", side_effect=RuntimeError("ES down")), patch(f"{MOD}.search_tickets", side_effect=RuntimeError("Mantis down")), patch(f"{MOD}.enrich_ip", return_value={}), @@ -263,3 +288,284 @@ def test_investigate_timeline_populated() -> None: assert len(ctx.timeline) == 2 timestamps = [e["timestamp"] for e in ctx.timeline] assert timestamps == sorted(timestamps) + + +# --------------------------------------------------------------------------- +# query_auth_history — search_params contract +# --------------------------------------------------------------------------- + + +def test_query_auth_history_search_params() -> None: + """run_query is called with correct search_params for both kerberos and ntlm.""" + with patch(f"{MOD}.run_query", return_value=[]) as mock_rq: + query_auth_history(PRIVATE_SRC, PRIVATE_DEST, SENSOR, TIME_RANGE) + + assert mock_rq.call_count == 2 + _, first_call_kwargs = mock_rq.call_args_list[0] + first_call_pos = mock_rq.call_args_list[0][0] + # Extract the search_params dict (second positional arg) + sp = first_call_pos[1] + assert sp["src_ip"] == PRIVATE_SRC + assert sp["dest_ip"] == PRIVATE_DEST + assert sp["sensor"] == SENSOR + assert sp["time_range"] == TIME_RANGE + assert sp["limit"] == 200 + assert sp["raise_on_error"] is False + + +def test_query_auth_history_both_protocols_queried() -> None: + """Both kerberos and ntlm modules are passed to run_query.""" + from src.querier.zeek_modules import MODULES + + with patch(f"{MOD}.run_query", return_value=[]) as mock_rq: + query_auth_history(PRIVATE_SRC, PRIVATE_DEST, SENSOR, TIME_RANGE) + + called_modules = [c[0][0] for c in mock_rq.call_args_list] + assert MODULES["kerberos"] in called_modules + assert MODULES["ntlm"] in called_modules + + +def test_query_auth_history_returns_tuple() -> None: + with patch(f"{MOD}.run_query", side_effect=[[KRB_RECORD], [NTLM_RECORD]]): + krb, ntlm = query_auth_history(PRIVATE_SRC, PRIVATE_DEST, SENSOR, TIME_RANGE) + + assert krb == [KRB_RECORD] + assert ntlm == [NTLM_RECORD] + + +# --------------------------------------------------------------------------- +# query_attack_chain — sensor filter and ES response parsing +# --------------------------------------------------------------------------- + + +def test_query_attack_chain_includes_sensor_filter() -> None: + """When sensor is not 'all', a host.name terms filter is added to the query.""" + with patch(f"{MOD}.query_opensearch", return_value={"hits": {"hits": []}}) as mock_qs: + query_attack_chain(PRIVATE_SRC, SENSOR, TIME_RANGE) + + body = mock_qs.call_args[0][0] + must_clauses = body["query"]["bool"]["must"] + host_filter = next( + (c for c in must_clauses if "terms" in c and "host.name" in c["terms"]), None + ) + assert host_filter is not None + assert SENSOR in host_filter["terms"]["host.name"] + + +def test_query_attack_chain_omits_sensor_filter_for_all() -> None: + """When sensor='all', no host.name filter is added.""" + with patch(f"{MOD}.query_opensearch", return_value={"hits": {"hits": []}}) as mock_qs: + query_attack_chain(PRIVATE_SRC, "all", TIME_RANGE) + + body = mock_qs.call_args[0][0] + must_clauses = body["query"]["bool"]["must"] + host_filter = next( + (c for c in must_clauses if "terms" in c and "host.name" in c.get("terms", {})), None + ) + assert host_filter is None + + +def test_query_attack_chain_empty_response() -> None: + with patch(f"{MOD}.query_opensearch", return_value=None): + result = query_attack_chain(PRIVATE_SRC, SENSOR, TIME_RANGE) + assert result == [] + + +def test_query_attack_chain_parses_fixture_hit() -> None: + """parse_hit is applied to each ES hit — spot-check against the fixture.""" + raw_hit = NOTICE_HIT + with patch(f"{MOD}.query_opensearch", return_value={"hits": {"hits": [raw_hit]}}): + result = query_attack_chain(PRIVATE_SRC, SENSOR, TIME_RANGE) + + assert len(result) == 1 + parsed = result[0] + assert parsed["timestamp"] == "2024-06-01T11:00:00.000Z" + assert parsed["notice_note"] == "ATTACK::Credential_Access" + assert parsed["src_ip"] == PRIVATE_SRC + assert parsed["dest_ip"] == PRIVATE_DEST + + +def test_query_attack_chain_attack_prefix_filter() -> None: + """Query includes a prefix filter for ATTACK:: notices.""" + with patch(f"{MOD}.query_opensearch", return_value={"hits": {"hits": []}}) as mock_qs: + query_attack_chain(PRIVATE_SRC, SENSOR, TIME_RANGE) + + body = mock_qs.call_args[0][0] + must_clauses = body["query"]["bool"]["must"] + prefix_filter = next( + (c for c in must_clauses if "prefix" in c and "zeek.notice.note" in c["prefix"]), None + ) + assert prefix_filter is not None + assert prefix_filter["prefix"]["zeek.notice.note"] == "ATTACK::" + + +# --------------------------------------------------------------------------- +# Integration: investigate() with fixture-based ES responses +# --------------------------------------------------------------------------- + + +def test_investigate_parses_attack_chain_from_fixture() -> None: + """End-to-end: attack chain hits from fixture produce correctly typed timeline events.""" + patches = _base_patches( + chain_hits=[NOTICE_HIT], + ) + with patches[0], patches[1], patches[2], patches[3], patches[4]: + ctx = investigate(PRIVATE_SRC, PRIVATE_DEST, SENSOR, TIME_RANGE) + + notice_events = [e for e in ctx.timeline if e["type"] == "notice"] + assert len(notice_events) == 1 + assert notice_events[0]["notice_note"] == "ATTACK::Credential_Access" + + +def test_investigate_tickets_populated() -> None: + """Mantis tickets are stored per-IP in src_tickets / dest_tickets.""" + mock_ticket = {"id": 42, "summary": f"Suspicious activity from {PRIVATE_SRC}"} + patches = _base_patches(tickets=[mock_ticket]) + with patches[0], patches[1], patches[2], patches[3], patches[4]: + ctx = investigate(PRIVATE_SRC, PRIVATE_DEST, SENSOR, TIME_RANGE) + + assert ctx.src_tickets == [mock_ticket] + assert ctx.dest_tickets == [mock_ticket] + + +def test_investigate_context_gather_skips_enrichment_for_private() -> None: + """enrich_ip is never called when both IPs are private.""" + patches = _base_patches() + with patches[0], patches[1], patches[2], patches[3], patches[4] as mock_enrich: + investigate(PRIVATE_SRC, PRIVATE_DEST, SENSOR, TIME_RANGE) + + mock_enrich.assert_not_called() + + +# --------------------------------------------------------------------------- +# MCP tool — JSON serialization and error handling +# --------------------------------------------------------------------------- + +# Load the MCP server module via its file path (it is not an installed package). +_MCP_SERVER_PATH = Path(__file__).parent.parent / "mcp" / "opensearch" / "server.py" + +try: + import importlib.util as _ilu + + _spec = _ilu.spec_from_file_location("_mcp_server", _MCP_SERVER_PATH) + mcp_server = _ilu.module_from_spec(_spec) # type: ignore[arg-type] + _spec.loader.exec_module(mcp_server) # type: ignore[union-attr] + _MCP_AVAILABLE = True +except Exception: + mcp_server = None # type: ignore[assignment] + _MCP_AVAILABLE = False + +_skip_no_mcp = pytest.mark.skipif(not _MCP_AVAILABLE, reason="MCP server not importable") + + +def _make_incident_context(**overrides) -> IncidentContext: + """Return a minimal IncidentContext suitable for asdict() serialization.""" + defaults = dict( + trigger_type="ip_pair", + trigger={"src_ip": PRIVATE_SRC, "dest_ip": PRIVATE_DEST}, + src_ip=PRIVATE_SRC, + dest_ip=PRIVATE_DEST, + sensor=SENSOR, + time_range=TIME_RANGE, + kerberos_history=[KRB_RECORD], + ntlm_history=[NTLM_RECORD], + attack_chain=[NOTICE_RECORD], + src_tickets=[], + dest_tickets=[], + src_enrichment=None, + dest_enrichment=None, + timeline=[], + errors={}, + ) + defaults.update(overrides) + return IncidentContext(**defaults) + + +@_skip_no_mcp +def test_mcp_investigate_returns_ok_json() -> None: + """MCP investigate tool returns JSON with status='ok' and all top-level keys.""" + ctx = _make_incident_context() + with patch("src.correlator.incident_context.investigate", return_value=ctx): + result_str = mcp_server.investigate(PRIVATE_SRC, PRIVATE_DEST, SENSOR, TIME_RANGE) + + result = json.loads(result_str) + assert result["status"] == "ok" + data = result["data"] + for key in ( + "src_ip", + "dest_ip", + "sensor", + "time_range", + "trigger_type", + "kerberos_history", + "ntlm_history", + "attack_chain", + "errors", + ): + assert key in data, f"Missing key: {key}" + + +@_skip_no_mcp +def test_mcp_investigate_profile_trimmed() -> None: + """Profile dicts are trimmed to compact summaries — full DeviceProfile keys stripped.""" + from src.profiler.device_profiler import DeviceProfile + + profile = DeviceProfile( + ip=PRIVATE_SRC, + sensor=SENSOR, + time_range=TIME_RANGE, + role="workstation", + confidence=0.85, + os_family="Windows", + hostname="workstation01", + ) + ctx = _make_incident_context(src_profile=profile) + with patch("src.correlator.incident_context.investigate", return_value=ctx): + result_str = mcp_server.investigate(PRIVATE_SRC, PRIVATE_DEST, SENSOR, TIME_RANGE) + + result = json.loads(result_str) + assert result["status"] == "ok" + trimmed = result["data"]["src_profile"] + # Only summary keys present — full DeviceProfile fields like dest_port_distribution stripped + expected_keys = { + "ip", + "hostname", + "role", + "confidence", + "os_family", + "software", + "users", + "inbound_services", + } + assert set(trimmed.keys()) == expected_keys + assert trimmed["role"] == "workstation" + assert trimmed["ip"] == PRIVATE_SRC + assert "dest_port_distribution" not in trimmed + assert "bytes_sent" not in trimmed + + +@_skip_no_mcp +def test_mcp_investigate_error_json() -> None: + """MCP investigate tool returns JSON with status='error' when backend raises.""" + with patch( + "src.correlator.incident_context.investigate", + side_effect=RuntimeError("OpenSearch unreachable"), + ): + result_str = mcp_server.investigate(PRIVATE_SRC, PRIVATE_DEST, SENSOR, TIME_RANGE) + + result = json.loads(result_str) + assert result["status"] == "error" + assert "OpenSearch unreachable" in result["message"] + + +@_skip_no_mcp +def test_mcp_investigate_null_profiles_preserved() -> None: + """None profiles (public IPs) are preserved as null in JSON — not trimmed.""" + ctx = _make_incident_context(src_profile=None, dest_profile=None) + with patch("src.correlator.incident_context.investigate", return_value=ctx): + result_str = mcp_server.investigate(PUBLIC_SRC, PUBLIC_DEST, SENSOR, TIME_RANGE) + + result = json.loads(result_str) + assert result["status"] == "ok" + assert result["data"]["src_profile"] is None + assert result["data"]["dest_profile"] is None From b5978d0b9b99ac73f15d489f8189d6d7733c5b70 Mon Sep 17 00:00:00 2001 From: liamadale Date: Thu, 30 Apr 2026 13:43:28 -0700 Subject: [PATCH 057/109] fix(correlator): parallel profiles, timeline key override, ticket dedup - Parallelize profile_device calls in /api/investigate/profiles route using ThreadPoolExecutor(max_workers=2) to halve wall-clock time - Fix build_timeline dict construction so explicit type/timestamp keys always win over source record keys ({**rec, "type": ...} not {"type": ..., **rec}) - Deduplicate Mantis tickets in /api/investigate/tickets route so tickets matching both IPs only appear once - Add test_timeline_explicit_keys_override_rec regression test --- apps/opensearch_web/app.py | 30 ++++++++++++++++++++---------- src/correlator/incident_context.py | 6 +++--- tests/test_correlator.py | 20 ++++++++++++++++++++ 3 files changed, 43 insertions(+), 13 deletions(-) diff --git a/apps/opensearch_web/app.py b/apps/opensearch_web/app.py index fb4311f..23a3af3 100644 --- a/apps/opensearch_web/app.py +++ b/apps/opensearch_web/app.py @@ -623,6 +623,8 @@ def investigate_view(src_ip: str, dest_ip: str): # ------------------------------------------------------------------ @app.route("/api/investigate/profiles") def api_investigate_profiles(): + from concurrent.futures import ThreadPoolExecutor + from src.profiler.device_profiler import profile_device from src.querier.zeek_modules.base import is_private @@ -636,17 +638,22 @@ def api_investigate_profiles(): src_error = None dest_error = None - if is_private(src_ip): - try: - src_profile = profile_device(src_ip, time_range=time_range, sensor=sensor) - except Exception as exc: - src_error = str(exc) + def _profile(ip: str): # type: ignore[no-untyped-def] + return profile_device(ip, time_range=time_range, sensor=sensor) - if is_private(dest_ip): - try: - dest_profile = profile_device(dest_ip, time_range=time_range, sensor=sensor) - except Exception as exc: - dest_error = str(exc) + with ThreadPoolExecutor(max_workers=2) as pool: + f_src = pool.submit(_profile, src_ip) if is_private(src_ip) else None + f_dest = pool.submit(_profile, dest_ip) if is_private(dest_ip) else None + if f_src is not None: + try: + src_profile = f_src.result() + except Exception as exc: + src_error = str(exc) + if f_dest is not None: + try: + dest_profile = f_dest.result() + except Exception as exc: + dest_error = str(exc) return render_template( "partials/investigate_profiles.html", @@ -722,6 +729,9 @@ def api_investigate_tickets(): try: src_tickets = search_tickets(src_ip) if src_ip else [] dest_tickets = search_tickets(dest_ip) if dest_ip else [] + # Deduplicate: drop dest tickets already shown under src + src_ids = {t.get("id") for t in src_tickets} + dest_tickets = [t for t in dest_tickets if t.get("id") not in src_ids] error = None except Exception as exc: src_tickets, dest_tickets, error = [], [], str(exc) diff --git a/src/correlator/incident_context.py b/src/correlator/incident_context.py index b2cf121..ea94f26 100644 --- a/src/correlator/incident_context.py +++ b/src/correlator/incident_context.py @@ -117,11 +117,11 @@ def build_timeline(ctx: IncidentContext) -> list[dict]: events: list[dict] = [] for rec in ctx.kerberos_history: - events.append({"type": "kerberos", "timestamp": rec.get("timestamp", ""), **rec}) + events.append({**rec, "type": "kerberos", "timestamp": rec.get("timestamp", "")}) for rec in ctx.ntlm_history: - events.append({"type": "ntlm", "timestamp": rec.get("timestamp", ""), **rec}) + events.append({**rec, "type": "ntlm", "timestamp": rec.get("timestamp", "")}) for rec in ctx.attack_chain: - events.append({"type": "notice", "timestamp": rec.get("timestamp", ""), **rec}) + events.append({**rec, "type": "notice", "timestamp": rec.get("timestamp", "")}) events.sort(key=lambda e: e.get("timestamp", "")) return events diff --git a/tests/test_correlator.py b/tests/test_correlator.py index 42e8673..34f7833 100644 --- a/tests/test_correlator.py +++ b/tests/test_correlator.py @@ -167,6 +167,26 @@ def test_timeline_does_not_mutate_source_records() -> None: assert "type" not in original +def test_timeline_explicit_keys_override_rec() -> None: + """Explicit type/timestamp must win even when the source record has those keys.""" + rec_with_type = { + "timestamp": "2024-06-01T10:00:00.000Z", + "type": "SHOULD_BE_OVERRIDDEN", + "src_ip": PRIVATE_SRC, + } + ctx = IncidentContext( + trigger_type="ip_pair", + trigger={}, + src_ip=PRIVATE_SRC, + dest_ip=PRIVATE_DEST, + sensor=SENSOR, + time_range=TIME_RANGE, + kerberos_history=[rec_with_type], + ) + timeline = build_timeline(ctx) + assert timeline[0]["type"] == "kerberos" + + # --------------------------------------------------------------------------- # investigate() — IP routing logic # --------------------------------------------------------------------------- From 2cad4dafa85a38f62c589507a2730958de6225e2 Mon Sep 17 00:00:00 2001 From: liamadale Date: Thu, 30 Apr 2026 14:54:57 -0700 Subject: [PATCH 058/109] feat(profiler): public IP profiling with sensor presence and reverse DNS Add PublicIPProfile dataclass and profile_public_ip() orchestrator that runs 8 parallel aggregation queries: sensor presence, reverse DNS, conn to/from, SSL/TLS server-side, HTTP server-side, SSH inbound, and RDP inbound. Add public IP role classifier with weighted heuristics for 8 roles: web_server, scanner, cdn_node, mail_server, dns_server, ssh_server, vpn_endpoint, and c2_suspect. --- src/profiler/public_ip_profiler.py | 446 +++++++++++++++++++++++++ src/profiler/public_role_classifier.py | 178 ++++++++++ 2 files changed, 624 insertions(+) create mode 100644 src/profiler/public_ip_profiler.py create mode 100644 src/profiler/public_role_classifier.py diff --git a/src/profiler/public_ip_profiler.py b/src/profiler/public_ip_profiler.py new file mode 100644 index 0000000..2ece02c --- /dev/null +++ b/src/profiler/public_ip_profiler.py @@ -0,0 +1,446 @@ +"""Public IP profiler — network-perspective profiles for external hosts. + +Runs 8 parallel aggregation queries against OpenSearch to build a +PublicIPProfile: sensor presence, reverse DNS, services exposed, +TLS/cert info, HTTP server headers, and inbound attack signals. +""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass, field + +from src.querier.zeek_modules.base import INDEX, query_opensearch +from src.utils.ip_org import lookup_org + +_PARAMS = {"path": f"{INDEX}/_search", "method": "POST"} + + +# --------------------------------------------------------------------------- +# PublicIPProfile dataclass +# --------------------------------------------------------------------------- + + +@dataclass +class PublicIPProfile: + """Network-perspective profile for a public IP address.""" + + ip: str + time_range: str + + # Sensor presence + sensors: list[dict] = field(default_factory=list) + total_records: int = 0 + + # Identity + org: dict | None = None + reverse_dns: list[dict] = field(default_factory=list) + + # Services exposed (our traffic TO this IP) + services: list[dict] = field(default_factory=list) + internal_client_count: int = 0 + bytes_to: int = 0 + bytes_from: int = 0 + + # TLS (server-side) + ja4s_fingerprints: list[dict] = field(default_factory=list) + tls_versions: list[dict] = field(default_factory=list) + ssl_subjects: list[str] = field(default_factory=list) + ssl_issuers: list[str] = field(default_factory=list) + + # HTTP (server-side) + http_server_headers: list[str] = field(default_factory=list) + http_top_uris: list[dict] = field(default_factory=list) + + # Inbound activity FROM this IP (scanner/attacker signals) + inbound_ports_targeted: list[dict] = field(default_factory=list) + internal_targets_count: int = 0 + ssh_inbound: bool = False + ssh_server_versions: list[str] = field(default_factory=list) + rdp_inbound: bool = False + rdp_usernames: list[str] = field(default_factory=list) + + # Classification + role: str = "unknown" + confidence: float = 0.0 + + # Timestamps + first_seen: str = "" + last_seen: str = "" + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _buckets(aggs: dict, key: str) -> list[dict]: + """Extract buckets from an aggregation.""" + return aggs.get(key, {}).get("buckets", []) + + +def _ip_should(ip: str) -> list[dict]: + """Match IP as either source or destination.""" + return [{"term": {"source.ip": ip}}, {"term": {"destination.ip": ip}}] + + +# --------------------------------------------------------------------------- +# Query builders (8 queries) +# --------------------------------------------------------------------------- + + +def _sensor_presence_query(ip: str, time_range: str) -> dict: + """Which sensors have seen this IP, and how many records each.""" + return { + "size": 0, + "query": { + "bool": { + "must": [ + {"range": {"@timestamp": {"gte": time_range, "lte": "now"}}}, + {"bool": {"should": _ip_should(ip)}}, + ] + } + }, + "aggs": { + "sensors": {"terms": {"field": "host.name", "size": 50}}, + "time_range": {"stats": {"field": "@timestamp"}}, + }, + } + + +def _reverse_dns_query(ip: str, time_range: str) -> dict: + """Domains that resolved to this IP (from DNS answer logs).""" + return { + "size": 0, + "query": { + "bool": { + "must": [ + {"range": {"@timestamp": {"gte": time_range, "lte": "now"}}}, + {"term": {"event.dataset": "dns"}}, + {"term": {"zeek.dns.answers": ip}}, + ] + } + }, + "aggs": {"domains": {"terms": {"field": "zeek.dns.query", "size": 20}}}, + } + + +def _conn_to_query(ip: str, time_range: str) -> dict: + """Conn records where this IP is the destination (our traffic TO it).""" + return { + "size": 0, + "query": { + "bool": { + "must": [ + {"range": {"@timestamp": {"gte": time_range, "lte": "now"}}}, + {"term": {"event.dataset": "conn"}}, + {"term": {"destination.ip": ip}}, + ] + } + }, + "aggs": { + "dest_ports": { + "terms": {"field": "destination.port", "size": 20}, + "aggs": {"app_proto": {"terms": {"field": "network.application", "size": 1}}}, + }, + "unique_clients": {"cardinality": {"field": "source.ip"}}, + "bytes_to": {"sum": {"field": "destination.bytes"}}, + "bytes_from": {"sum": {"field": "source.bytes"}}, + "time_range": {"stats": {"field": "@timestamp"}}, + }, + } + + +def _conn_from_query(ip: str, time_range: str) -> dict: + """Conn records where this IP is the source (inbound from it).""" + return { + "size": 0, + "query": { + "bool": { + "must": [ + {"range": {"@timestamp": {"gte": time_range, "lte": "now"}}}, + {"term": {"event.dataset": "conn"}}, + {"term": {"source.ip": ip}}, + ] + } + }, + "aggs": { + "inbound_ports": { + "terms": {"field": "destination.port", "size": 20}, + }, + "unique_targets": {"cardinality": {"field": "destination.ip"}}, + }, + } + + +def _ssl_to_query(ip: str, time_range: str) -> dict: + """SSL/TLS records where this IP is the server (destination).""" + return { + "size": 0, + "query": { + "bool": { + "must": [ + {"range": {"@timestamp": {"gte": time_range, "lte": "now"}}}, + {"term": {"event.dataset": "ssl"}}, + {"term": {"destination.ip": ip}}, + ] + } + }, + "aggs": { + "ja4s": {"terms": {"field": "tls.ja4s", "size": 10}}, + "tls_versions": {"terms": {"field": "network.protocol_version", "size": 5}}, + "subjects": {"terms": {"field": "zeek.ssl.subject", "size": 10}}, + "issuers": {"terms": {"field": "zeek.ssl.issuer", "size": 10}}, + }, + } + + +def _http_to_query(ip: str, time_range: str) -> dict: + """HTTP records where this IP is the server (destination).""" + return { + "size": 0, + "query": { + "bool": { + "must": [ + {"range": {"@timestamp": {"gte": time_range, "lte": "now"}}}, + {"term": {"event.dataset": "http"}}, + {"term": {"destination.ip": ip}}, + ] + } + }, + "aggs": { + "server_headers": {"terms": {"field": "zeek.http.server_header_names", "size": 10}}, + "top_uris": {"terms": {"field": "zeek.http.uri", "size": 10}}, + }, + } + + +def _ssh_from_query(ip: str, time_range: str) -> dict: + """SSH records where this IP is the source (inbound SSH from it).""" + return { + "size": 0, + "query": { + "bool": { + "must": [ + {"range": {"@timestamp": {"gte": time_range, "lte": "now"}}}, + {"term": {"event.dataset": "ssh"}}, + {"term": {"source.ip": ip}}, + ] + } + }, + "aggs": { + "server_versions": {"terms": {"field": "zeek.ssh.server", "size": 5}}, + }, + } + + +def _rdp_from_query(ip: str, time_range: str) -> dict: + """RDP records where this IP is the source (inbound RDP from it).""" + return { + "size": 0, + "query": { + "bool": { + "must": [ + {"range": {"@timestamp": {"gte": time_range, "lte": "now"}}}, + {"term": {"event.dataset": "rdp"}}, + {"term": {"source.ip": ip}}, + ] + } + }, + "aggs": { + "cookies": {"terms": {"field": "zeek.rdp.cookie", "size": 10}}, + }, + } + + +# --------------------------------------------------------------------------- +# Response parsers +# --------------------------------------------------------------------------- + + +def _parse_sensor_presence(aggs: dict) -> dict: + sensors = [{"sensor": b["key"], "count": b["doc_count"]} for b in _buckets(aggs, "sensors")] + total = sum(s["count"] for s in sensors) + ts = aggs.get("time_range", {}) + return { + "sensors": sensors, + "total_records": total, + "first_seen": ts.get("min_as_string", ""), + "last_seen": ts.get("max_as_string", ""), + } + + +def _parse_reverse_dns(aggs: dict) -> dict: + return { + "reverse_dns": [ + {"domain": b["key"], "count": b["doc_count"]} for b in _buckets(aggs, "domains") + ] + } + + +def _parse_conn_to(aggs: dict) -> dict: + services = [] + for b in _buckets(aggs, "dest_ports"): + proto_buckets = b.get("app_proto", {}).get("buckets", []) + app_proto = proto_buckets[0]["key"] if proto_buckets else "" + services.append( + { + "port": int(b["key"]), + "app_proto": app_proto, + "count": b["doc_count"], + } + ) + ts = aggs.get("time_range", {}) + return { + "services": services, + "internal_client_count": int(aggs.get("unique_clients", {}).get("value", 0)), + "bytes_to": int(aggs.get("bytes_to", {}).get("value", 0)), + "bytes_from": int(aggs.get("bytes_from", {}).get("value", 0)), + "first_seen": ts.get("min_as_string", ""), + "last_seen": ts.get("max_as_string", ""), + } + + +def _parse_conn_from(aggs: dict) -> dict: + return { + "inbound_ports_targeted": [ + {"port": int(b["key"]), "count": b["doc_count"]} + for b in _buckets(aggs, "inbound_ports") + ], + "internal_targets_count": int(aggs.get("unique_targets", {}).get("value", 0)), + } + + +def _parse_ssl_to(aggs: dict) -> dict: + return { + "ja4s_fingerprints": [ + {"hash": b["key"], "count": b["doc_count"]} for b in _buckets(aggs, "ja4s") + ], + "tls_versions": [ + {"version": b["key"], "count": b["doc_count"]} for b in _buckets(aggs, "tls_versions") + ], + "ssl_subjects": [b["key"] for b in _buckets(aggs, "subjects")], + "ssl_issuers": [b["key"] for b in _buckets(aggs, "issuers")], + } + + +def _parse_http_to(aggs: dict) -> dict: + return { + "http_server_headers": [b["key"] for b in _buckets(aggs, "server_headers")], + "http_top_uris": [ + {"uri": b["key"], "count": b["doc_count"]} for b in _buckets(aggs, "top_uris") + ], + } + + +def _parse_ssh_from(aggs: dict) -> dict: + count = sum(b["doc_count"] for b in _buckets(aggs, "server_versions")) + # If no server_versions buckets, check if the query itself had hits + total = aggs.get("server_versions", {}).get("sum_other_doc_count", 0) + count + return { + "ssh_inbound": total > 0 or count > 0, + "ssh_server_versions": [b["key"] for b in _buckets(aggs, "server_versions")], + } + + +def _parse_rdp_from(aggs: dict) -> dict: + cookies = _buckets(aggs, "cookies") + total = sum(b["doc_count"] for b in cookies) + return { + "rdp_inbound": total > 0, + "rdp_usernames": [b["key"] for b in cookies], + } + + +# --------------------------------------------------------------------------- +# Orchestrator +# --------------------------------------------------------------------------- + + +def profile_public_ip( + ip: str, + *, + time_range: str = "now-7d", +) -> PublicIPProfile: + """Profile a public IP using 8 parallel Zeek log aggregations. + + Args: + ip: Public IP address to profile. + time_range: Elasticsearch date-math range (default: now-7d). + + Returns: + PublicIPProfile with all fields populated. + """ + org = lookup_org(ip) + + queries: dict[str, dict] = { + "sensor": _sensor_presence_query(ip, time_range), + "rdns": _reverse_dns_query(ip, time_range), + "conn_to": _conn_to_query(ip, time_range), + "conn_from": _conn_from_query(ip, time_range), + "ssl_to": _ssl_to_query(ip, time_range), + "http_to": _http_to_query(ip, time_range), + "ssh_from": _ssh_from_query(ip, time_range), + "rdp_from": _rdp_from_query(ip, time_range), + } + + results: dict[str, dict] = {} + with ThreadPoolExecutor(max_workers=8) as ex: + futures = { + ex.submit(query_opensearch, body, _PARAMS): name for name, body in queries.items() + } + for f in as_completed(futures): + name = futures[f] + raw = f.result() + results[name] = raw.get("aggregations", {}) if raw else {} + + sensor = _parse_sensor_presence(results.get("sensor", {})) + rdns = _parse_reverse_dns(results.get("rdns", {})) + conn_to = _parse_conn_to(results.get("conn_to", {})) + conn_from = _parse_conn_from(results.get("conn_from", {})) + ssl_to = _parse_ssl_to(results.get("ssl_to", {})) + http_to = _parse_http_to(results.get("http_to", {})) + ssh_from = _parse_ssh_from(results.get("ssh_from", {})) + rdp_from = _parse_rdp_from(results.get("rdp_from", {})) + + first_seen = min( + (t for t in [sensor["first_seen"], conn_to["first_seen"]] if t), + default="", + ) + last_seen = max( + (t for t in [sensor["last_seen"], conn_to["last_seen"]] if t), + default="", + ) + + profile = PublicIPProfile( + ip=ip, + time_range=time_range, + org=org, + sensors=sensor["sensors"], + total_records=sensor["total_records"], + reverse_dns=rdns["reverse_dns"], + services=conn_to["services"], + internal_client_count=conn_to["internal_client_count"], + bytes_to=conn_to["bytes_to"], + bytes_from=conn_to["bytes_from"], + ja4s_fingerprints=ssl_to["ja4s_fingerprints"], + tls_versions=ssl_to["tls_versions"], + ssl_subjects=ssl_to["ssl_subjects"], + ssl_issuers=ssl_to["ssl_issuers"], + http_server_headers=http_to["http_server_headers"], + http_top_uris=http_to["http_top_uris"], + inbound_ports_targeted=conn_from["inbound_ports_targeted"], + internal_targets_count=conn_from["internal_targets_count"], + ssh_inbound=ssh_from["ssh_inbound"], + ssh_server_versions=ssh_from["ssh_server_versions"], + rdp_inbound=rdp_from["rdp_inbound"], + rdp_usernames=rdp_from["rdp_usernames"], + first_seen=first_seen, + last_seen=last_seen, + ) + + from src.profiler.public_role_classifier import classify_public_role + + profile.role, profile.confidence = classify_public_role(profile) + + return profile diff --git a/src/profiler/public_role_classifier.py b/src/profiler/public_role_classifier.py new file mode 100644 index 0000000..978c922 --- /dev/null +++ b/src/profiler/public_role_classifier.py @@ -0,0 +1,178 @@ +"""Public IP role classification — weighted heuristics. + +Pure logic — operates on a populated PublicIPProfile, no I/O. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from src.profiler.public_ip_profiler import PublicIPProfile + + +def _has_service(profile: PublicIPProfile, port: int) -> bool: + return any(s["port"] == port for s in profile.services) + + +def _has_known_issuer(profile: PublicIPProfile) -> bool: + known = ("let's encrypt", "digicert", "comodo", "globalsign", "sectigo") + return any(any(k in iss.lower() for k in known) for iss in profile.ssl_issuers) + + +def _low_bytes_ratio(profile: PublicIPProfile) -> bool: + if profile.internal_targets_count == 0: + return False + total = profile.bytes_to + profile.bytes_from + return (total / profile.internal_targets_count) < 1024 + + +_MAIL_PORTS = {25, 465, 587, 993, 143} + + +PUBLIC_ROLE_HEURISTICS: dict[str, list[tuple[float, str, object]]] = { + "web_server": [ + (0.30, "HTTPS service", lambda p: _has_service(p, 443)), + (0.20, "HTTP service", lambda p: _has_service(p, 80)), + (0.20, "Multiple clients", lambda p: p.internal_client_count > 5), + (0.15, "Valid TLS cert", lambda p: _has_known_issuer(p)), + (0.15, "Has reverse DNS", lambda p: len(p.reverse_dns) > 0), + ], + "scanner": [ + (0.30, "High target count", lambda p: p.internal_targets_count > 20), + ( + 0.25, + "Many ports probed", + lambda p: len(p.inbound_ports_targeted) > 5, + ), + ( + 0.25, + "Inbound conn from IP", + lambda p: p.internal_targets_count > 0, + ), + (0.20, "Low bytes per target", lambda p: _low_bytes_ratio(p)), + ], + "cdn_node": [ + ( + 0.35, + "Known CDN org", + lambda p: p.org and p.org.get("category") == "cdn", + ), + (0.25, "HTTPS primary", lambda p: _has_service(p, 443)), + (0.20, "High client count", lambda p: p.internal_client_count > 20), + (0.20, "Multiple domains", lambda p: len(p.reverse_dns) > 2), + ], + "mail_server": [ + ( + 0.35, + "Mail port", + lambda p: any(_has_service(p, pt) for pt in _MAIL_PORTS), + ), + ( + 0.30, + "MX-related DNS", + lambda p: any( + "mx" in d["domain"].lower() or "mail" in d["domain"].lower() for d in p.reverse_dns + ), + ), + (0.20, "Has reverse DNS", lambda p: len(p.reverse_dns) > 0), + (0.15, "Valid TLS cert", lambda p: _has_known_issuer(p)), + ], + "dns_server": [ + (0.40, "Port 53 service", lambda p: _has_service(p, 53)), + (0.30, "Many clients", lambda p: p.internal_client_count > 10), + ( + 0.30, + "No web ports", + lambda p: _has_service(p, 53) and not _has_service(p, 443), + ), + ], + "ssh_server": [ + (0.40, "Port 22 service", lambda p: _has_service(p, 22)), + ( + 0.30, + "SSH server version", + lambda p: len(p.ssh_server_versions) > 0, + ), + ( + 0.30, + "Few clients", + lambda p: _has_service(p, 22) and p.internal_client_count <= 5, + ), + ], + "vpn_endpoint": [ + ( + 0.30, + "VPN ports", + lambda p: any(_has_service(p, pt) for pt in (1194, 500, 4500, 51820)), + ), + ( + 0.30, + "High bytes bidirectional", + lambda p: (p.bytes_to + p.bytes_from) > 10_000_000, + ), + (0.20, "Few clients", lambda p: len(p.services) > 0 and p.internal_client_count <= 3), + (0.20, "HTTPS or VPN port", lambda p: _has_service(p, 443)), + ], + "c2_suspect": [ + ( + 0.30, + "Self-signed cert", + lambda p: len(p.ssl_subjects) > 0 and not _has_known_issuer(p), + ), + (0.25, "Unusual ports", lambda p: _has_unusual_ports(p)), + ( + 0.25, + "Low client count", + lambda p: 0 < p.internal_client_count <= 2, + ), + ( + 0.20, + "No reverse DNS", + lambda p: len(p.services) > 0 and len(p.reverse_dns) == 0, + ), + ], +} + +_COMMON_PORTS = { + 22, + 25, + 53, + 80, + 143, + 443, + 465, + 587, + 993, + 8080, + 8443, + 123, + 500, + 4500, +} + + +def _has_unusual_ports(profile: PublicIPProfile) -> bool: + return any(s["port"] not in _COMMON_PORTS for s in profile.services) + + +def classify_public_role( + profile: PublicIPProfile, +) -> tuple[str, float]: + """Classify public IP role using weighted heuristics. + + Returns: + (role, confidence) where confidence is 0.0–1.0. + """ + best_role = "unknown" + best_confidence = 0.0 + + for role, signals in PUBLIC_ROLE_HEURISTICS.items(): + matched = sum(w for w, _, fn in signals if fn(profile)) + total = sum(w for w, _, _ in signals) + confidence = matched / total if total > 0 else 0.0 + if confidence > best_confidence: + best_confidence = confidence + best_role = role + + return best_role, round(best_confidence, 2) From 674b307303abe0d50df6a12067f80cb55ba0d6e3 Mon Sep 17 00:00:00 2001 From: liamadale Date: Thu, 30 Apr 2026 14:55:10 -0700 Subject: [PATCH 059/109] feat(opensearch): public device card, profile buttons, and investigate enhancements Add public_device_card.html partial with sensor presence, reverse DNS, services, TLS, and inbound attack signal sections. Update ip_pivot.html to show both Profile and Enrich buttons for public IPs targeting separate slots. Update /api/profile route to route public IPs to profile_public_ip(). Update /api/investigate/profiles to profile and enrich public IPs in parallel. Add Suricata Alerts and Zeek Notices sections to the investigate view with bidirectional IP pair queries. Add Profile button next to Enrich in the record detail panel for both private and public IPs. Add Expand All toggle to both profile card footers. Surface Investigate button for all log types in detail panel. Fix device-card and mini-table overflow in investigate profiles grid. Add public_role_icon Jinja global. --- apps/opensearch_web/app.py | 164 +++++++++++++++--- apps/opensearch_web/static/pisces.css | 9 +- apps/opensearch_web/templates/base.html | 8 + apps/opensearch_web/templates/ip_pivot.html | 14 +- .../templates/partials/device_card.html | 1 + .../partials/investigate_profiles.html | 45 ++++- .../partials/public_device_card.html | 162 +++++++++++++++++ .../templates/partials/record_detail.html | 46 ++++- apps/shared/jinja_globals.py | 12 ++ 9 files changed, 415 insertions(+), 46 deletions(-) create mode 100644 apps/opensearch_web/templates/partials/public_device_card.html diff --git a/apps/opensearch_web/app.py b/apps/opensearch_web/app.py index 23a3af3..2d2a8b7 100644 --- a/apps/opensearch_web/app.py +++ b/apps/opensearch_web/app.py @@ -593,17 +593,22 @@ def api_sensor_summary(): def api_profile(ip: str): from src.querier.zeek_modules.base import is_private - if not is_private(ip): - return '

Device profiling is for private IPs only.

' - - from src.profiler.device_profiler import profile_device - sensor = request.args.get("sensor", "all") time_range = request.args.get("time_range", "now-7d") compact = request.args.get("compact") == "1" - profile = profile_device(ip, time_range=time_range, sensor=sensor) - return render_template("partials/device_card.html", profile=profile, compact=compact) + if is_private(ip): + from src.profiler.device_profiler import profile_device + + profile = profile_device(ip, time_range=time_range, sensor=sensor) + return render_template("partials/device_card.html", profile=profile, compact=compact) + else: + from src.profiler.public_ip_profiler import profile_public_ip + + profile = profile_public_ip(ip, time_range=time_range) + return render_template( + "partials/public_device_card.html", profile=profile, compact=compact + ) # ------------------------------------------------------------------ # GET /investigate// — one-click incident context page @@ -625,7 +630,9 @@ def investigate_view(src_ip: str, dest_ip: str): def api_investigate_profiles(): from concurrent.futures import ThreadPoolExecutor + from src.enricher.threat_intel import enrich_ip from src.profiler.device_profiler import profile_device + from src.profiler.public_ip_profiler import profile_public_ip from src.querier.zeek_modules.base import is_private src_ip = request.args.get("src_ip", "") @@ -635,25 +642,47 @@ def api_investigate_profiles(): src_profile = None dest_profile = None + src_enrichment = None + dest_enrichment = None src_error = None dest_error = None - def _profile(ip: str): # type: ignore[no-untyped-def] - return profile_device(ip, time_range=time_range, sensor=sensor) + def _do_src(): + nonlocal src_profile, src_enrichment, src_error + try: + if is_private(src_ip): + src_profile = profile_device(src_ip, time_range=time_range, sensor=sensor) + else: + src_profile = profile_public_ip(src_ip, time_range=time_range) + src_enrichment = enrich_ip(src_ip, offer_fp=False) + except Exception as exc: + src_error = str(exc) + + def _do_dest(): + nonlocal dest_profile, dest_enrichment, dest_error + try: + if is_private(dest_ip): + dest_profile = profile_device(dest_ip, time_range=time_range, sensor=sensor) + else: + dest_profile = profile_public_ip(dest_ip, time_range=time_range) + dest_enrichment = enrich_ip(dest_ip, offer_fp=False) + except Exception as exc: + dest_error = str(exc) with ThreadPoolExecutor(max_workers=2) as pool: - f_src = pool.submit(_profile, src_ip) if is_private(src_ip) else None - f_dest = pool.submit(_profile, dest_ip) if is_private(dest_ip) else None - if f_src is not None: - try: - src_profile = f_src.result() - except Exception as exc: - src_error = str(exc) - if f_dest is not None: - try: - dest_profile = f_dest.result() - except Exception as exc: - dest_error = str(exc) + pool.submit(_do_src) + f_dest = pool.submit(_do_dest) + f_dest.result() # wait for both + + def _enrich_urls(ip: str) -> dict: + from src.enricher import abuseipdb, greynoise, shodan, virustotal + + return { + "greynoise": greynoise.URL.format(ip=ip), + "abuseipdb": abuseipdb.URL.format(ip=ip), + "shodan": shodan.URL.format(ip=ip), + "virustotal": virustotal.URL.format(ip=ip), + } return render_template( "partials/investigate_profiles.html", @@ -661,6 +690,10 @@ def _profile(ip: str): # type: ignore[no-untyped-def] dest_ip=dest_ip, src_profile=src_profile, dest_profile=dest_profile, + src_enrichment=src_enrichment, + dest_enrichment=dest_enrichment, + src_urls=_enrich_urls(src_ip) if src_enrichment else None, + dest_urls=_enrich_urls(dest_ip) if dest_enrichment else None, src_error=src_error, dest_error=dest_error, ) @@ -716,6 +749,95 @@ def api_investigate_chain(): error=error, ) + # ------------------------------------------------------------------ + # GET /api/investigate/notices — HTMX: Zeek notices for IP pair + # ------------------------------------------------------------------ + @app.route("/api/investigate/notices") + def api_investigate_notices(): + src_ip = request.args.get("src_ip", "") + dest_ip = request.args.get("dest_ip", "") + sensor = request.args.get("sensor", "all") + time_range = request.args.get("time_range", "now-24h") + + try: + from src.querier.zeek_modules.base import run_query + + base = { + "sensor": sensor, + "time_range": time_range, + "limit": 200, + "no_filters": False, + "public_only": False, + "raise_on_error": False, + } + fwd = run_query(MODULES["notice"], {**base, "src_ip": src_ip}) + fwd = [r for r in fwd if r.get("dest_ip") == dest_ip] + rev = run_query(MODULES["notice"], {**base, "src_ip": dest_ip}) + rev = [r for r in rev if r.get("dest_ip") == src_ip] + records = fwd + rev + records.sort(key=lambda r: r.get("timestamp", "")) + error = None + except Exception as exc: + records, error = [], str(exc) + + return render_template( + "partials/investigate_notices.html", + notices=records, + src_ip=src_ip, + dest_ip=dest_ip, + error=error, + ) + + # ------------------------------------------------------------------ + # GET /api/investigate/suricata — HTMX: Suricata alerts for IP pair + # ------------------------------------------------------------------ + @app.route("/api/investigate/suricata") + def api_investigate_suricata(): + src_ip = request.args.get("src_ip", "") + dest_ip = request.args.get("dest_ip", "") + sensor = request.args.get("sensor", "all") + time_range = request.args.get("time_range", "now-24h") + + try: + from src.querier.zeek_modules.base import run_query + + base = { + "sensor": sensor, + "time_range": time_range, + "limit": 200, + "no_filters": False, + "public_only": False, + "raise_on_error": False, + "exclude_stream": True, + } + # src→dest direction + fwd = run_query(MODULES["suricata_alert"], {**base, "src_ip": src_ip}) + fwd = [r for r in fwd if r.get("dest_ip") == dest_ip] + # dest→src direction + rev = run_query(MODULES["suricata_alert"], {**base, "src_ip": dest_ip}) + rev = [r for r in rev if r.get("dest_ip") == src_ip] + records = fwd + rev + records.sort(key=lambda r: r.get("timestamp", "")) + error = None + except Exception as exc: + records, error = [], str(exc) + + return render_template( + "partials/investigate_suricata.html", + alerts=records, + src_ip=src_ip, + dest_ip=dest_ip, + error=error, + ) + + return render_template( + "partials/investigate_suricata.html", + alerts=records, + src_ip=src_ip, + dest_ip=dest_ip, + error=error, + ) + # ------------------------------------------------------------------ # GET /api/investigate/tickets — HTMX: Mantis tickets for src + dest # ------------------------------------------------------------------ diff --git a/apps/opensearch_web/static/pisces.css b/apps/opensearch_web/static/pisces.css index 994482b..7618fc6 100644 --- a/apps/opensearch_web/static/pisces.css +++ b/apps/opensearch_web/static/pisces.css @@ -1427,17 +1427,18 @@ tr.detail-row td { /* --------------------------------------------------------------------------- Device profile card --------------------------------------------------------------------------- */ -.device-card { border: 1px solid var(--border); border-radius: 8px; margin: 1rem 0; } +.device-card { border: 1px solid var(--border); border-radius: 8px; margin: 1rem 0; min-width: 0; overflow: hidden; } .device-card-header { padding: .75rem 1rem; border-bottom: 1px solid var(--border); display: flex; justify-content: space-between; align-items: center; flex-wrap: wrap; gap: .5rem; } .device-card-identity { font-size: 1.1rem; } .device-card-domain { color: var(--on-surface-dim); } .device-card-meta { display: flex; gap: .5rem; align-items: center; } .device-card-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 1px; background: var(--border); } -.device-card-section { padding: .75rem 1rem; background: var(--surface); } +.device-card-section { padding: .75rem 1rem; background: var(--surface); overflow: hidden; } .device-card-section h4, .device-card-section summary { margin: 0 0 .5rem; font-size: .8rem; color: var(--on-surface-dim); text-transform: uppercase; letter-spacing: .05em; cursor: pointer; } .device-card-section h4 { cursor: default; } .device-card-footer { padding: .5rem 1rem; font-size: .75rem; color: var(--on-surface-dim); border-top: 1px solid var(--border); } +.device-card-footer .btn-link { background: none; border: none; color: var(--primary); cursor: pointer; font-size: .75rem; padding: 0; margin-right: .5rem; text-decoration: underline; } .device-card-users { padding: 0 1rem .5rem; font-size: .8rem; color: var(--on-surface-dim); border-bottom: 1px solid var(--border); } .role-badge { padding: 2px 8px; border-radius: 4px; font-size: .8rem; font-weight: 600; } .role-domain_controller { background: color-mix(in srgb, var(--blue) 15%, transparent); color: var(--blue); } @@ -1449,9 +1450,9 @@ tr.detail-row td { .role-unknown { background: var(--surface-dim); color: var(--on-surface-dim); } .os-badge { padding: 2px 8px; border-radius: 4px; font-size: .75rem; background: var(--surface-dim); } .confidence { font-weight: 400; opacity: .7; margin-left: .25rem; } -.mini-table { width: 100%; font-size: .8rem; border-collapse: collapse; } +.mini-table { width: 100%; font-size: .8rem; border-collapse: collapse; table-layout: fixed; } .mini-table th { text-align: left; font-weight: 600; color: var(--on-surface-dim); border-bottom: 1px solid var(--border); padding: 2px 4px; } -.mini-table td { padding: 2px 4px; } +.mini-table td { padding: 2px 4px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } .mini-label { font-size: .75rem; color: var(--on-surface-dim); margin: .25rem 0 .125rem; } .tag-list { list-style: none; padding: 0; margin: 0; display: flex; flex-wrap: wrap; gap: .25rem; } .tag { padding: 2px 6px; border-radius: 3px; font-size: .75rem; background: var(--surface-dim); } diff --git a/apps/opensearch_web/templates/base.html b/apps/opensearch_web/templates/base.html index d0b44ac..f835028 100644 --- a/apps/opensearch_web/templates/base.html +++ b/apps/opensearch_web/templates/base.html @@ -334,6 +334,14 @@ }); }; + window.toggleDetailsAll = function(btn) { + var card = btn.closest(".device-card"); + var details = card.querySelectorAll("details"); + var allOpen = Array.from(details).every(function(d) { return d.open; }); + details.forEach(function(d) { d.open = !allOpen; }); + btn.textContent = allOpen ? "Expand all" : "Collapse all"; + }; + window.handleDetailToggle = function(el) { var row = el.tagName === "TR" ? el : el.closest("tr"); var isSelected = row.classList.contains("row-selected"); diff --git a/apps/opensearch_web/templates/ip_pivot.html b/apps/opensearch_web/templates/ip_pivot.html index c85e423..ff3053b 100644 --- a/apps/opensearch_web/templates/ip_pivot.html +++ b/apps/opensearch_web/templates/ip_pivot.html @@ -8,19 +8,18 @@

IP Pivot: {{ ip }}

{{ fmt_time_window(search_params.get('time_range', 'now-24h')) }} - {% if is_private(ip) %} - {% else %} + {% if not is_private(ip) %} @@ -30,7 +29,10 @@

IP Pivot: {{ ip }}

-
+
+
+
+
{% for lt, records in results.items() %} {% set mod = MODULES[lt] %} diff --git a/apps/opensearch_web/templates/partials/device_card.html b/apps/opensearch_web/templates/partials/device_card.html index 13ea137..7fd568e 100644 --- a/apps/opensearch_web/templates/partials/device_card.html +++ b/apps/opensearch_web/templates/partials/device_card.html @@ -149,6 +149,7 @@

Software

{# Footer #}
+ {% if profile.first_seen %} First: {{ profile.first_seen[:16] }} · Last: {{ profile.last_seen[:16] }} {% endif %} diff --git a/apps/opensearch_web/templates/partials/investigate_profiles.html b/apps/opensearch_web/templates/partials/investigate_profiles.html index 3987449..07feb5c 100644 --- a/apps/opensearch_web/templates/partials/investigate_profiles.html +++ b/apps/opensearch_web/templates/partials/investigate_profiles.html @@ -1,5 +1,6 @@ {# Investigate — device profiles partial. - Receives: src_ip, dest_ip, src_profile, dest_profile, src_error, dest_error #} + Receives: src_ip, dest_ip, src_profile, dest_profile, + src_enrichment, dest_enrichment, src_error, dest_error #}
{# Source IP #} @@ -11,11 +12,24 @@ {% if src_error %}

{{ src_error }}

{% elif src_profile %} - {% with profile=src_profile, compact=False %} - {% include "partials/device_card.html" %} - {% endwith %} + {% if is_private(src_ip) %} + {% with profile=src_profile, compact=False %} + {% include "partials/device_card.html" %} + {% endwith %} + {% else %} + {% with profile=src_profile, compact=False %} + {% include "partials/public_device_card.html" %} + {% endwith %} + {% endif %} {% else %} -

Public IP — device profiling not available.

+

No profile data available.

+ {% endif %} + {% if src_enrichment %} +
+ {% with result=src_enrichment, urls=src_urls, ip=src_ip %} + {% include "partials/enrich_card.html" ignore missing %} + {% endwith %} +
{% endif %}
@@ -28,11 +42,24 @@ {% if dest_error %}

{{ dest_error }}

{% elif dest_profile %} - {% with profile=dest_profile, compact=False %} - {% include "partials/device_card.html" %} - {% endwith %} + {% if is_private(dest_ip) %} + {% with profile=dest_profile, compact=False %} + {% include "partials/device_card.html" %} + {% endwith %} + {% else %} + {% with profile=dest_profile, compact=False %} + {% include "partials/public_device_card.html" %} + {% endwith %} + {% endif %} {% else %} -

Public IP — device profiling not available.

+

No profile data available.

+ {% endif %} + {% if dest_enrichment %} +
+ {% with result=dest_enrichment, urls=dest_urls, ip=dest_ip %} + {% include "partials/enrich_card.html" ignore missing %} + {% endwith %} +
{% endif %}
diff --git a/apps/opensearch_web/templates/partials/public_device_card.html b/apps/opensearch_web/templates/partials/public_device_card.html new file mode 100644 index 0000000..c2c978c --- /dev/null +++ b/apps/opensearch_web/templates/partials/public_device_card.html @@ -0,0 +1,162 @@ +{# Public IP profile card — network-perspective view of an external host. + Receives: profile (PublicIPProfile), compact (bool, optional) #} +{% set compact = compact|default(false) %} +
+ + {# Header #} +
+
+ + {{ profile.ip }} + {% if profile.org %} + · {{ profile.org.get('name', '') }} + {% if profile.org.get('category') %}({{ profile.org.get('category') }}){% endif %} + + {% endif %} +
+
+ + {{ profile.role | replace('_', ' ') | title }} + {{ (profile.confidence * 100) | int }}% + +
+
+ + {# Signal grid #} +
+ + {# Sensor presence #} +
+

Seen by {{ profile.sensors | length }} sensor{{ 's' if profile.sensors | length != 1 else '' }}

+ {% if profile.sensors %} + + + + {% for s in profile.sensors %} + + {% endfor %} + +
Sensor#
{{ s.sensor }}{{ "{:,}".format(s.count) }}
+ {% endif %} +
+ + {# Reverse DNS #} + {% if profile.reverse_dns %} +
+

Reverse DNS

+ + + + {% for d in profile.reverse_dns %} + + {% endfor %} + +
Domain#
{{ d.domain }}{{ "{:,}".format(d.count) }}
+
+ {% endif %} + + {# Services exposed #} +
+

Services (our traffic TO this IP)

+ {% if profile.services %} + + + + {% for svc in profile.services %} + + {% endfor %} + +
PortProto#
{{ svc.port }}{{ svc.app_proto or '—' }}{{ "{:,}".format(svc.count) }}
+

+ Internal clients: {{ "{:,}".format(profile.internal_client_count) }} + · Bytes: {{ profile.bytes_to | fmt_bytes }} in / {{ profile.bytes_from | fmt_bytes }} out +

+ {% else %} +

No outbound connections to this IP

+ {% endif %} +
+ + {% set open_attr = '' if compact else 'open' %} + + {# TLS #} + {% if profile.tls_versions or profile.ssl_issuers %} +
+ TLS + {% if profile.tls_versions %} +

+ {% for v in profile.tls_versions %}{{ v.version }} ({{ v.count }}){{ ', ' if not loop.last else '' }}{% endfor %} +

+ {% endif %} + {% if profile.ssl_issuers %} +

Issuer:

+
    {% for iss in profile.ssl_issuers %}
  • {{ iss }}
    • {% endfor %}
+ {% endif %} + {% if profile.ja4s_fingerprints %} + + + + {% for fp in profile.ja4s_fingerprints %} + + {% endfor %} + +
JA4S#
{{ fp.hash[:24] }}…{{ fp.count }}
+ {% endif %} +
+ {% endif %} + + {# HTTP #} + {% if profile.http_top_uris %} +
+ HTTP ({{ profile.http_top_uris | length }} URIs) + + + + {% for u in profile.http_top_uris %} + + {% endfor %} + +
URI#
{{ u.uri }}{{ u.count }}
+
+ {% endif %} + + {# Inbound attack signals #} + {% if profile.inbound_ports_targeted or profile.ssh_inbound or profile.rdp_inbound %} +
+ Inbound FROM this IP + {% if profile.inbound_ports_targeted %} + + + + {% for p in profile.inbound_ports_targeted %} + + {% endfor %} + +
Port targeted#
{{ p.port }}{{ "{:,}".format(p.count) }}
+

{{ "{:,}".format(profile.internal_targets_count) }} internal host{{ 's' if profile.internal_targets_count != 1 else '' }} targeted

+ {% endif %} + {% if profile.ssh_inbound %} +

SSH inbound{% if profile.ssh_server_versions %}: {{ profile.ssh_server_versions | join(', ') }}{% endif %}

+ {% endif %} + {% if profile.rdp_inbound %} +

RDP inbound{% if profile.rdp_usernames %}: {{ profile.rdp_usernames | join(', ') }}{% endif %}

+ {% endif %} +
+ {% elif not compact %} +
+

Inbound FROM this IP

+

No inbound connections detected

+
+ {% endif %} + +
+ + {# Footer #} +
+ + {% if profile.first_seen %} + First: {{ profile.first_seen[:16] }} · Last: {{ profile.last_seen[:16] }} + {% endif %} + · {{ "{:,}".format(profile.total_records) }} total records +
+ +
diff --git a/apps/opensearch_web/templates/partials/record_detail.html b/apps/opensearch_web/templates/partials/record_detail.html index 135b105..cbfbf26 100644 --- a/apps/opensearch_web/templates/partials/record_detail.html +++ b/apps/opensearch_web/templates/partials/record_detail.html @@ -24,18 +24,22 @@
{% if supports_fp is not defined or supports_fp %}
+ {% if record.get('src_ip') and record.get('dest_ip') %} + + Investigate + + {% else %} + + {% endif %} - {% if log_type == "notice" and record.get('src_ip') and record.get('dest_ip') %} - - Investigate - - {% endif %}
{% endif %}

@@ -92,6 +96,7 @@ {{ src_ip }} {% if is_private(src_ip) %} +
+
{% else %} +
+ +
{% endif %} +
{% endif %} @@ -120,6 +139,7 @@ {{ dest_ip }} {% if is_private(dest_ip) %} +
+
{% else %} +
+ +
{% endif %} +
{% endif %} diff --git a/apps/shared/jinja_globals.py b/apps/shared/jinja_globals.py index 26f3ba6..5a47553 100644 --- a/apps/shared/jinja_globals.py +++ b/apps/shared/jinja_globals.py @@ -23,6 +23,18 @@ def register_shared_helpers(app: Flask) -> None: "unknown": "circle-question", }.get(role, "circle-question") + app.jinja_env.globals["public_role_icon"] = lambda role: { + "web_server": "globe", + "scanner": "radar", + "cdn_node": "cloud", + "mail_server": "envelope", + "dns_server": "server", + "ssh_server": "terminal", + "vpn_endpoint": "shield-halved", + "c2_suspect": "skull-crossbones", + "unknown": "circle-question", + }.get(role, "circle-question") + from src.profiler.ja4_decoder import decode_ja4 app.jinja_env.globals["decode_ja4"] = decode_ja4 From 32bcde6c6098ec31851c96eead1749d5916644e0 Mon Sep 17 00:00:00 2001 From: liamadale Date: Thu, 30 Apr 2026 14:55:16 -0700 Subject: [PATCH 060/109] feat(opensearch): Suricata alerts and Zeek notices investigate partials --- .../opensearch_web/templates/investigate.html | 30 +++++++++++++ .../partials/investigate_notices.html | 42 ++++++++++++++++++ .../partials/investigate_suricata.html | 44 +++++++++++++++++++ 3 files changed, 116 insertions(+) create mode 100644 apps/opensearch_web/templates/partials/investigate_notices.html create mode 100644 apps/opensearch_web/templates/partials/investigate_suricata.html diff --git a/apps/opensearch_web/templates/investigate.html b/apps/opensearch_web/templates/investigate.html index 690b494..2be95d5 100644 --- a/apps/opensearch_web/templates/investigate.html +++ b/apps/opensearch_web/templates/investigate.html @@ -71,6 +71,36 @@

Attack Chain

+{# ── Suricata Alerts ──────────────────────────────────────────────── #} +
+
+

Suricata Alerts

+ IDS alerts between {{ src_ip }} ↔ {{ dest_ip }} +
+
+
+
+
+
+
+ +{# ── Zeek Notices ─────────────────────────────────────────────────── #} +
+
+

Zeek Notices

+ All notice types between {{ src_ip }} ↔ {{ dest_ip }} +
+
+
+
+
+
+
+ {# ── Related Tickets ──────────────────────────────────────────────── #}
diff --git a/apps/opensearch_web/templates/partials/investigate_notices.html b/apps/opensearch_web/templates/partials/investigate_notices.html new file mode 100644 index 0000000..63d2534 --- /dev/null +++ b/apps/opensearch_web/templates/partials/investigate_notices.html @@ -0,0 +1,42 @@ +{# Investigate — Zeek notices partial. + Receives: notices (list[dict]), src_ip, dest_ip, error #} + +{% if error %} +

{{ error }}

+{% elif not notices %} +

No Zeek notices between {{ src_ip }} and {{ dest_ip }} in this window.

+{% else %} +
+ + + + + + + + + + + + + + {% for rec in notices %} + + + + + + + + + + {% endfor %} + +
#TimeSensorSrc IPDst IPNoticeMessage
{{ loop.index }} + {{ rec.get('timestamp') | fmt_ts(full=True) }} + {{ rec.get('sensor', '—') }}{{ rec.get('src_ip', '—') }}{{ rec.get('dest_ip', '—') }}{{ rec.get('notice_note', '—') }}{{ (rec.get('notice_msg', '') or '')[:100] or '—' }}
+
+

+ {{ notices | length }} notice{{ 's' if notices | length != 1 else '' }} +

+{% endif %} diff --git a/apps/opensearch_web/templates/partials/investigate_suricata.html b/apps/opensearch_web/templates/partials/investigate_suricata.html new file mode 100644 index 0000000..66541dc --- /dev/null +++ b/apps/opensearch_web/templates/partials/investigate_suricata.html @@ -0,0 +1,44 @@ +{# Investigate — Suricata alerts partial. + Receives: alerts (list[dict]), src_ip, dest_ip, error #} + +{% if error %} +

{{ error }}

+{% elif not alerts %} +

No Suricata alerts between {{ src_ip }} and {{ dest_ip }} in this window.

+{% else %} +
+ + + + + + + + + + + + + + + {% for rec in alerts %} + + + + + + + + + + + {% endfor %} + +
#TimeSensorSrc IPDst IPSevRuleCategory
{{ loop.index }} + {{ rec.get('timestamp') | fmt_ts(full=True) }} + {{ rec.get('sensor', '—') }}{{ rec.get('src_ip', '—') }}{{ rec.get('dest_ip', '—') }}{{ rec.get('severity', '—') }}{{ (rec.get('rule_name', '') or '')[:80] or '—' }}{{ rec.get('rule_category', '—') }}
+
+

+ {{ alerts | length }} alert{{ 's' if alerts | length != 1 else '' }} +

+{% endif %} From 88f5205a1d170a294fbea7102afd73462ecd1ab4 Mon Sep 17 00:00:00 2001 From: liamadale Date: Thu, 30 Apr 2026 14:55:23 -0700 Subject: [PATCH 061/109] feat(mcp): profile_device supports public IPs Route public IPs to profile_public_ip() in the MCP profile_device tool and the investigate() correlator. Sensor becomes optional for public IPs. Update IncidentContext type hints to accept both profile types. --- mcp/opensearch/server.py | 28 +++++++++++++++++----------- src/correlator/incident_context.py | 12 +++++++++--- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/mcp/opensearch/server.py b/mcp/opensearch/server.py index c0e0f5b..8a60d44 100644 --- a/mcp/opensearch/server.py +++ b/mcp/opensearch/server.py @@ -1214,29 +1214,35 @@ def create_fp_filter( @mcp.tool() def profile_device( ip: str, - sensor: str, + sensor: str = "all", time_range: str = "now-7d", ) -> str: - """Profile a private IP by aggregating cross-protocol Zeek signals into a device card. + """Profile an IP by aggregating cross-protocol Zeek signals into a device card. - Runs 9 parallel aggregation queries (conn, DNS, SSL, HTTP, SMB, RDP, SSH) - to identify the device's role, OS, installed software, inbound services, - hostnames, fingerprints, and behavioral patterns — all from network - telemetry without endpoint agents. + For private IPs: runs 9 parallel aggregation queries (conn, DNS, SSL, HTTP, + SMB, RDP, SSH) to identify the device's role, OS, installed software, inbound + services, hostnames, fingerprints, and behavioral patterns. - The sensor parameter is required because private IPs overlap across sensors. + For public IPs: runs 8 parallel queries to build a network-perspective profile + showing sensor presence, reverse DNS, services exposed, TLS/cert info, and + inbound attack signals. Sensor is optional for public IPs. Args: - ip: Private IP address to profile (e.g. 10.0.0.50). - sensor: Sensor hostname — required, not optional. + ip: IP address to profile (private or public). + sensor: Sensor hostname — required for private IPs, optional for public. time_range: ES date-math range (default: now-7d). """ try: from dataclasses import asdict - from src.profiler.device_profiler import profile_device as _profile_device + if is_private(ip): + from src.profiler.device_profiler import profile_device as _profile_device + + profile = _profile_device(ip, time_range=time_range, sensor=sensor) + else: + from src.profiler.public_ip_profiler import profile_public_ip - profile = _profile_device(ip, time_range=time_range, sensor=sensor) + profile = profile_public_ip(ip, time_range=time_range) return _ok(asdict(profile)) except Exception as exc: return _err(str(exc)) diff --git a/src/correlator/incident_context.py b/src/correlator/incident_context.py index ea94f26..a138afe 100644 --- a/src/correlator/incident_context.py +++ b/src/correlator/incident_context.py @@ -10,6 +10,7 @@ if TYPE_CHECKING: from src.profiler.device_profiler import DeviceProfile + from src.profiler.public_ip_profiler import PublicIPProfile @dataclass @@ -24,9 +25,9 @@ class IncidentContext: sensor: str time_range: str - # Device profiles (None if IP is public or profiling failed) - src_profile: DeviceProfile | None = None - dest_profile: DeviceProfile | None = None + # Device profiles (None if profiling failed) + src_profile: DeviceProfile | PublicIPProfile | None = None + dest_profile: DeviceProfile | PublicIPProfile | None = None # Auth history between src ↔ dest kerberos_history: list[dict] = field(default_factory=list) @@ -57,6 +58,7 @@ class IncidentContext: from src.enricher.threat_intel import enrich_ip # noqa: E402 from src.mantis.mantis_search import search as search_tickets # noqa: E402 from src.profiler.device_profiler import profile_device # noqa: E402 +from src.profiler.public_ip_profiler import profile_public_ip # noqa: E402 from src.querier.zeek_modules import MODULES # noqa: E402 from src.querier.zeek_modules.base import ( # noqa: E402 INDEX, @@ -155,10 +157,14 @@ def investigate( def _profile_src() -> None: if is_private(src_ip): ctx.src_profile = profile_device(src_ip, time_range=time_range, sensor=sensor) + else: + ctx.src_profile = profile_public_ip(src_ip, time_range=time_range) def _profile_dest() -> None: if is_private(dest_ip): ctx.dest_profile = profile_device(dest_ip, time_range=time_range, sensor=sensor) + else: + ctx.dest_profile = profile_public_ip(dest_ip, time_range=time_range) def _auth_history() -> None: ctx.kerberos_history, ctx.ntlm_history = query_auth_history( From 6ea5d1a683d69bf7fb913da1a2d15ccf45361b11 Mon Sep 17 00:00:00 2001 From: liamadale Date: Thu, 30 Apr 2026 14:55:29 -0700 Subject: [PATCH 062/109] test(profiler): public IP profiler unit and integration tests Add 25 tests covering query builders, parsers, role classifier, and profile_public_ip() orchestrator with mocked ES. Update correlator tests to expect PublicIPProfile for public IPs. --- tests/test_correlator.py | 10 +- tests/test_public_ip_profiler.py | 366 +++++++++++++++++++++++++++++++ 2 files changed, 371 insertions(+), 5 deletions(-) create mode 100644 tests/test_public_ip_profiler.py diff --git a/tests/test_correlator.py b/tests/test_correlator.py index 34f7833..e8c36b0 100644 --- a/tests/test_correlator.py +++ b/tests/test_correlator.py @@ -207,25 +207,25 @@ def test_investigate_both_private() -> None: def test_investigate_one_public() -> None: - """Private src → profiled; public dest → enriched, not profiled.""" + """Private src → profiled; public dest → profiled (public) + enriched.""" patches = _base_patches(enrichment=MOCK_ENRICHMENT) with patches[0], patches[1], patches[2], patches[3], patches[4]: ctx = investigate(PRIVATE_SRC, PUBLIC_DEST, SENSOR, TIME_RANGE) assert ctx.src_profile is not None - assert ctx.dest_profile is None + assert ctx.dest_profile is not None # now a PublicIPProfile assert ctx.dest_enrichment is not None assert ctx.src_enrichment is None def test_investigate_both_public() -> None: - """Both public → no profiling, both enrichments populated.""" + """Both public → both profiled (public) + both enrichments populated.""" patches = _base_patches(enrichment=MOCK_ENRICHMENT) with patches[0], patches[1], patches[2], patches[3], patches[4]: ctx = investigate(PUBLIC_SRC, PUBLIC_DEST, SENSOR, TIME_RANGE) - assert ctx.src_profile is None - assert ctx.dest_profile is None + assert ctx.src_profile is not None # now a PublicIPProfile + assert ctx.dest_profile is not None # now a PublicIPProfile assert ctx.src_enrichment is not None assert ctx.dest_enrichment is not None diff --git a/tests/test_public_ip_profiler.py b/tests/test_public_ip_profiler.py new file mode 100644 index 0000000..117703a --- /dev/null +++ b/tests/test_public_ip_profiler.py @@ -0,0 +1,366 @@ +"""Tests for public_ip_profiler — query builders, parsers, classifier, orchestrator.""" + +from __future__ import annotations + +import os +import sys +from unittest.mock import patch + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from src.profiler.public_ip_profiler import ( + PublicIPProfile, + _conn_from_query, + _conn_to_query, + _http_to_query, + _parse_conn_from, + _parse_conn_to, + _parse_http_to, + _parse_rdp_from, + _parse_reverse_dns, + _parse_sensor_presence, + _parse_ssh_from, + _parse_ssl_to, + _rdp_from_query, + _reverse_dns_query, + _sensor_presence_query, + _ssh_from_query, + _ssl_to_query, + profile_public_ip, +) +from src.profiler.public_role_classifier import classify_public_role + +# --------------------------------------------------------------------------- +# Query builder tests — verify correct IP field and dataset filters +# --------------------------------------------------------------------------- + + +class TestQueryBuilders: + """Verify query builders produce correct ES query structure.""" + + def test_sensor_presence_uses_both_ip_fields(self): + q = _sensor_presence_query("1.2.3.4", "now-7d") + should = q["query"]["bool"]["must"][1]["bool"]["should"] + assert {"term": {"source.ip": "1.2.3.4"}} in should + assert {"term": {"destination.ip": "1.2.3.4"}} in should + + def test_reverse_dns_filters_on_answers(self): + q = _reverse_dns_query("1.2.3.4", "now-7d") + must = q["query"]["bool"]["must"] + assert {"term": {"zeek.dns.answers": "1.2.3.4"}} in must + assert {"term": {"event.dataset": "dns"}} in must + + def test_conn_to_uses_dest_ip(self): + q = _conn_to_query("1.2.3.4", "now-7d") + must = q["query"]["bool"]["must"] + assert {"term": {"destination.ip": "1.2.3.4"}} in must + assert {"term": {"event.dataset": "conn"}} in must + + def test_conn_from_uses_src_ip(self): + q = _conn_from_query("1.2.3.4", "now-7d") + must = q["query"]["bool"]["must"] + assert {"term": {"source.ip": "1.2.3.4"}} in must + + def test_ssl_to_uses_dest_ip(self): + q = _ssl_to_query("1.2.3.4", "now-7d") + must = q["query"]["bool"]["must"] + assert {"term": {"destination.ip": "1.2.3.4"}} in must + assert {"term": {"event.dataset": "ssl"}} in must + + def test_http_to_uses_dest_ip(self): + q = _http_to_query("1.2.3.4", "now-7d") + must = q["query"]["bool"]["must"] + assert {"term": {"destination.ip": "1.2.3.4"}} in must + + def test_ssh_from_uses_src_ip(self): + q = _ssh_from_query("1.2.3.4", "now-7d") + must = q["query"]["bool"]["must"] + assert {"term": {"source.ip": "1.2.3.4"}} in must + assert {"term": {"event.dataset": "ssh"}} in must + + def test_rdp_from_uses_src_ip(self): + q = _rdp_from_query("1.2.3.4", "now-7d") + must = q["query"]["bool"]["must"] + assert {"term": {"source.ip": "1.2.3.4"}} in must + assert {"term": {"event.dataset": "rdp"}} in must + + +# --------------------------------------------------------------------------- +# Parser tests — extract fields from mock ES aggregation responses +# --------------------------------------------------------------------------- + +MOCK_SENSOR_AGGS = { + "sensors": { + "buckets": [ + {"key": "hedgehog-east", "doc_count": 1423}, + {"key": "hedgehog-west", "doc_count": 892}, + ] + }, + "time_range": { + "min_as_string": "2026-04-24T08:30:00.000Z", + "max_as_string": "2026-04-30T13:41:00.000Z", + }, +} + +MOCK_RDNS_AGGS = { + "domains": { + "buckets": [ + {"key": "example.com", "doc_count": 1200}, + {"key": "www.example.com", "doc_count": 892}, + ] + } +} + +MOCK_CONN_TO_AGGS = { + "dest_ports": { + "buckets": [ + { + "key": 443, + "doc_count": 2100, + "app_proto": {"buckets": [{"key": "ssl", "doc_count": 2100}]}, + }, + { + "key": 80, + "doc_count": 556, + "app_proto": {"buckets": [{"key": "http", "doc_count": 556}]}, + }, + ] + }, + "unique_clients": {"value": 45}, + "bytes_to": {"value": 12300000000}, + "bytes_from": {"value": 500000}, + "time_range": { + "min_as_string": "2026-04-24T08:30:00.000Z", + "max_as_string": "2026-04-30T13:41:00.000Z", + }, +} + +MOCK_CONN_FROM_AGGS = { + "inbound_ports": { + "buckets": [ + {"key": 22, "doc_count": 150}, + {"key": 80, "doc_count": 30}, + ] + }, + "unique_targets": {"value": 15}, +} + +MOCK_SSL_TO_AGGS = { + "ja4s": {"buckets": [{"key": "abc123", "doc_count": 2100}]}, + "tls_versions": {"buckets": [{"key": "TLSv1.3", "doc_count": 2000}]}, + "subjects": {"buckets": [{"key": "CN=example.com", "doc_count": 2100}]}, + "issuers": {"buckets": [{"key": "CN=Let's Encrypt", "doc_count": 2100}]}, +} + +MOCK_HTTP_TO_AGGS = { + "server_headers": {"buckets": [{"key": "nginx", "doc_count": 500}]}, + "top_uris": { + "buckets": [ + {"key": "/", "doc_count": 300}, + {"key": "/api/v1", "doc_count": 200}, + ] + }, +} + +MOCK_SSH_FROM_AGGS = { + "server_versions": { + "buckets": [{"key": "SSH-2.0-OpenSSH_8.9", "doc_count": 50}], + "sum_other_doc_count": 0, + } +} + +MOCK_RDP_FROM_AGGS = {"cookies": {"buckets": [{"key": "admin", "doc_count": 10}]}} + + +class TestParsers: + """Verify parsers extract correct fields from mock aggregations.""" + + def test_parse_sensor_presence(self): + result = _parse_sensor_presence(MOCK_SENSOR_AGGS) + assert len(result["sensors"]) == 2 + assert result["sensors"][0]["sensor"] == "hedgehog-east" + assert result["total_records"] == 1423 + 892 + assert result["first_seen"] == "2026-04-24T08:30:00.000Z" + + def test_parse_reverse_dns(self): + result = _parse_reverse_dns(MOCK_RDNS_AGGS) + assert len(result["reverse_dns"]) == 2 + assert result["reverse_dns"][0]["domain"] == "example.com" + + def test_parse_conn_to(self): + result = _parse_conn_to(MOCK_CONN_TO_AGGS) + assert len(result["services"]) == 2 + assert result["services"][0]["port"] == 443 + assert result["services"][0]["app_proto"] == "ssl" + assert result["internal_client_count"] == 45 + assert result["bytes_to"] == 12300000000 + + def test_parse_conn_from(self): + result = _parse_conn_from(MOCK_CONN_FROM_AGGS) + assert len(result["inbound_ports_targeted"]) == 2 + assert result["internal_targets_count"] == 15 + + def test_parse_ssl_to(self): + result = _parse_ssl_to(MOCK_SSL_TO_AGGS) + assert len(result["ja4s_fingerprints"]) == 1 + assert result["tls_versions"][0]["version"] == "TLSv1.3" + assert "CN=Let's Encrypt" in result["ssl_issuers"] + + def test_parse_http_to(self): + result = _parse_http_to(MOCK_HTTP_TO_AGGS) + assert "nginx" in result["http_server_headers"] + assert len(result["http_top_uris"]) == 2 + + def test_parse_ssh_from(self): + result = _parse_ssh_from(MOCK_SSH_FROM_AGGS) + assert result["ssh_inbound"] is True + assert "SSH-2.0-OpenSSH_8.9" in result["ssh_server_versions"] + + def test_parse_ssh_from_empty(self): + result = _parse_ssh_from({}) + assert result["ssh_inbound"] is False + assert result["ssh_server_versions"] == [] + + def test_parse_rdp_from(self): + result = _parse_rdp_from(MOCK_RDP_FROM_AGGS) + assert result["rdp_inbound"] is True + assert "admin" in result["rdp_usernames"] + + def test_parse_rdp_from_empty(self): + result = _parse_rdp_from({}) + assert result["rdp_inbound"] is False + + +# --------------------------------------------------------------------------- +# Role classifier tests +# --------------------------------------------------------------------------- + + +def _make_profile(**kwargs) -> PublicIPProfile: + """Create a PublicIPProfile with overrides.""" + defaults = {"ip": "1.2.3.4", "time_range": "now-7d"} + defaults.update(kwargs) + return PublicIPProfile(**defaults) + + +class TestPublicRoleClassifier: + """Verify role classification for common scenarios.""" + + def test_web_server(self): + p = _make_profile( + services=[ + {"port": 443, "app_proto": "ssl", "count": 2100}, + {"port": 80, "app_proto": "http", "count": 556}, + ], + internal_client_count=20, + ssl_issuers=["CN=Let's Encrypt Authority X3"], + reverse_dns=[{"domain": "example.com", "count": 1200}], + ) + role, conf = classify_public_role(p) + assert role == "web_server" + assert conf >= 0.8 + + def test_scanner(self): + p = _make_profile( + inbound_ports_targeted=[{"port": i, "count": 5} for i in range(22, 35)], + internal_targets_count=50, + bytes_to=100, + bytes_from=100, + ) + role, conf = classify_public_role(p) + assert role == "scanner" + assert conf >= 0.7 + + def test_cdn_node(self): + p = _make_profile( + org={"name": "Cloudflare", "category": "cdn"}, + services=[{"port": 443, "app_proto": "ssl", "count": 5000}], + internal_client_count=100, + reverse_dns=[ + {"domain": "a.example.com", "count": 100}, + {"domain": "b.example.com", "count": 100}, + {"domain": "c.example.com", "count": 100}, + ], + ) + role, conf = classify_public_role(p) + assert role == "cdn_node" + assert conf >= 0.8 + + def test_dns_server(self): + p = _make_profile( + services=[{"port": 53, "app_proto": "dns", "count": 10000}], + internal_client_count=200, + ) + role, conf = classify_public_role(p) + assert role == "dns_server" + assert conf >= 0.7 + + def test_unknown_empty_profile(self): + p = _make_profile() + role, conf = classify_public_role(p) + assert role == "unknown" + assert conf == 0.0 + + +# --------------------------------------------------------------------------- +# Orchestrator integration test (mocked ES) +# --------------------------------------------------------------------------- + + +def _mock_query_opensearch(body, params): + """Return appropriate mock response based on query content.""" + must = body.get("query", {}).get("bool", {}).get("must", []) + for clause in must: + ds = clause.get("term", {}).get("event.dataset") + if ds == "dns": + return {"aggregations": MOCK_RDNS_AGGS} + if ds == "conn": + # Distinguish conn_to vs conn_from by IP field + for c in must: + if c.get("term", {}).get("destination.ip"): + return {"aggregations": MOCK_CONN_TO_AGGS} + if c.get("term", {}).get("source.ip"): + return {"aggregations": MOCK_CONN_FROM_AGGS} + if ds == "ssl": + return {"aggregations": MOCK_SSL_TO_AGGS} + if ds == "http": + return {"aggregations": MOCK_HTTP_TO_AGGS} + if ds == "ssh": + return {"aggregations": MOCK_SSH_FROM_AGGS} + if ds == "rdp": + return {"aggregations": MOCK_RDP_FROM_AGGS} + # Default: sensor presence query (no event.dataset filter) + return {"aggregations": MOCK_SENSOR_AGGS} + + +class TestProfilePublicIp: + """Integration test for profile_public_ip with mocked ES.""" + + @patch("src.profiler.public_ip_profiler.query_opensearch", side_effect=_mock_query_opensearch) + @patch("src.profiler.public_ip_profiler.lookup_org", return_value=None) + def test_full_profile(self, mock_org, mock_es): + profile = profile_public_ip("1.2.3.4", time_range="now-7d") + + assert profile.ip == "1.2.3.4" + assert len(profile.sensors) == 2 + assert profile.total_records == 2315 + assert len(profile.reverse_dns) == 2 + assert len(profile.services) == 2 + assert profile.internal_client_count == 45 + assert profile.ssh_inbound is True + assert profile.rdp_inbound is True + assert profile.role != "unknown" + assert profile.confidence > 0 + + @patch("src.profiler.public_ip_profiler.query_opensearch", return_value=None) + @patch("src.profiler.public_ip_profiler.lookup_org", return_value=None) + def test_empty_profile_no_errors(self, mock_org, mock_es): + """IP with zero records across all sensors → empty profile, no errors.""" + profile = profile_public_ip("5.6.7.8", time_range="now-7d") + + assert profile.ip == "5.6.7.8" + assert profile.sensors == [] + assert profile.total_records == 0 + assert profile.services == [] + assert profile.role == "unknown" + assert profile.confidence == 0.0 From 51c66d5742bc7c9f5126f6aef1c47ed8c424f69f Mon Sep 17 00:00:00 2001 From: liamadale Date: Thu, 30 Apr 2026 15:01:53 -0700 Subject: [PATCH 063/109] feat(opensearch): escalation indicator on investigate ticket cards Show a red border and 'Escalated by {name}' pill on ticket cards where is_escalated is true in the indexed ticket data. --- apps/opensearch_web/static/pisces.css | 4 ++++ .../templates/partials/investigate_tickets.html | 6 ++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/opensearch_web/static/pisces.css b/apps/opensearch_web/static/pisces.css index 7618fc6..99c084a 100644 --- a/apps/opensearch_web/static/pisces.css +++ b/apps/opensearch_web/static/pisces.css @@ -911,6 +911,10 @@ tr.detail-row td { border: 1px solid var(--outline); background: var(--surface-container-highest); } +.mantis-ticket-card.mantis-ticket-escalated { + border-color: var(--red); + border-width: 2px; +} .mantis-ticket-header { display: flex; align-items: center; diff --git a/apps/opensearch_web/templates/partials/investigate_tickets.html b/apps/opensearch_web/templates/partials/investigate_tickets.html index 15220bb..449c0e0 100644 --- a/apps/opensearch_web/templates/partials/investigate_tickets.html +++ b/apps/opensearch_web/templates/partials/investigate_tickets.html @@ -15,10 +15,11 @@

{% if src_tickets %}
{% for t in src_tickets %} -
+
#{{ t.id }} {{ t.summary }} + {% if t.is_escalated %} Escalated{% if t.escalated_by %} by {{ t.escalated_by }}{% endif %}{% endif %} {{ t.status or '—' }} {% if t.severity %}{{ t.severity }}{% endif %}
@@ -47,10 +48,11 @@

{% if dest_tickets %}
{% for t in dest_tickets %} -
+
#{{ t.id }} {{ t.summary }} + {% if t.is_escalated %} Escalated{% if t.escalated_by %} by {{ t.escalated_by }}{% endif %}{% endif %} {{ t.status or '—' }} {% if t.severity %}{{ t.severity }}{% endif %}
From 693b2c1bc17b50f9da7b3f8b68877e8151669ebd Mon Sep 17 00:00:00 2001 From: liamadale Date: Thu, 30 Apr 2026 15:16:39 -0700 Subject: [PATCH 064/109] feat(opensearch): move Investigate button to global search bar Remove Investigate button from IP pivot page header. Add it next to the Search button in the global search bar so it is accessible from any page. Reads Src IP and Dst IP from the search bar fields and shows a validation notice if either is empty. --- apps/opensearch_web/templates/base.html | 18 ++++++++++++++++++ apps/opensearch_web/templates/ip_pivot.html | 12 ------------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/apps/opensearch_web/templates/base.html b/apps/opensearch_web/templates/base.html index f835028..b8f3e79 100644 --- a/apps/opensearch_web/templates/base.html +++ b/apps/opensearch_web/templates/base.html @@ -147,6 +147,10 @@ Search + +
@@ -101,13 +98,4 @@

IP Pivot: {{ ip }}

{% endif %}
{% endfor %} - - {% endblock content %} From f7946dfc2bc461a1878d7b5940f61a1f05d8517e Mon Sep 17 00:00:00 2001 From: liamadale Date: Thu, 30 Apr 2026 15:28:24 -0700 Subject: [PATCH 065/109] style(opensearch): tinted pill styling for src/dst headers in detail panel Match the search bar's red-tinted (source) and blue-tinted (destination) pill styling on the Source/Destination headers in the record detail popout panel. --- apps/opensearch_web/static/pisces.css | 14 ++++++++++++++ .../templates/partials/record_detail.html | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/apps/opensearch_web/static/pisces.css b/apps/opensearch_web/static/pisces.css index 99c084a..a183d07 100644 --- a/apps/opensearch_web/static/pisces.css +++ b/apps/opensearch_web/static/pisces.css @@ -804,6 +804,20 @@ tr.detail-row td { align-items: center; gap: 6px; } +.enrich-col-header.enrich-col-src, +.enrich-col-header.enrich-col-dst { + padding: 4px 10px; + border-radius: 6px; + border: 1px solid var(--outline); +} +.enrich-col-header.enrich-col-src { + border-color: color-mix(in srgb, var(--red) 25%, var(--outline)); + background: color-mix(in srgb, var(--red) 4%, var(--surface-container)); +} +.enrich-col-header.enrich-col-dst { + border-color: color-mix(in srgb, var(--primary) 25%, var(--outline)); + background: color-mix(in srgb, var(--primary) 4%, var(--surface-container)); +} /* Legacy enrich triggers */ .enrich-triggers { display: flex; flex-wrap: wrap; gap: 0.4rem; margin-top: 0.6rem; } diff --git a/apps/opensearch_web/templates/partials/record_detail.html b/apps/opensearch_web/templates/partials/record_detail.html index cbfbf26..63c18f4 100644 --- a/apps/opensearch_web/templates/partials/record_detail.html +++ b/apps/opensearch_web/templates/partials/record_detail.html @@ -90,7 +90,7 @@
{% if src_ip %} -
+
Source — {{ src_ip }} @@ -133,7 +133,7 @@
{% if dest_ip %} -
+
Destination — {{ dest_ip }} From 535aaafa626ceba7cdd6c92322dd7e463733c9ae Mon Sep 17 00:00:00 2001 From: liamadale Date: Tue, 5 May 2026 11:18:31 -0700 Subject: [PATCH 066/109] feat(dashboard): add per-sensor log count time series chart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a new aggregation (agg_logs_by_sensor_over_time) that buckets total log counts per sensor using a terms → date_histogram pipeline. The resulting aligned time series is rendered as a multi-line ECharts chart with a scrollable legend sorted by busiest sensor, giving analysts a quick view of which sensors are generating the most traffic and whether any sensor has gone silent or spiked. --- apps/dashboard_web/opensearch/__init__.py | 2 + apps/dashboard_web/opensearch/aggregations.py | 62 +++++++++++++ .../templates/opensearch/section.html | 93 ++++++++++++++++++- 3 files changed, 152 insertions(+), 5 deletions(-) diff --git a/apps/dashboard_web/opensearch/__init__.py b/apps/dashboard_web/opensearch/__init__.py index bf7fd35..ff6abdf 100644 --- a/apps/dashboard_web/opensearch/__init__.py +++ b/apps/dashboard_web/opensearch/__init__.py @@ -3,6 +3,7 @@ from apps.dashboard_web import cache as dcache from apps.dashboard_web.opensearch.aggregations import ( agg_conn_volume_over_time, + agg_logs_by_sensor_over_time, agg_notice_over_time, agg_opensearch_sensors, agg_opensearch_top_ips, @@ -30,6 +31,7 @@ def section(): "conn_over_time": agg_conn_volume_over_time(time_range, sensors), "sensors": agg_opensearch_sensors(time_range), "top_ips": agg_opensearch_top_ips(time_range, sensors), + "sensor_trend": agg_logs_by_sensor_over_time(time_range, sensors), } except Exception as exc: data = {"error": str(exc)} diff --git a/apps/dashboard_web/opensearch/aggregations.py b/apps/dashboard_web/opensearch/aggregations.py index 00e336e..bc24861 100644 --- a/apps/dashboard_web/opensearch/aggregations.py +++ b/apps/dashboard_web/opensearch/aggregations.py @@ -244,6 +244,68 @@ def agg_conn_volume_over_time(time_range: str, sensors: list | None = None) -> d } +def agg_logs_by_sensor_over_time(time_range: str, sensors: list | None = None) -> dict: + """Total log count per sensor as aligned time series (terms → date_histogram).""" + interval = _interval_for_range(time_range) + body, params = build_base_query( + must_not=[], + extra_must=[], + source_fields=[], + limit=0, + time_range=time_range, + sensors=sensors, + datasets=["all"], + public_only=False, + src_ip_filter=None, + direction=None, + ) + body["size"] = 0 + body.pop("sort", None) + body.pop("_source", None) + body["aggs"] = { + "by_sensor": { + "terms": {"field": "host.name", "size": 50, "order": {"_count": "desc"}}, + "aggs": { + "over_time": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": interval, + "min_doc_count": 0, + } + } + }, + } + } + raw = query_opensearch(body, params) + sensor_buckets = ( + raw.get("aggregations", {}).get("by_sensor", {}).get("buckets", []) if raw else [] + ) + + # Collect all unique timestamps in order across all sensors + all_ts: dict[str, None] = {} + for sb in sensor_buckets: + for tb in sb.get("over_time", {}).get("buckets", []): + all_ts[tb["key_as_string"]] = None + timestamps = list(all_ts.keys()) + + # Build per-sensor series aligned to the shared timestamp list + series = [] + for sb in sensor_buckets: + ts_map = { + tb["key_as_string"]: tb["doc_count"] + for tb in sb.get("over_time", {}).get("buckets", []) + } + series.append( + { + "sensor": sb["key"], + "counts": [ts_map.get(t, 0) for t in timestamps], + "total": sb["doc_count"], + } + ) + + return {"timestamps": timestamps, "series": series, "interval": interval} + + def agg_new_ips_delta(time_range: str, sensors: list | None = None) -> dict: """Compare unique source IPs in the current window vs the previous window. diff --git a/apps/dashboard_web/opensearch/templates/opensearch/section.html b/apps/dashboard_web/opensearch/templates/opensearch/section.html index c2bd447..c651f33 100644 --- a/apps/dashboard_web/opensearch/templates/opensearch/section.html +++ b/apps/dashboard_web/opensearch/templates/opensearch/section.html @@ -60,15 +60,25 @@ {% endif %}
+
+
Log Count by Sensor ({{ data.sensor_trend.interval }} intervals)
+ {% if not data.sensor_trend.timestamps %} +
No sensor data in this window
+ {% else %} +
+ {% endif %} +
+
From 7bf64b9e0c5215a79a371214e72a2d79885f8e1b Mon Sep 17 00:00:00 2001 From: liamadale Date: Tue, 5 May 2026 12:01:35 -0700 Subject: [PATCH 067/109] feat(opensearch): replace auth history with search all logs section Remove the Authentication History panel from the investigate view and add a Search All Logs section below the Timeline. The new section lazy-loads per-module record counts in parallel via a new /api/investigate/log_counts endpoint and renders pill-shaped buttons grouped by log category, each with a count badge. Clicking a button navigates to the log search view pre-filtered by src_ip. --- apps/opensearch_web/app.py | 44 ++++++++++++++++++ apps/opensearch_web/static/pisces.css | 46 +++++++++++++++++++ .../opensearch_web/templates/investigate.html | 30 ++++++------ .../partials/investigate_log_counts.html | 22 +++++++++ 4 files changed, 127 insertions(+), 15 deletions(-) create mode 100644 apps/opensearch_web/templates/partials/investigate_log_counts.html diff --git a/apps/opensearch_web/app.py b/apps/opensearch_web/app.py index 2d2a8b7..ab1fa51 100644 --- a/apps/opensearch_web/app.py +++ b/apps/opensearch_web/app.py @@ -933,4 +933,48 @@ def _chain() -> list[dict]: errors=errors, ) + # ------------------------------------------------------------------ + # GET /api/investigate/log_counts — HTMX: per-module record counts for src_ip + # ------------------------------------------------------------------ + @app.route("/api/investigate/log_counts") + def api_investigate_log_counts(): + from concurrent.futures import ThreadPoolExecutor + from concurrent.futures import as_completed as _as_completed + + src_ip = request.args.get("src_ip", "") + sensor = request.args.get("sensor", "all") + time_range = request.args.get("time_range", "now-24h") + + search_params: dict = { + "time_range": time_range, + "time_from": None, + "time_to": None, + "sensor": sensor, + "limit": 500, + "public_only": False, + "src_ip": src_ip or None, + "dest_ip": None, + "direction": None, + "no_filters": False, + "use_cache": False, + } + + counts: dict[str, int] = {} + with ThreadPoolExecutor(max_workers=len(MODULES)) as pool: + futures = {pool.submit(cached_run_query, lt, dict(search_params)): lt for lt in MODULES} + for fut in _as_completed(futures): + lt = futures[fut] + try: + counts[lt] = len(fut.result()) + except Exception: + counts[lt] = 0 + + return render_template( + "partials/investigate_log_counts.html", + counts=counts, + src_ip=src_ip, + sensor=sensor, + time_range=time_range, + ) + return app diff --git a/apps/opensearch_web/static/pisces.css b/apps/opensearch_web/static/pisces.css index a183d07..e8160ba 100644 --- a/apps/opensearch_web/static/pisces.css +++ b/apps/opensearch_web/static/pisces.css @@ -1621,3 +1621,49 @@ tr.detail-row td { .investigate-tickets-grid { grid-template-columns: 1fr; } .investigate-profiles-skeleton { grid-template-columns: 1fr; } } + +/* ── Investigate: log search buttons ─────────────────────────────── */ +.log-count-group { margin-bottom: 1.2rem; } +.log-count-group:last-child { margin-bottom: 0; } +.log-count-group-label { + font-size: 0.7rem; + text-transform: uppercase; + letter-spacing: 0.08em; + color: var(--on-surface-dim); + margin-bottom: 0.4rem; + display: flex; + align-items: center; + gap: 0.35rem; +} +.log-count-btns { display: flex; flex-wrap: wrap; gap: 0.4rem; } +.log-count-btn { + display: inline-flex; + align-items: center; + gap: 0.4rem; + padding: 0.3rem 0.7rem; + border-radius: var(--radius-pill); + background: var(--surface-container-high); + color: var(--on-surface); + text-decoration: none; + font-size: 0.8rem; + border: 1px solid var(--outline); + transition: background 0.15s, border-color 0.15s; +} +.log-count-btn:hover { + background: var(--surface-container-highest); + border-color: var(--primary); + color: var(--on-surface); +} +.log-count-btn--empty { opacity: 0.38; } +.log-count-badge { + background: var(--primary); + color: #fff; + border-radius: var(--radius-pill); + padding: 0 0.45rem; + font-size: 0.7rem; + font-weight: 600; + min-width: 1.35rem; + text-align: center; + line-height: 1.5; +} +.log-count-btn--empty .log-count-badge { background: var(--on-surface-dim); } diff --git a/apps/opensearch_web/templates/investigate.html b/apps/opensearch_web/templates/investigate.html index 2be95d5..5bc44d0 100644 --- a/apps/opensearch_web/templates/investigate.html +++ b/apps/opensearch_web/templates/investigate.html @@ -41,21 +41,6 @@

Device Profiles

-{# ── Authentication History ───────────────────────────────────────── #} -
-
-

Authentication History

- Kerberos + NTLM between {{ src_ip }} ↔ {{ dest_ip }} -
-
-
-
-
-
-
- {# ── Attack Chain ─────────────────────────────────────────────────── #}
@@ -130,4 +115,19 @@

Timeline

+ +{# ── Search All Logs ──────────────────────────────────────────────── #} +
+
+

Search All Logs

+ Browse log types filtered by {{ src_ip }} +
+
+
+
+
+
+
{% endblock content %} diff --git a/apps/opensearch_web/templates/partials/investigate_log_counts.html b/apps/opensearch_web/templates/partials/investigate_log_counts.html new file mode 100644 index 0000000..6b51bd6 --- /dev/null +++ b/apps/opensearch_web/templates/partials/investigate_log_counts.html @@ -0,0 +1,22 @@ +{% for cat in category_order %} + {% set lts = modules_by_category.get(cat, []) %} + {% if lts %} +
+
+ + {{ category_labels[cat] }} +
+
+ {% for lt in lts %} + {% set cnt = counts.get(lt, 0) %} + + + {{ lt.replace('_', ' ').title() }} + {{ cnt }} + + {% endfor %} +
+
+ {% endif %} +{% endfor %} From 29c17ce19e84245a9dab0ab06d80157b1581990e Mon Sep 17 00:00:00 2001 From: liamadale Date: Wed, 6 May 2026 09:55:09 -0700 Subject: [PATCH 068/109] feat(threat-model): add batch private IP profiling pipeline Add a --profile-private-ips flag to mantis_threat_model.py that batch-profiles all RFC1918 IPs from the infra registry using profile_device() (11 parallel OpenSearch aggregations per IP). Results are written incrementally to a sidecar file (data/tickets/enriched/private_ip_profiles.json). Already-profiled IPs are skipped on re-runs unless --profile-force is set. Writes are atomic (rename from .tmp) to avoid partial reads by the web app. New CLI flags: --profile-output, --profile-time-range, --profile-sensor, --profile-workers, --profile-force. --- src/mantis/mantis_threat_model.py | 55 +++++++ src/mantis/threat_model/__init__.py | 2 + .../threat_model/private_ip_profiles.py | 139 ++++++++++++++++++ 3 files changed, 196 insertions(+) create mode 100644 src/mantis/threat_model/private_ip_profiles.py diff --git a/src/mantis/mantis_threat_model.py b/src/mantis/mantis_threat_model.py index 00b3021..1dbd70d 100644 --- a/src/mantis/mantis_threat_model.py +++ b/src/mantis/mantis_threat_model.py @@ -26,6 +26,7 @@ generate_infra_registry, generate_threat_db, generate_undetermined_registry, + profile_private_ips, ) from src.mantis.threat_model._shared import _PROGRESS_INTERVAL, console from src.mantis.ticket_enrichment import ( @@ -298,6 +299,49 @@ def main() -> None: "all registries are regenerated with the updated signals." ), ) + parser.add_argument( + "--profile-private-ips", + action="store_true", + default=False, + dest="profile_private_ips", + help=( + "Batch-profile private IPs from the infra registry using device_profiler. " + "Results are written to --profile-output (default: private_ip_profiles.json). " + "Already-profiled IPs are skipped unless --profile-force is set." + ), + ) + parser.add_argument( + "--profile-output", + default=os.path.join(_BASE, "data", "tickets", "enriched", "private_ip_profiles.json"), + dest="profile_output", + help="Output path for the device profile sidecar (JSON)", + ) + parser.add_argument( + "--profile-time-range", + default="now-7d", + dest="profile_time_range", + help="Elasticsearch date-math range for profiling queries (default: now-7d)", + ) + parser.add_argument( + "--profile-sensor", + default="all", + dest="profile_sensor", + help="Sensor hostname to profile against, or 'all' for cross-sensor aggregate (default: all)", # noqa: E501 + ) + parser.add_argument( + "--profile-workers", + type=int, + default=5, + dest="profile_workers", + help="Number of concurrent IP profilers (each runs 11 parallel ES queries, default: 5)", + ) + parser.add_argument( + "--profile-force", + action="store_true", + default=False, + dest="profile_force", + help="Re-profile IPs that already exist in the sidecar", + ) args = parser.parse_args() if not os.path.exists(args.input): @@ -333,6 +377,17 @@ def main() -> None: generate_dns_resolver_registry(tickets, args.dns_output) generate_undetermined_registry(tickets, args.undetermined_output, provider) + if args.profile_private_ips: + console.print("\n[bold cyan]Private IP profiling pass (--profile-private-ips)[/bold cyan]") + profile_private_ips( + infra_path=args.infra_output, + output_path=args.profile_output, + time_range=args.profile_time_range, + sensor=args.profile_sensor, + workers=args.profile_workers, + force=args.profile_force, + ) + if args.enrich: console.print("\n[bold cyan]API enrichment pass (--enrich)[/bold cyan]") enrich_undetermined_ips(args.undetermined_output, provider) diff --git a/src/mantis/threat_model/__init__.py b/src/mantis/threat_model/__init__.py index 7402292..2448527 100644 --- a/src/mantis/threat_model/__init__.py +++ b/src/mantis/threat_model/__init__.py @@ -12,6 +12,7 @@ from src.mantis.threat_model.false_positives import generate_fp_candidates from src.mantis.threat_model.infrastructure import generate_infra_registry from src.mantis.threat_model.malicious import generate_threat_db +from src.mantis.threat_model.private_ip_profiles import profile_private_ips from src.mantis.threat_model.undetermined import ( enrich_undetermined_ips, generate_undetermined_registry, @@ -24,4 +25,5 @@ "generate_threat_db", "generate_undetermined_registry", "enrich_undetermined_ips", + "profile_private_ips", ] diff --git a/src/mantis/threat_model/private_ip_profiles.py b/src/mantis/threat_model/private_ip_profiles.py new file mode 100644 index 0000000..616cc60 --- /dev/null +++ b/src/mantis/threat_model/private_ip_profiles.py @@ -0,0 +1,139 @@ +"""Batch device profiling for private IPs in the infrastructure registry. + +Reads the infra registry (known_infra_ips.json), filters to RFC1918 addresses, +and runs profile_device() for each using a thread pool. Results are written to +a sidecar file (private_ip_profiles.json) that the web app loads at startup. + +Supports incremental re-runs: IPs already in the sidecar are skipped unless +force=True is passed. +""" + +from __future__ import annotations + +import ipaddress +import os +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import asdict +from datetime import datetime + +from src.mantis.threat_model._shared import console +from src.utils.cache import dump_json, load_json + +_PROGRESS_INTERVAL = 50 + + +def _is_rfc1918(ip: str) -> bool: + """Return True for RFC1918/link-local/loopback addresses that profile_device() accepts.""" + try: + addr = ipaddress.ip_address(ip) + return addr.is_private and not addr.is_unspecified + except ValueError: + return False + + +def _profile_one( + ip: str, + time_range: str, + sensor: str, +) -> dict | None: + """Run profile_device() for one IP, returning a serialisable dict or None on error.""" + try: + from src.profiler.device_profiler import profile_device + + profile = profile_device(ip, time_range=time_range, sensor=sensor) + record = asdict(profile) + # orjson requires string dict keys — port distributions use int keys + record["dest_port_distribution"] = { + str(k): v for k, v in record["dest_port_distribution"].items() + } + record["profiled_at"] = datetime.now().isoformat(timespec="seconds") + return record + except Exception as exc: + console.print(f"[yellow] profile skipped {ip}: {exc}[/yellow]") + return None + + +def profile_private_ips( + infra_path: str, + output_path: str, + time_range: str = "now-7d", + sensor: str = "all", + workers: int = 5, + force: bool = False, +) -> None: + """Batch-profile private IPs from the infra registry and write a sidecar file. + + Args: + infra_path: Path to known_infra_ips.json. + output_path: Destination path for private_ip_profiles.json. + time_range: Elasticsearch date-math range string (default: now-7d). + sensor: Sensor hostname or "all" for cross-sensor aggregate. + workers: Number of concurrent IP profilers (each spawns 11 ES queries internally). + force: Re-profile IPs already present in the sidecar. + """ + if not os.path.exists(infra_path): + console.print(f"[red]Infra registry not found: {infra_path}[/red]") + console.print("[dim]Run the threat model generator first.[/dim]") + return + + infra: list[dict] = load_json(infra_path) # type: ignore[assignment] + private_ips = [r["ip"] for r in infra if _is_rfc1918(r["ip"])] + + console.print( + f"[dim]Found {len(private_ips):,} private IPs in infra registry " + f"(total infra records: {len(infra):,})[/dim]" + ) + + # Load existing sidecar for incremental skip. + existing: dict[str, dict] = {} + if os.path.exists(output_path): + loaded: list[dict] = load_json(output_path) # type: ignore[assignment] + existing = {r["ip"]: r for r in loaded if "ip" in r} + console.print(f"[dim]Existing sidecar: {len(existing):,} profiles loaded[/dim]") + + to_profile = [ip for ip in private_ips if force or ip not in existing] + skipped = len(private_ips) - len(to_profile) + + if skipped: + console.print( + f"[dim]Skipping {skipped:,} already-profiled IPs (use --profile-force to re-run)[/dim]" + ) + + if not to_profile: + console.print("[green]All private IPs already profiled — nothing to do.[/green]") + return + + console.print( + f"[bold]Profiling {len(to_profile):,} private IPs[/bold]" + f" workers={workers} sensor={sensor} range={time_range}" + ) + + done = 0 + errors = 0 + + with ThreadPoolExecutor(max_workers=workers) as pool: + futures = {pool.submit(_profile_one, ip, time_range, sensor): ip for ip in to_profile} + for future in as_completed(futures): + ip = futures[future] + result = future.result() + done += 1 + if result is not None: + existing[ip] = result + else: + errors += 1 + + if done % _PROGRESS_INTERVAL == 0 or done == len(to_profile): + console.print( + f"[dim] profiled: {done:,}/{len(to_profile):,} errors: {errors}[/dim]" + ) + + # Write sidecar atomically. + records = sorted(existing.values(), key=lambda r: r["ip"]) + tmp = output_path + ".tmp" + dump_json(records, tmp) + os.rename(tmp, output_path) + + console.print( + f"[green]Done — {len(records):,} profiles written to {output_path}[/green]" + f" ({errors} errors skipped)" + ) From 6948b10ab5036a073c8d430739c15cd1674945c3 Mon Sep 17 00:00:00 2001 From: liamadale Date: Wed, 6 May 2026 09:55:25 -0700 Subject: [PATCH 069/109] feat(threat-model): display device profiles for internal IPs in web UI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire the private IP profile sidecar into the threat model web app: - data.py loads private_ip_profiles.json at startup and exposes PROFILES_BY_IP; _infra_row() attaches hostname, role, and OS hints for use in the table - infra_rows.html shows hostname and role badge inline in the IP column - threat_card.html adds an INTERNAL verdict label and renders a device profile body section for infra IPs; if no sidecar profile exists it triggers a live HTMX load via the new /api/ip//profile route - app.py adds GET /api/ip//profile — runs profile_device() on demand for private IPs and returns the device_profile_card.html partial - device_profile_card.html (new partial) renders identity, OS, network stats, inbound services, risk signals, and top DNS domains in a two- column grid --- apps/threat_model/app.py | 38 +++++- apps/threat_model/data.py | 7 + .../partials/device_profile_card.html | 120 ++++++++++++++++++ .../templates/partials/infra_rows.html | 6 +- .../templates/partials/threat_card.html | 18 +++ 5 files changed, 187 insertions(+), 2 deletions(-) create mode 100644 apps/threat_model/templates/partials/device_profile_card.html diff --git a/apps/threat_model/app.py b/apps/threat_model/app.py index 99a25e3..9c524ed 100644 --- a/apps/threat_model/app.py +++ b/apps/threat_model/app.py @@ -15,6 +15,7 @@ INFRA_ROWS, MALICIOUS_BY_IP, MALICIOUS_ROWS, + PROFILES_BY_IP, TICKETS_BY_ID, UNDETERMINED_ROWS, _fp_row, @@ -203,12 +204,14 @@ def api_search(): ticket_slice = tickets[:TICKETS_PER_CARD_PAGE] raw_mal = MALICIOUS_BY_IP.get(ip) raw_fp = FP_BY_IP.get(ip) + verdict = classify_ip(ip) return render_template( "partials/threat_card.html", ip=ip, - verdict=classify_ip(ip), + verdict=verdict, malicious=_malicious_row(raw_mal) if raw_mal else None, fp=_fp_row(raw_fp) if raw_fp else None, + device_profile=PROFILES_BY_IP.get(ip) if verdict == "infra" else None, tickets=ticket_slice, ticket_page=1, ticket_pages=ticket_pages, @@ -237,12 +240,45 @@ def api_ip_card(ip: str): verdict=verdict, malicious=_malicious_row(raw_mal) if raw_mal and verdict != "fp" else None, fp=_fp_row(raw_fp) if raw_fp else None, + device_profile=PROFILES_BY_IP.get(ip) if verdict == "infra" else None, tickets=ticket_slice, ticket_page=1, ticket_pages=ticket_pages, ticket_total=len(tickets), ) + # ------------------------------------------------------------------ + # GET /api/ip//profile — HTMX: live device profile for private IPs + # ------------------------------------------------------------------ + @app.route("/api/ip//profile") + def api_ip_profile(ip: str): + from dataclasses import asdict + + from src.profiler.device_profiler import profile_device + from src.querier.zeek_modules.base import is_private + + if not is_private(ip): + return ( + "

Not a private IP.

" + ), 400 + + time_range = request.args.get("time_range", "now-7d") + sensor = request.args.get("sensor", "all") + + try: + profile = profile_device(ip, time_range=time_range, sensor=sensor) + dp = asdict(profile) + dp["dest_port_distribution"] = { + str(k): v for k, v in dp["dest_port_distribution"].items() + } + except Exception as exc: + return ( + f"

" + f"Profile unavailable: {exc}

" + ), 200 + + return render_template("partials/device_profile_card.html", device_profile=dp) + # ------------------------------------------------------------------ # GET /api/ip//tickets?page=N — HTMX: paginate ticket list # ------------------------------------------------------------------ diff --git a/apps/threat_model/data.py b/apps/threat_model/data.py index 6c414b2..2108986 100644 --- a/apps/threat_model/data.py +++ b/apps/threat_model/data.py @@ -57,6 +57,7 @@ def _load_optional(name: str) -> list: _raw_infra = _load_optional("enriched/known_infra_ips.json") _raw_dns_resolvers = _load_optional("enriched/dns_resolver_ips.json") _raw_undetermined = _load_optional("enriched/undetermined_ips.json") +_raw_profiles = _load_optional("enriched/private_ip_profiles.json") # --------------------------------------------------------------------------- # Indices @@ -72,6 +73,7 @@ def _load_optional(name: str) -> list: FP_BY_IP: dict[str, dict] = {r["ip"]: r for r in _raw_fp} INFRA_BY_IP: dict[str, dict] = {r["ip"]: r for r in _raw_infra} UNDETERMINED_BY_IP: dict[str, dict] = {r["ip"]: r for r in _raw_undetermined} +PROFILES_BY_IP: dict[str, dict] = {r["ip"]: r for r in _raw_profiles} # Build DNS resolver index from enriched file; fall back to known-list entries # for any resolver not yet in the enriched file. @@ -199,6 +201,7 @@ def _fp_row(r: dict) -> dict: def _infra_row(r: dict) -> dict: _org = r.get("org") or {} org = _org if isinstance(_org, dict) else {} + profile = PROFILES_BY_IP.get(r["ip"]) return { "ip": r["ip"], "org_name": org.get("name") or "—", @@ -210,6 +213,10 @@ def _infra_row(r: dict) -> dict: "ticket_count": len(r.get("ticket_ids") or []), "protocols_str": ", ".join(r.get("protocols_seen") or []), "attacks_count": len(r.get("attacks_against") or []), + "has_profile": profile is not None, + "profile_hostname": profile.get("hostname") if profile else None, + "profile_role": profile.get("role") if profile else None, + "profile_os": profile.get("os_family") if profile else None, } diff --git a/apps/threat_model/templates/partials/device_profile_card.html b/apps/threat_model/templates/partials/device_profile_card.html new file mode 100644 index 0000000..f42934f --- /dev/null +++ b/apps/threat_model/templates/partials/device_profile_card.html @@ -0,0 +1,120 @@ +{% set dp = device_profile %} +
+ + {# ── Left column ── #} +
+ + {# Identity #} + {% if dp.hostname or dp.dhcp_hostname or dp.mac or dp.users %} +
+ Hostname + {{ dp.hostname or dp.dhcp_hostname or '—' }}{% if dp.ad_domain %}.{{ dp.ad_domain }}{% endif %} +
+ {% if dp.mac %} +
+ MAC + {{ dp.mac }} +
+ {% endif %} + {% if dp.users %} +
+ Users + {{ dp.users | join(', ') }} +
+ {% endif %} + {% endif %} + + {# Classification #} +
+ Role + + {% if dp.role and dp.role != 'unknown' %} + {{ dp.role | replace('_', ' ') | title }} + {{ (dp.confidence * 100) | int }}% + {% else %} + — + {% endif %} + +
+
+ OS + {{ dp.os_family or '—' }} +
+ {% if dp.software %} +
+ Software + {{ dp.software | join(', ') }} +
+ {% endif %} + + {# Network summary #} +
+ Bytes Out + {{ dp.bytes_sent | filesizeformat if dp.bytes_sent else '—' }} +
+
+ Bytes In + {{ dp.bytes_received | filesizeformat if dp.bytes_received else '—' }} +
+
+ Unique Dests + {{ dp.unique_dest_count or '—' }} +
+ +
+ + {# ── Right column ── #} +
+ + {# Inbound services #} + {% if dp.inbound_services %} +
+ Inbound Services + + {% for svc in dp.inbound_services[:6] %} + + + + + {% endfor %} +
{{ svc.port }}/{{ svc.app_proto or 'tcp' }}{{ svc.count }} conns
+
+ {% endif %} + + {# Risk signals #} + {% set risk = [] %} + {% if dp.rdp_inbound %}{% set _ = risk.append('RDP inbound' + (' (' + dp.rdp_usernames[:3]|join(', ') + ')' if dp.rdp_usernames else '')) %}{% endif %} + {% if dp.ssh_inbound %}{% set _ = risk.append('SSH inbound') %}{% endif %} + {% if dp.admin_targets %}{% set _ = risk.append('RDP admin → ' + dp.admin_targets[:3]|join(', ')) %}{% endif %} + {% if dp.ssh_admin_targets %}{% set _ = risk.append('SSH admin → ' + dp.ssh_admin_targets[:3]|join(', ')) %}{% endif %} + {% if risk %} +
+ Risk Signals + + {% for r in risk %}{{ r }}{% endfor %} + +
+ {% endif %} + + {# Top DNS domains #} + {% if dp.dns_top_domains %} +
+ Top DNS Domains +
    + {% for d in dp.dns_top_domains[:5] %} +
  • {{ d.domain }} ×{{ d.count }}
    • + {% endfor %} +
+
+ {% endif %} + +
+ +
+ +{# Footer: profiled-at metadata #} +
+ Profiled {{ dp.profiled_at[:10] if dp.profiled_at else '?' }} +  ·  sensor={{ dp.sensor }} +  ·  range={{ dp.time_range }} +
diff --git a/apps/threat_model/templates/partials/infra_rows.html b/apps/threat_model/templates/partials/infra_rows.html index 4caca81..5af3103 100644 --- a/apps/threat_model/templates/partials/infra_rows.html +++ b/apps/threat_model/templates/partials/infra_rows.html @@ -1,6 +1,10 @@ {% for r in rows %} - {{ r.ip }} + + {{ r.ip }} + {% if r.profile_hostname %} · {{ r.profile_hostname }}{% endif %} + {% if r.profile_role and r.profile_role != 'unknown' %}{{ r.profile_role | replace('_',' ') | title }}{% endif %} + {% if r.org_icon %}{% endif %}{{ r.org_name }} {{ r.org_category|replace('_',' ')|title }} {{ r.actor|replace('_',' ')|title if r.actor else '—' }} diff --git a/apps/threat_model/templates/partials/threat_card.html b/apps/threat_model/templates/partials/threat_card.html index 3f4d3b7..f0e517a 100644 --- a/apps/threat_model/templates/partials/threat_card.html +++ b/apps/threat_model/templates/partials/threat_card.html @@ -8,6 +8,8 @@ FALSE POSITIVE {% elif verdict == 'observed' %} OBSERVED + {% elif verdict == 'infra' %} + INTERNAL {% else %} UNKNOWN {% endif %} @@ -26,6 +28,8 @@ score: {{ fp.score }} {% elif verdict == 'observed' %} {{ ticket_total }} ticket{{ 's' if ticket_total != 1 else '' }} + {% elif verdict == 'infra' %} + {% if ticket_total > 0 %}{{ ticket_total }} ticket{{ 's' if ticket_total != 1 else '' }}{% endif %} {% else %} not in database {% endif %} @@ -97,6 +101,20 @@ + {% elif verdict == 'infra' %} +
+ {% if device_profile %} + {% include "partials/device_profile_card.html" %} + {% else %} +
+

+ Loading device profile… +

+
+ {% endif %} +
{% endif %} {# Ticket section #} From 82e4125222443036cbbf5bc9b180100722abb547 Mon Sep 17 00:00:00 2001 From: liamadale Date: Mon, 11 May 2026 09:39:41 -0700 Subject: [PATCH 070/109] refactor(querier): replace silent None returns with typed exceptions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce OpenSearchConnectionError and OpenSearchAuthError in base.py. query_opensearch() now raises instead of returning None on connectivity failure, credential misconfiguration, or non-OK HTTP status. All call sites in the query layer (run_query, list_sensors, list_log_types, list_indices, match_all_sample, pe, x509, fleet_scanner, opensearch_querier CLI) are updated to either catch and print to console or let the exception propagate to the caller. Removes the raise_on_error search_params escape hatch — exceptions now propagate unconditionally, which is the correct default for a library function. --- src/profiler/fleet_scanner.py | 13 ++++-- src/querier/opensearch_querier.py | 8 +++- src/querier/zeek_modules/base.py | 72 ++++++++++++++++++++----------- src/querier/zeek_modules/pe.py | 15 +++++-- src/querier/zeek_modules/x509.py | 16 +++++-- 5 files changed, 88 insertions(+), 36 deletions(-) diff --git a/src/profiler/fleet_scanner.py b/src/profiler/fleet_scanner.py index 08a96b6..127a775 100644 --- a/src/profiler/fleet_scanner.py +++ b/src/profiler/fleet_scanner.py @@ -19,7 +19,13 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) -from src.querier.zeek_modules.base import INDEX, is_private, query_opensearch +from src.querier.zeek_modules.base import ( + INDEX, + OpenSearchAuthError, + OpenSearchConnectionError, + is_private, + query_opensearch, +) _PARAMS = {"path": f"{INDEX}/_search", "method": "POST"} @@ -128,8 +134,9 @@ def scan_fleet( Returns: List of DeviceClusters sorted by size descending. """ - raw = query_opensearch(_fleet_ja4_query(sensor, time_range), _PARAMS) - if not raw: + try: + raw = query_opensearch(_fleet_ja4_query(sensor, time_range), _PARAMS) + except (OpenSearchConnectionError, OpenSearchAuthError): return [] buckets = raw.get("aggregations", {}).get("per_ip", {}).get("buckets", []) diff --git a/src/querier/opensearch_querier.py b/src/querier/opensearch_querier.py index fabd4ad..dd5b574 100755 --- a/src/querier/opensearch_querier.py +++ b/src/querier/opensearch_querier.py @@ -26,6 +26,8 @@ from src.querier.zeek_modules.base import ( FILTERS_DIR, TIME_RANGES, + OpenSearchAuthError, + OpenSearchConnectionError, build_base_query, console, interactive_loop, @@ -207,7 +209,11 @@ def main() -> None: return # 7. Execute query - records = run_query(module, search_params) + try: + records = run_query(module, search_params) + except (OpenSearchConnectionError, OpenSearchAuthError) as exc: + console.print(f"[red]{exc}[/red]") + return if not records: console.print( "[yellow]No records returned. Filters may be too aggressive" diff --git a/src/querier/zeek_modules/base.py b/src/querier/zeek_modules/base.py index 295c95f..61ba9a6 100644 --- a/src/querier/zeek_modules/base.py +++ b/src/querier/zeek_modules/base.py @@ -29,6 +29,15 @@ console = Console(file=sys.stderr) + +class OpenSearchConnectionError(RuntimeError): + """Raised when OpenSearch is unreachable or the URL / credentials are not configured.""" + + +class OpenSearchAuthError(RuntimeError): + """Raised when OpenSearch rejects the supplied credentials (HTTP 401).""" + + # Backwards-compatible aliases — zeek modules import these names from .base _fmt_bytes = fmt_bytes _fmt_dur = fmt_dur @@ -186,14 +195,18 @@ def _cache_path(args_hash: str) -> str: def _opensearch_session() -> tuple: - """Return (base_url, authenticated Session) or (None, None) on missing creds.""" + """Return (base_url, authenticated Session). + + Raises OpenSearchConnectionError when credentials are not configured. + """ opensearch_url = os.environ.get("OPENSEARCH_URL", OPENSEARCH_URL) username = os.environ.get("PISCES_USERNAME", "") password = os.environ.get("PISCES_PASSWORD", "") if not username or not password: - console.print("[red]PISCES_USERNAME and PISCES_PASSWORD must be set in .env[/red]") - return None, None + raise OpenSearchConnectionError( + "PISCES_USERNAME and PISCES_PASSWORD must be set — check your .env file" + ) session = requests.Session() session.auth = (username, password) @@ -207,10 +220,12 @@ def _opensearch_session() -> tuple: return opensearch_url, session -def query_opensearch(body: dict, params: dict) -> dict | None: +def query_opensearch(body: dict, params: dict) -> dict: + """Submit a query to OpenSearch. + + Raises OpenSearchConnectionError or OpenSearchAuthError on failure. + """ base_url, session = _opensearch_session() - if session is None: - return None try: resp = session.post( @@ -220,18 +235,19 @@ def query_opensearch(body: dict, params: dict) -> dict | None: timeout=30, ) except requests.RequestException as exc: - console.print(f"[red]OpenSearch request failed: {exc}[/red]") - return None + raise OpenSearchConnectionError( + f"Cannot reach OpenSearch at {base_url} — are you on the VPN? ({exc})" + ) from exc if resp.status_code == 401: - console.print( - "[red]OpenSearch authentication failed — check PISCES_USERNAME/PASSWORD[/red]" + raise OpenSearchAuthError( + "OpenSearch rejected the credentials — check PISCES_USERNAME/PASSWORD" ) - return None if not resp.ok: - console.print(f"[red]OpenSearch error {resp.status_code}: {resp.text[:300]}[/red]") - return None + raise OpenSearchConnectionError( + f"OpenSearch returned HTTP {resp.status_code}: {resp.text[:300]}" + ) return resp.json() @@ -511,10 +527,6 @@ def run_query(module, search_params: dict) -> list: f" ({search_params.get('time_range', 'now-24h')})...[/dim]" ) raw = query_opensearch(body, params) - if raw is None: - if search_params.get("raise_on_error"): - raise RuntimeError("OpenSearch query failed — check credentials and OPENSEARCH_URL") - return [] if not search_params.get("profile"): _save_cache(raw, cpath) @@ -766,8 +778,10 @@ def list_sensors(time_range: str = "now-7d") -> None: params = {"path": f"{INDEX}/_search", "method": "POST"} console.print(f"[dim]Querying host.name values ({time_range})...[/dim]") - raw = query_opensearch(body, params) - if raw is None: + try: + raw = query_opensearch(body, params) + except (OpenSearchConnectionError, OpenSearchAuthError) as exc: + console.print(f"[red]{exc}[/red]") return buckets = raw.get("aggregations", {}).get("sensors", {}).get("buckets", []) @@ -817,8 +831,10 @@ def list_log_types(time_range: str = "now-7d") -> None: params = {"path": f"{INDEX}/_search", "method": "POST"} console.print(f"[dim]Querying event.dataset values ({time_range})...[/dim]") - raw = query_opensearch(body, params) - if raw is None: + try: + raw = query_opensearch(body, params) + except (OpenSearchConnectionError, OpenSearchAuthError) as exc: + console.print(f"[red]{exc}[/red]") return buckets = raw.get("aggregations", {}).get("log_types", {}).get("buckets", []) @@ -844,8 +860,10 @@ def list_log_types(time_range: str = "now-7d") -> None: def list_indices() -> None: """List all indices in the cluster sorted by doc count.""" - base_url, session = _opensearch_session() - if session is None: + try: + base_url, session = _opensearch_session() + except (OpenSearchConnectionError, OpenSearchAuthError) as exc: + console.print(f"[red]{exc}[/red]") return try: @@ -861,7 +879,7 @@ def list_indices() -> None: timeout=30, ) except requests.RequestException as exc: - console.print(f"[red]OpenSearch request failed: {exc}[/red]") + console.print(f"[red]Cannot reach OpenSearch at {base_url}: {exc}[/red]") return if not resp.ok: @@ -905,8 +923,10 @@ def match_all_sample(time_range: str = "now-24h", limit: int = 3) -> None: params = {"path": f"{INDEX}/_search", "method": "POST"} console.print(f"[dim]match_all against '{INDEX}' ({time_range}, limit {limit})...[/dim]") - raw = query_opensearch(body, params) - if raw is None: + try: + raw = query_opensearch(body, params) + except (OpenSearchConnectionError, OpenSearchAuthError) as exc: + console.print(f"[red]{exc}[/red]") return total = raw.get("hits", {}).get("total", {}) diff --git a/src/querier/zeek_modules/pe.py b/src/querier/zeek_modules/pe.py index af76e8b..c0602bc 100644 --- a/src/querier/zeek_modules/pe.py +++ b/src/querier/zeek_modules/pe.py @@ -6,7 +6,15 @@ from rich import box from rich.table import Table -from .base import INDEX, ZeekModule, _sensor_str, console, query_opensearch +from .base import ( + INDEX, + OpenSearchAuthError, + OpenSearchConnectionError, + ZeekModule, + _sensor_str, + console, + query_opensearch, +) _tl = threading.local() @@ -146,8 +154,9 @@ def prepare_hits(self, hits: list) -> None: "_source": ["zeek.files.fuid", "zeek.files.sha256", "zeek.files.md5"], } params = {"path": f"{INDEX}/_search", "method": "POST"} - raw = query_opensearch(body, params) - if raw is None: + try: + raw = query_opensearch(body, params) + except (OpenSearchConnectionError, OpenSearchAuthError): continue for h in raw.get("hits", {}).get("hits", []): s = h.get("_source", {}).get("zeek", {}).get("files", {}) diff --git a/src/querier/zeek_modules/x509.py b/src/querier/zeek_modules/x509.py index 7f4ff2d..8088a72 100644 --- a/src/querier/zeek_modules/x509.py +++ b/src/querier/zeek_modules/x509.py @@ -7,7 +7,16 @@ from rich import box from rich.table import Table -from .base import INDEX, ZeekModule, _sensor_str, console, is_private, query_opensearch +from .base import ( + INDEX, + OpenSearchAuthError, + OpenSearchConnectionError, + ZeekModule, + _sensor_str, + console, + is_private, + query_opensearch, +) _tl = threading.local() @@ -108,8 +117,9 @@ def prepare_hits(self, hits: list) -> None: "_source": ["network.community_id", "source.ip", "destination.ip"], } params = {"path": f"{INDEX}/_search", "method": "POST"} - raw = query_opensearch(body, params) - if raw is None: + try: + raw = query_opensearch(body, params) + except (OpenSearchConnectionError, OpenSearchAuthError): continue for h in raw.get("hits", {}).get("hits", []): s = h.get("_source", {}) From cfc499b3fc5ff64f6124fe5f4eb2fd41764dbdfa Mon Sep 17 00:00:00 2001 From: liamadale Date: Mon, 11 May 2026 09:40:00 -0700 Subject: [PATCH 071/109] feat(web): surface OpenSearch and Mantis errors in web routes Web routes now catch OpenSearchConnectionError and OpenSearchAuthError and pass an error string to their templates rather than silently returning empty results. - opensearch_web: overview, ip_pivot, log_view, log_rows partial, and the device card route all handle connectivity/auth exceptions - opensearch_web/queries: run_cross_protocol_query re-raises the first connection error after collecting partial results from all threads - dashboard_web aggregations and malcolm query helpers wrap query_opensearch in _safe_query() returning None on failure, keeping dashboard charts gracefully empty rather than crashing - shared/blueprints: Mantis search errors are caught and forwarded to the results partial --- apps/dashboard_web/opensearch/aggregations.py | 26 ++++--- apps/dashboard_web/opensearch/malcolm.py | 19 ++++- apps/opensearch_web/app.py | 70 +++++++++++++++---- apps/opensearch_web/queries.py | 15 +++- apps/shared/blueprints.py | 10 ++- 5 files changed, 112 insertions(+), 28 deletions(-) diff --git a/apps/dashboard_web/opensearch/aggregations.py b/apps/dashboard_web/opensearch/aggregations.py index bc24861..f34cbbf 100644 --- a/apps/dashboard_web/opensearch/aggregations.py +++ b/apps/dashboard_web/opensearch/aggregations.py @@ -2,12 +2,22 @@ from src.querier.zeek_modules.base import ( FILTERS_DIR, + OpenSearchAuthError, + OpenSearchConnectionError, build_base_query, load_with_remap, query_opensearch, ) +def _safe_query(body: dict, params: dict) -> dict | None: + """query_opensearch wrapper that returns None on connectivity/auth errors.""" + try: + return query_opensearch(body, params) + except (OpenSearchConnectionError, OpenSearchAuthError): + return None + + def parse_sensors(raw: str) -> list | None: """Parse a comma-separated sensor string into a list, or None for 'all'.""" if not raw or raw.strip().lower() == "all": @@ -53,7 +63,7 @@ def agg_opensearch_sensors(time_range: str) -> dict: } } } - raw = query_opensearch(body, params) + raw = _safe_query(body, params) buckets = raw.get("aggregations", {}).get("sensors", {}).get("buckets", []) if raw else [] return { "labels": [b["key"] for b in buckets], @@ -81,7 +91,7 @@ def agg_opensearch_notice_count(time_range: str, sensors: list | None = None) -> body["size"] = 0 body.pop("sort", None) body.pop("_source", None) - raw = query_opensearch(body, params) + raw = _safe_query(body, params) if not raw: return 0 return raw.get("hits", {}).get("total", {}).get("value", 0) @@ -127,7 +137,7 @@ def agg_suricata_alert_count(time_range: str, sensors: list | None = None) -> in body["size"] = 0 body.pop("sort", None) body.pop("_source", None) - raw = query_opensearch(body, params) + raw = _safe_query(body, params) if not raw: return 0 return raw.get("hits", {}).get("total", {}).get("value", 0) @@ -160,7 +170,7 @@ def agg_suricata_over_time(time_range: str, sensors: list | None = None) -> dict } } } - raw = query_opensearch(body, params) + raw = _safe_query(body, params) buckets = raw.get("aggregations", {}).get("over_time", {}).get("buckets", []) if raw else [] return { "timestamps": [b["key_as_string"] for b in buckets], @@ -199,7 +209,7 @@ def agg_notice_over_time(time_range: str, sensors: list | None = None) -> dict: } } } - raw = query_opensearch(body, params) + raw = _safe_query(body, params) buckets = raw.get("aggregations", {}).get("over_time", {}).get("buckets", []) if raw else [] return { "timestamps": [b["key_as_string"] for b in buckets], @@ -235,7 +245,7 @@ def agg_conn_volume_over_time(time_range: str, sensors: list | None = None) -> d } } } - raw = query_opensearch(body, params) + raw = _safe_query(body, params) buckets = raw.get("aggregations", {}).get("over_time", {}).get("buckets", []) if raw else [] return { "timestamps": [b["key_as_string"] for b in buckets], @@ -276,7 +286,7 @@ def agg_logs_by_sensor_over_time(time_range: str, sensors: list | None = None) - }, } } - raw = query_opensearch(body, params) + raw = _safe_query(body, params) sensor_buckets = ( raw.get("aggregations", {}).get("by_sensor", {}).get("buckets", []) if raw else [] ) @@ -332,7 +342,7 @@ def _unique_count(tr: str) -> int: body["aggs"] = { "uniq": {"cardinality": {"field": "source.ip", "precision_threshold": 3000}} } - raw = query_opensearch(body, params) + raw = _safe_query(body, params) if not raw: return 0 return raw.get("aggregations", {}).get("uniq", {}).get("value", 0) diff --git a/apps/dashboard_web/opensearch/malcolm.py b/apps/dashboard_web/opensearch/malcolm.py index d6a279d..f960cdb 100644 --- a/apps/dashboard_web/opensearch/malcolm.py +++ b/apps/dashboard_web/opensearch/malcolm.py @@ -6,7 +6,20 @@ import concurrent.futures -from src.querier.zeek_modules.base import build_base_query, query_opensearch +from src.querier.zeek_modules.base import ( + OpenSearchAuthError, + OpenSearchConnectionError, + build_base_query, + query_opensearch, +) + + +def _safe_query(body: dict, params: dict) -> dict | None: + """query_opensearch wrapper that returns None on connectivity/auth errors.""" + try: + return query_opensearch(body, params) + except (OpenSearchConnectionError, OpenSearchAuthError): + return None def _terms(field: str, time_range: str, datasets: list, size: int = 20) -> dict: @@ -27,7 +40,7 @@ def _terms(field: str, time_range: str, datasets: list, size: int = 20) -> dict: body.pop("sort", None) body.pop("_source", None) body["aggs"] = {"r": {"terms": {"field": field, "size": size, "order": {"_count": "desc"}}}} - raw = query_opensearch(body, params) + raw = _safe_query(body, params) buckets = raw.get("aggregations", {}).get("r", {}).get("buckets", []) if raw else [] return { "labels": [b["key"] for b in buckets], @@ -60,7 +73,7 @@ def _sum_terms( "aggs": {"total": {"sum": {"field": sum_field}}}, } } - raw = query_opensearch(body, params) + raw = _safe_query(body, params) buckets = raw.get("aggregations", {}).get("r", {}).get("buckets", []) if raw else [] buckets = sorted(buckets, key=lambda b: -b.get("total", {}).get("value", 0)) return { diff --git a/apps/opensearch_web/app.py b/apps/opensearch_web/app.py index ab1fa51..cd0a7b9 100644 --- a/apps/opensearch_web/app.py +++ b/apps/opensearch_web/app.py @@ -24,7 +24,11 @@ MODULES, MODULES_BY_CATEGORY, ) -from src.querier.zeek_modules.base import TIME_RANGES +from src.querier.zeek_modules.base import ( + TIME_RANGES, + OpenSearchAuthError, + OpenSearchConnectionError, +) from src.utils.format import fmt_dur # ------------------------------------------------------------------ @@ -125,7 +129,12 @@ def inject_nav_data() -> dict: @app.route("/") def overview(): search_params = build_search_params_from_request(request) - rows = run_cross_protocol_query(search_params) + error = None + rows = [] + try: + rows = run_cross_protocol_query(search_params) + except (OpenSearchConnectionError, OpenSearchAuthError) as exc: + error = str(exc) # Build per-category module lists, excluding non-IP modules (pe, capture_loss) ip_modules_by_category = { cat: [lt for lt in lts if MODULES[lt].SUPPORTS_IP_FILTER] @@ -134,6 +143,7 @@ def overview(): return render_template( "overview.html", rows=rows, + error=error, search_params=search_params, ip_modules_by_category=ip_modules_by_category, ) @@ -147,16 +157,22 @@ def ip_pivot(ip: str): search_params["src_ip"] = ip results: dict = {} + error = None for lt, mod in MODULES.items(): if not mod.SUPPORTS_IP_FILTER: continue # pe, capture_loss have no src_ip to pivot on sp = dict(search_params) - results[lt] = cached_run_query(lt, sp) + try: + results[lt] = cached_run_query(lt, sp) + except (OpenSearchConnectionError, OpenSearchAuthError) as exc: + error = str(exc) + break return render_template( "ip_pivot.html", ip=ip, results=results, + error=error, search_params=search_params, ) @@ -175,12 +191,19 @@ def log_view(log_type: str): # unless the user has drilled into a specific value. has_drill_filter = bool(mod.SUMMARY_PARAM and search_params.get(mod.SUMMARY_PARAM)) summary_mode = bool(mod.SUMMARY_FIELD) and not has_drill_filter - records = [] if summary_mode else cached_run_query(log_type, search_params) + error = None + records = [] + if not summary_mode: + try: + records = cached_run_query(log_type, search_params) + except (OpenSearchConnectionError, OpenSearchAuthError) as exc: + error = str(exc) return render_template( "log_view.html", log_type=log_type, records=records, + error=error, search_params=search_params, extra_keys=extra_keys, detail_fields=mod.DETAIL_FIELDS, @@ -200,11 +223,22 @@ def api_search(log_type: str): mod = MODULES[log_type] extra_keys = mod.EXTRA_PARAMS search_params = build_search_params_from_request(request, extra_keys) - records = cached_run_query(log_type, search_params) + try: + records = cached_run_query(log_type, search_params) + except (OpenSearchConnectionError, OpenSearchAuthError) as exc: + return render_template( + "partials/log_rows.html", + log_type=log_type, + records=[], + error=str(exc), + detail_fields=mod.DETAIL_FIELDS, + web_columns=mod.WEB_COLUMNS, + ) return render_template( "partials/log_rows.html", log_type=log_type, records=records, + error=None, detail_fields=mod.DETAIL_FIELDS, web_columns=mod.WEB_COLUMNS, ) @@ -597,17 +631,25 @@ def api_profile(ip: str): time_range = request.args.get("time_range", "now-7d") compact = request.args.get("compact") == "1" - if is_private(ip): - from src.profiler.device_profiler import profile_device + try: + if is_private(ip): + from src.profiler.device_profiler import profile_device - profile = profile_device(ip, time_range=time_range, sensor=sensor) - return render_template("partials/device_card.html", profile=profile, compact=compact) - else: - from src.profiler.public_ip_profiler import profile_public_ip + profile = profile_device(ip, time_range=time_range, sensor=sensor) + return render_template( + "partials/device_card.html", profile=profile, compact=compact + ) + else: + from src.profiler.public_ip_profiler import profile_public_ip - profile = profile_public_ip(ip, time_range=time_range) - return render_template( - "partials/public_device_card.html", profile=profile, compact=compact + profile = profile_public_ip(ip, time_range=time_range) + return render_template( + "partials/public_device_card.html", profile=profile, compact=compact + ) + except (OpenSearchConnectionError, OpenSearchAuthError) as exc: + return ( + f'

' + f' {exc}

' ) # ------------------------------------------------------------------ diff --git a/apps/opensearch_web/queries.py b/apps/opensearch_web/queries.py index ef3baf1..be268b3 100644 --- a/apps/opensearch_web/queries.py +++ b/apps/opensearch_web/queries.py @@ -5,7 +5,12 @@ from apps.opensearch_web import cache as wcache from src.querier.zeek_modules import MODULES -from src.querier.zeek_modules.base import console, run_query +from src.querier.zeek_modules.base import ( + OpenSearchAuthError, + OpenSearchConnectionError, + console, + run_query, +) def build_search_params_from_request(request, extra_keys=None) -> dict: @@ -46,16 +51,24 @@ def run_cross_protocol_query(search_params: dict) -> list: """ ip_modules = {lt: mod for lt, mod in MODULES.items() if mod.SUPPORTS_IP_FILTER} results_by_type: dict = {} + first_conn_error: Exception | None = None with ThreadPoolExecutor(max_workers=len(ip_modules)) as ex: futures = {ex.submit(cached_run_query, lt, search_params): lt for lt in ip_modules} for f in as_completed(futures): lt = futures[f] try: results_by_type[lt] = f.result() + except (OpenSearchConnectionError, OpenSearchAuthError) as exc: + if first_conn_error is None: + first_conn_error = exc + results_by_type[lt] = [] except Exception as exc: results_by_type[lt] = [] console.print(f"[yellow]Cross-protocol query failed for {lt}: {exc}[/yellow]") + if first_conn_error is not None: + raise first_conn_error + ip_data: dict = defaultdict(lambda: {"per_protocol": {lt: 0 for lt in ip_modules}, "total": 0}) for lt, records in results_by_type.items(): for rec in records: diff --git a/apps/shared/blueprints.py b/apps/shared/blueprints.py index 15bc367..f9eb5de 100644 --- a/apps/shared/blueprints.py +++ b/apps/shared/blueprints.py @@ -71,9 +71,15 @@ def api_mantis_search() -> Any: query = request.args.get("query", "").strip() idx = request.args.get("idx", "0") city = resolve_city(request) - tickets = search(query, city=city) if query else [] + error = None + tickets: list = [] + if query: + try: + tickets = search(query, city=city) + except Exception as exc: + error = str(exc) return render_template( - "partials/mantis_results.html", tickets=tickets, query=query, idx=idx + "partials/mantis_results.html", tickets=tickets, query=query, idx=idx, error=error ) return bp From d80cd57fd1e5225bcc64c0d375f41a7d3bc9ef61 Mon Sep 17 00:00:00 2001 From: liamadale Date: Mon, 11 May 2026 09:40:08 -0700 Subject: [PATCH 072/109] feat(ui): add error banners for OpenSearch, Mantis, and missing threat model data Templates now render a dismissible error banner when an error string is passed from the route, replacing silent empty tables. - overview, log_view, ip_pivot: red query-error-banner with exclamation icon when OpenSearch is unreachable or credentials are invalid - log_rows partial: inline error cell spanning all columns for HTMX re-search responses - mantis_results partial: error state when Mantis search throws - threat_model index: yellow info banner when DATA_AVAILABLE is False, directing the user to run the indexing and enrichment scripts - pisces.css: query-error-banner and query-error-cell styles - tm.css: tm-no-data-banner style with code element formatting --- apps/opensearch_web/static/pisces.css | 27 +++++++++++++++++++ apps/opensearch_web/templates/ip_pivot.html | 7 +++++ apps/opensearch_web/templates/log_view.html | 7 +++++ apps/opensearch_web/templates/overview.html | 7 ++++- .../templates/partials/log_rows.html | 7 +++++ .../templates/partials/mantis_results.html | 7 ++++- apps/threat_model/app.py | 2 ++ apps/threat_model/data.py | 3 +++ apps/threat_model/static/tm.css | 26 ++++++++++++++++++ apps/threat_model/templates/index.html | 11 ++++++++ 10 files changed, 102 insertions(+), 2 deletions(-) diff --git a/apps/opensearch_web/static/pisces.css b/apps/opensearch_web/static/pisces.css index e8160ba..94a410d 100644 --- a/apps/opensearch_web/static/pisces.css +++ b/apps/opensearch_web/static/pisces.css @@ -1667,3 +1667,30 @@ tr.detail-row td { line-height: 1.5; } .log-count-btn--empty .log-count-badge { background: var(--on-surface-dim); } + +/* ── Query error banner ───────────────────────────────── */ +.query-error-banner { + display: flex; + align-items: flex-start; + gap: .6rem; + padding: .85rem 1.1rem; + margin: 1rem 0; + background: color-mix(in srgb, var(--red) 12%, var(--surface)); + border: 1px solid color-mix(in srgb, var(--red) 40%, transparent); + border-radius: var(--radius-card); + color: var(--on-surface); + font-size: .875rem; + line-height: 1.5; +} +.query-error-banner i { + color: var(--red); + margin-top: .15rem; + flex-shrink: 0; +} +.query-error-cell { + text-align: left; + color: var(--red); + padding: .75rem 1rem; + font-size: .85rem; +} +.query-error-cell i { margin-right: .4rem; } diff --git a/apps/opensearch_web/templates/ip_pivot.html b/apps/opensearch_web/templates/ip_pivot.html index 4c402ad..98b2ab5 100644 --- a/apps/opensearch_web/templates/ip_pivot.html +++ b/apps/opensearch_web/templates/ip_pivot.html @@ -31,6 +31,13 @@

IP Pivot: {{ ip }}

+{% if error %} +
+ + OpenSearch unavailable — {{ error }} +
+{% endif %} + {% for lt, records in results.items() %} {% set mod = MODULES[lt] %} {% set web_columns = mod.WEB_COLUMNS %} diff --git a/apps/opensearch_web/templates/log_view.html b/apps/opensearch_web/templates/log_view.html index 39ba006..ffbbb98 100644 --- a/apps/opensearch_web/templates/log_view.html +++ b/apps/opensearch_web/templates/log_view.html @@ -18,6 +18,13 @@

{{ log_type }}

+{% if error %} +
+ + OpenSearch unavailable — {{ error }} +
+{% endif %} + {# Protocol-specific filter form (only shown if the module has extra params) #} {% if extra_keys %}
Cross-Protocol IP Activity Click a cell count to drill into that protocol. Click an IP to pivot.

-{% if rows %} +{% if error %} +
+ + OpenSearch unavailable — {{ error }} +
+{% elif rows %}
diff --git a/apps/opensearch_web/templates/partials/log_rows.html b/apps/opensearch_web/templates/partials/log_rows.html index a68a4fe..d91e2e7 100644 --- a/apps/opensearch_web/templates/partials/log_rows.html +++ b/apps/opensearch_web/templates/partials/log_rows.html @@ -1,3 +1,9 @@ +{% if error %} + +{% else %} {% for rec in records %} {% set idx = loop.index %} {% endfor %} +{% endif %} diff --git a/apps/shared/templates/partials/mantis_results.html b/apps/shared/templates/partials/mantis_results.html index 375c255..5a55e5c 100644 --- a/apps/shared/templates/partials/mantis_results.html +++ b/apps/shared/templates/partials/mantis_results.html @@ -1,4 +1,9 @@ -{% if not tickets %} +{% if error %} +

+ + Mantis unavailable: {{ error }} +

+{% elif not tickets %}

{% if query %}No tickets found for {{ query }}.{% else %}Enter a query above.{% endif %} diff --git a/apps/threat_model/app.py b/apps/threat_model/app.py index 9c524ed..569e7e0 100644 --- a/apps/threat_model/app.py +++ b/apps/threat_model/app.py @@ -9,6 +9,7 @@ ALL_BLOCKLISTS, ALL_FP_CATEGORIES, ALL_INFRA_CATEGORIES, + DATA_AVAILABLE, DNS_RESOLVER_ROWS, FP_BY_IP, FP_ROWS, @@ -188,6 +189,7 @@ def index(): infra_total=len(INFRA_ROWS), dns_resolver_total=len(DNS_RESOLVER_ROWS), undetermined_total=len(UNDETERMINED_ROWS), + data_available=DATA_AVAILABLE, ) # ------------------------------------------------------------------ diff --git a/apps/threat_model/data.py b/apps/threat_model/data.py index 2108986..404b16c 100644 --- a/apps/threat_model/data.py +++ b/apps/threat_model/data.py @@ -59,6 +59,9 @@ def _load_optional(name: str) -> list: _raw_undetermined = _load_optional("enriched/undetermined_ips.json") _raw_profiles = _load_optional("enriched/private_ip_profiles.json") +# True when the pipeline has been run and data files exist. +DATA_AVAILABLE: bool = bool(_raw_tickets or _raw_malicious or _raw_fp) + # --------------------------------------------------------------------------- # Indices # --------------------------------------------------------------------------- diff --git a/apps/threat_model/static/tm.css b/apps/threat_model/static/tm.css index 3839631..af6bd77 100644 --- a/apps/threat_model/static/tm.css +++ b/apps/threat_model/static/tm.css @@ -1189,3 +1189,29 @@ tr.ticket-row-active { .org-icon.org-scanner { color: var(--yellow); } .org-icon.org-research { color: var(--purple); } .org-icon.org-private { color: var(--on-surface-dim); } + +/* ── No-data banner ────────────────────────────────────── */ +.tm-no-data-banner { + display: flex; + align-items: flex-start; + gap: .7rem; + padding: .9rem 1.2rem; + margin: 0 0 1rem; + background: color-mix(in srgb, var(--yellow) 10%, var(--surface)); + border: 1px solid color-mix(in srgb, var(--yellow) 35%, transparent); + border-radius: var(--radius-card); + font-size: .875rem; + line-height: 1.55; +} +.tm-no-data-banner i { + color: var(--yellow); + margin-top: .2rem; + flex-shrink: 0; +} +.tm-no-data-banner code { + font-family: var(--font-mono, monospace); + font-size: .8rem; + background: color-mix(in srgb, var(--on-surface) 10%, transparent); + padding: .1rem .35rem; + border-radius: 3px; +} diff --git a/apps/threat_model/templates/index.html b/apps/threat_model/templates/index.html index 7e2487c..ca7cabb 100644 --- a/apps/threat_model/templates/index.html +++ b/apps/threat_model/templates/index.html @@ -4,6 +4,17 @@ {% block content %} +{% if not data_available %} +

+ +
+ No threat model data found. + The enrichment pipeline hasn't been run yet, or the data files are missing from + data/tickets/. Run the indexing and enrichment scripts to populate this view. +
+
+{% endif %} + {# ── Search bar ────────────────────────────────────────────── #}
From 984a84e441a6ce470995d4b47b8b364b13538bc1 Mon Sep 17 00:00:00 2001 From: liamadale Date: Mon, 11 May 2026 10:04:40 -0700 Subject: [PATCH 073/109] feat(profiler): add conn_state-derived traffic summary to device cards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both DeviceProfile and PublicIPProfile now include a conn_state_distribution field (populated from an existing conn aggregation — no extra round-trips) and a conn_status string derived from the dominant conn_state code (e.g. "Connections completed", "No response — likely blocked", "Actively rejected"). The device card templates display a traffic summary bar showing ↑/↓ bytes, destination/client count, and the plain-English connection status. CSS utility classes (dct-*) are added to pisces.css to style the bar consistently across internal and public IP cards. --- apps/opensearch_web/static/pisces.css | 6 ++++ .../templates/partials/device_card.html | 13 +++++++++ .../partials/public_device_card.html | 15 +++++++--- src/profiler/device_profiler.py | 29 +++++++++++++++++++ src/profiler/public_ip_profiler.py | 27 +++++++++++++++++ 5 files changed, 86 insertions(+), 4 deletions(-) diff --git a/apps/opensearch_web/static/pisces.css b/apps/opensearch_web/static/pisces.css index 94a410d..7a4acf4 100644 --- a/apps/opensearch_web/static/pisces.css +++ b/apps/opensearch_web/static/pisces.css @@ -1458,6 +1458,12 @@ tr.detail-row td { .device-card-footer { padding: .5rem 1rem; font-size: .75rem; color: var(--on-surface-dim); border-top: 1px solid var(--border); } .device-card-footer .btn-link { background: none; border: none; color: var(--primary); cursor: pointer; font-size: .75rem; padding: 0; margin-right: .5rem; text-decoration: underline; } .device-card-users { padding: 0 1rem .5rem; font-size: .8rem; color: var(--on-surface-dim); border-bottom: 1px solid var(--border); } +.device-card-traffic { padding: .4rem 1rem; font-size: .8rem; border-bottom: 1px solid var(--border); display: flex; flex-wrap: wrap; gap: .25rem .5rem; align-items: center; } +.dct-up { color: var(--green); font-weight: 600; font-family: var(--mono); } +.dct-down { color: var(--on-surface); font-weight: 600; font-family: var(--mono); } +.dct-sep { color: var(--on-surface-dim); } +.dct-dests { color: var(--on-surface-dim); } +.dct-status { color: var(--on-surface-dim); font-style: italic; } .role-badge { padding: 2px 8px; border-radius: 4px; font-size: .8rem; font-weight: 600; } .role-domain_controller { background: color-mix(in srgb, var(--blue) 15%, transparent); color: var(--blue); } .role-file_server { background: color-mix(in srgb, var(--yellow) 15%, transparent); color: var(--yellow); } diff --git a/apps/opensearch_web/templates/partials/device_card.html b/apps/opensearch_web/templates/partials/device_card.html index 7fd568e..58de743 100644 --- a/apps/opensearch_web/templates/partials/device_card.html +++ b/apps/opensearch_web/templates/partials/device_card.html @@ -30,6 +30,19 @@ {% endif %}
+ {# Traffic summary #} +
+ ↑ {{ profile.bytes_sent | fmt_bytes }} + · + ↓ {{ profile.bytes_received | fmt_bytes }} + · + {{ profile.unique_dest_count }} dest{{ 's' if profile.unique_dest_count != 1 else '' }} + {% if profile.conn_status %} + · + {{ profile.conn_status }} + {% endif %} +
+ {# Signal grid #}
diff --git a/apps/opensearch_web/templates/partials/public_device_card.html b/apps/opensearch_web/templates/partials/public_device_card.html index c2c978c..7b7c0d8 100644 --- a/apps/opensearch_web/templates/partials/public_device_card.html +++ b/apps/opensearch_web/templates/partials/public_device_card.html @@ -67,10 +67,17 @@

Services (our traffic TO this IP)

{% endfor %}
+ + OpenSearch unavailable — {{ error }} +
No records.
-

- Internal clients: {{ "{:,}".format(profile.internal_client_count) }} - · Bytes: {{ profile.bytes_to | fmt_bytes }} in / {{ profile.bytes_from | fmt_bytes }} out -

+
+ ↑ {{ profile.bytes_to | fmt_bytes }} + · + ↓ {{ profile.bytes_from | fmt_bytes }} + · + {{ "{:,}".format(profile.internal_client_count) }} internal client{{ 's' if profile.internal_client_count != 1 else '' }} + {% if profile.conn_status %} + · + {{ profile.conn_status }} + {% endif %} +
{% else %}

No outbound connections to this IP

{% endif %} diff --git a/src/profiler/device_profiler.py b/src/profiler/device_profiler.py index 93315a2..3518f81 100644 --- a/src/profiler/device_profiler.py +++ b/src/profiler/device_profiler.py @@ -28,6 +28,26 @@ _PARAMS = {"path": f"{INDEX}/_search", "method": "POST"} +_CONN_BLOCKED = {"S0", "RSTOS0", "SH"} +_CONN_REJECTED = {"REJ", "RSTR", "RSTRH"} +_CONN_NORMAL = {"SF", "S2", "S3"} + + +def _derive_conn_status(state_dist: dict[str, int]) -> str: + """Translate a conn_state frequency map into a plain-English firewall signal.""" + if not state_dist: + return "" + dominant = max(state_dist, key=lambda k: state_dist[k]) + if dominant in _CONN_BLOCKED: + return "No response — likely blocked" + if dominant in _CONN_REJECTED: + return "Actively rejected" + if dominant in _CONN_NORMAL: + return "Connections completed" + if dominant == "S1": + return "Active connections" + return f"Mostly {dominant}" + # --------------------------------------------------------------------------- # DeviceProfile dataclass @@ -57,6 +77,8 @@ class DeviceProfile: protocol_mix: dict[str, int] = field(default_factory=dict) unique_dest_count: int = 0 bytes_sent: int = 0 + conn_state_distribution: dict[str, int] = field(default_factory=dict) + conn_status: str = "" ja4t_fingerprints: list[dict] = field(default_factory=list) # Inbound conn (device as server) @@ -125,6 +147,7 @@ def _conn_outbound_query(ip: str, time_range: str, sensor: str) -> dict: "app_protos": {"terms": {"field": "network.application", "size": 10}}, "unique_dests": {"cardinality": {"field": "destination.ip"}}, "total_bytes": {"sum": {"field": "source.bytes"}}, + "conn_states": {"terms": {"field": "zeek.conn.conn_state", "size": 10}}, "ja4t_fingerprints": {"terms": {"field": "zeek.conn.ja4t", "size": 5}}, "time_range": {"stats": {"field": "@timestamp"}}, }, @@ -393,12 +416,16 @@ def _parse_outbound(aggs: dict) -> dict: {"hash": b["key"], "count": b["doc_count"]} for b in aggs.get("ja4t_fingerprints", {}).get("buckets", []) ] + conn_state_dist = { + b["key"]: b["doc_count"] for b in aggs.get("conn_states", {}).get("buckets", []) + } ts = aggs.get("time_range", {}) return { "dest_port_distribution": dest_ports, "protocol_mix": app_protos, "unique_dest_count": int(aggs.get("unique_dests", {}).get("value", 0)), "bytes_sent": int(aggs.get("total_bytes", {}).get("value", 0)), + "conn_state_distribution": conn_state_dist, "ja4t_fingerprints": ja4t, "first_seen": ts.get("min_as_string", ""), "last_seen": ts.get("max_as_string", ""), @@ -611,6 +638,8 @@ def profile_device( protocol_mix=out["protocol_mix"], unique_dest_count=out["unique_dest_count"], bytes_sent=out["bytes_sent"], + conn_state_distribution=out["conn_state_distribution"], + conn_status=_derive_conn_status(out["conn_state_distribution"]), ja4t_fingerprints=out["ja4t_fingerprints"], inbound_services=inb["inbound_services"], inbound_client_count=inb["inbound_client_count"], diff --git a/src/profiler/public_ip_profiler.py b/src/profiler/public_ip_profiler.py index 2ece02c..9ea2bd5 100644 --- a/src/profiler/public_ip_profiler.py +++ b/src/profiler/public_ip_profiler.py @@ -15,6 +15,26 @@ _PARAMS = {"path": f"{INDEX}/_search", "method": "POST"} +_CONN_BLOCKED = {"S0", "RSTOS0", "SH"} +_CONN_REJECTED = {"REJ", "RSTR", "RSTRH"} +_CONN_NORMAL = {"SF", "S2", "S3"} + + +def _derive_conn_status(state_dist: dict[str, int]) -> str: + """Translate a conn_state frequency map into a plain-English firewall signal.""" + if not state_dist: + return "" + dominant = max(state_dist, key=lambda k: state_dist[k]) + if dominant in _CONN_BLOCKED: + return "No response — likely blocked" + if dominant in _CONN_REJECTED: + return "Actively rejected" + if dominant in _CONN_NORMAL: + return "Connections completed" + if dominant == "S1": + return "Active connections" + return f"Mostly {dominant}" + # --------------------------------------------------------------------------- # PublicIPProfile dataclass @@ -41,6 +61,8 @@ class PublicIPProfile: internal_client_count: int = 0 bytes_to: int = 0 bytes_from: int = 0 + conn_state_distribution: dict[str, int] = field(default_factory=dict) + conn_status: str = "" # TLS (server-side) ja4s_fingerprints: list[dict] = field(default_factory=list) @@ -146,6 +168,7 @@ def _conn_to_query(ip: str, time_range: str) -> dict: "unique_clients": {"cardinality": {"field": "source.ip"}}, "bytes_to": {"sum": {"field": "destination.bytes"}}, "bytes_from": {"sum": {"field": "source.bytes"}}, + "conn_states": {"terms": {"field": "zeek.conn.conn_state", "size": 10}}, "time_range": {"stats": {"field": "@timestamp"}}, }, } @@ -290,12 +313,14 @@ def _parse_conn_to(aggs: dict) -> dict: "count": b["doc_count"], } ) + conn_state_dist = {b["key"]: b["doc_count"] for b in _buckets(aggs, "conn_states")} ts = aggs.get("time_range", {}) return { "services": services, "internal_client_count": int(aggs.get("unique_clients", {}).get("value", 0)), "bytes_to": int(aggs.get("bytes_to", {}).get("value", 0)), "bytes_from": int(aggs.get("bytes_from", {}).get("value", 0)), + "conn_state_distribution": conn_state_dist, "first_seen": ts.get("min_as_string", ""), "last_seen": ts.get("max_as_string", ""), } @@ -423,6 +448,8 @@ def profile_public_ip( internal_client_count=conn_to["internal_client_count"], bytes_to=conn_to["bytes_to"], bytes_from=conn_to["bytes_from"], + conn_state_distribution=conn_to["conn_state_distribution"], + conn_status=_derive_conn_status(conn_to["conn_state_distribution"]), ja4s_fingerprints=ssl_to["ja4s_fingerprints"], tls_versions=ssl_to["tls_versions"], ssl_subjects=ssl_to["ssl_subjects"], From 8acc061a951df4ca4abf464a473abec29e3d1cf3 Mon Sep 17 00:00:00 2001 From: liamadale Date: Mon, 11 May 2026 21:15:32 -0700 Subject: [PATCH 074/109] perf(enricher): add persistent HTTP sessions to all enricher clients MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace one-off requests.get() calls in greynoise, abuseipdb, shodan, and virustotal with a module-level Session backed by a connection pool (pool_connections=4, pool_maxsize=8). This keeps TLS connections alive across calls — particularly valuable during parallel enrichment where all four providers are queried concurrently. --- src/enricher/abuseipdb.py | 5 ++++- src/enricher/greynoise.py | 5 ++++- src/enricher/shodan.py | 5 ++++- src/enricher/virustotal.py | 7 +++++-- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/enricher/abuseipdb.py b/src/enricher/abuseipdb.py index 6f0dab1..9f03a70 100644 --- a/src/enricher/abuseipdb.py +++ b/src/enricher/abuseipdb.py @@ -17,6 +17,9 @@ console = Console(file=sys.stderr) +_session = requests.Session() +_session.mount("https://", requests.adapters.HTTPAdapter(pool_connections=4, pool_maxsize=8)) + def check_ip(ip: str, max_age_days: int = 90) -> dict: """Query AbuseIPDB for an IP address. @@ -43,7 +46,7 @@ def check_ip(ip: str, max_age_days: int = 90) -> dict: params = {"ipAddress": ip, "maxAgeInDays": max_age_days, "verbose": True} try: - resp = requests.get(_BASE_URL, headers=headers, params=params, timeout=10) + resp = _session.get(_BASE_URL, headers=headers, params=params, timeout=10) except requests.RequestException as exc: return _error_result(f"Request failed: {exc}") diff --git a/src/enricher/greynoise.py b/src/enricher/greynoise.py index 02e33c1..8175c22 100644 --- a/src/enricher/greynoise.py +++ b/src/enricher/greynoise.py @@ -17,6 +17,9 @@ console = Console(file=sys.stderr) +_session = requests.Session() +_session.mount("https://", requests.adapters.HTTPAdapter(pool_connections=4, pool_maxsize=8)) + def check_ip(ip: str) -> dict: """Query GreyNoise community API for an IP. @@ -33,7 +36,7 @@ def check_ip(ip: str) -> dict: headers = {"key": api_key} if api_key else {} try: - resp = requests.get(f"{_BASE_URL}/{ip}", headers=headers, timeout=10) + resp = _session.get(f"{_BASE_URL}/{ip}", headers=headers, timeout=10) except requests.RequestException as exc: return { "classification": "not_found", diff --git a/src/enricher/shodan.py b/src/enricher/shodan.py index d55d0a8..f5e6616 100644 --- a/src/enricher/shodan.py +++ b/src/enricher/shodan.py @@ -17,6 +17,9 @@ console = Console(file=sys.stderr) +_session = requests.Session() +_session.mount("https://", requests.adapters.HTTPAdapter(pool_connections=4, pool_maxsize=8)) + def check_ip(ip: str) -> dict: """Query Shodan host API for an IP. @@ -39,7 +42,7 @@ def check_ip(ip: str) -> dict: return _error_result("SHODAN_API_KEY not set") try: - resp = requests.get(f"{_BASE_URL}/{ip}", params={"key": api_key}, timeout=10) + resp = _session.get(f"{_BASE_URL}/{ip}", params={"key": api_key}, timeout=10) except requests.RequestException as exc: return _error_result(f"Request failed: {exc}") diff --git a/src/enricher/virustotal.py b/src/enricher/virustotal.py index 6629f08..faf34ff 100644 --- a/src/enricher/virustotal.py +++ b/src/enricher/virustotal.py @@ -23,6 +23,9 @@ console = Console(file=sys.stderr) +_session = requests.Session() +_session.mount("https://", requests.adapters.HTTPAdapter(pool_connections=4, pool_maxsize=8)) + def check_ip(ip: str) -> dict: """Query VirusTotal for an IP address. @@ -47,7 +50,7 @@ def check_ip(ip: str) -> dict: headers = {"x-apikey": api_key} try: - resp = requests.get(f"{_BASE_URL}/{ip}", headers=headers, timeout=10) + resp = _session.get(f"{_BASE_URL}/{ip}", headers=headers, timeout=10) except requests.RequestException as exc: return _error_result(f"Request failed: {exc}") @@ -125,7 +128,7 @@ def check_hash(hash_value: str) -> dict: headers = {"x-apikey": api_key} try: - resp = requests.get(f"{_HASH_BASE_URL}/{hash_value}", headers=headers, timeout=10) + resp = _session.get(f"{_HASH_BASE_URL}/{hash_value}", headers=headers, timeout=10) except requests.RequestException as exc: return _hash_error_result(f"Request failed: {exc}") From 50d69a86f621783742deb344eae62141eb03ac91 Mon Sep 17 00:00:00 2001 From: liamadale Date: Mon, 11 May 2026 21:15:42 -0700 Subject: [PATCH 075/109] feat(enricher): add parallel execution and result caching to web enrich path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When called from the web UI (offer_fp=False), enrich_ip() now fires all four providers concurrently via ThreadPoolExecutor instead of sequentially, and caches results with a 6-hour TTL. The CLI path (offer_fp=True) is unchanged — GreyNoise runs first so the analyst can be offered an FP filter on benign IPs before the remaining providers are queried. --- src/enricher/threat_intel.py | 69 ++++++++++++++++++++++++++---------- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/src/enricher/threat_intel.py b/src/enricher/threat_intel.py index 215a96e..8dfbe4f 100755 --- a/src/enricher/threat_intel.py +++ b/src/enricher/threat_intel.py @@ -17,7 +17,9 @@ import argparse import os import sys +import time from concurrent.futures import ThreadPoolExecutor, as_completed +from threading import Lock from dotenv import load_dotenv from rich.console import Console @@ -31,6 +33,23 @@ console = Console(file=sys.stderr) +_ENRICH_TTL = 6 * 3600 # seconds; web path only +_enrich_cache: dict[str, tuple[float, dict]] = {} +_enrich_lock = Lock() + + +def _cache_get(ip: str) -> dict | None: + with _enrich_lock: + entry = _enrich_cache.get(ip) + if entry and time.time() - entry[0] < _ENRICH_TTL: + return entry[1] + return None + + +def _cache_put(ip: str, result: dict) -> None: + with _enrich_lock: + _enrich_cache[ip] = (time.time(), result) + def enrich_ip(ip: str, offer_fp: bool = True, urls_only: bool = False) -> dict: """Run the full enrichment pipeline for a single IP. @@ -49,6 +68,11 @@ def enrich_ip(ip: str, offer_fp: bool = True, urls_only: bool = False) -> dict: "virustotal": dict | None, } """ + if not urls_only and not offer_fp: + cached = _cache_get(ip) + if cached is not None: + return cached + result: dict = { "ip": ip, "greynoise": {}, @@ -61,29 +85,41 @@ def enrich_ip(ip: str, offer_fp: bool = True, urls_only: bool = False) -> dict: _display_urls(ip) return result + if not offer_fp: + # Web path: fire all four providers in parallel — no early-exit benefit here + # since there is no FP-filter prompt to show the analyst. + providers = { + "greynoise": greynoise.check_ip, + "abuseipdb": abuseipdb.check_ip, + "shodan": shodan.check_ip, + "virustotal": virustotal.check_ip, + } + with ThreadPoolExecutor(max_workers=4) as pool: + futures = {pool.submit(fn, ip): name for name, fn in providers.items()} + for future in as_completed(futures): + result[futures[future]] = future.result() + _cache_put(ip, result) + return result + + # CLI path: GreyNoise first so we can offer an FP filter on benign IPs. console.print(f"\n[bold cyan]Enriching {ip}...[/bold cyan]") - # ---- Step 1: GreyNoise ---- gn = greynoise.check_ip(ip) result["greynoise"] = gn classification = gn["classification"] - greynoise.display(ip, gn) if classification == "benign": - if offer_fp: - add_fp = ( - input("\nGreyNoise classifies this as benign. Add FP filter? [y/N]: ") - .strip() - .lower() - ) - if add_fp == "y": - # Lazy import to avoid circular dependency when called from querier - from src.querier.fp_manager import create_filter_interactive - - name = gn.get("name", "") - hint = f"{name} — GreyNoise benign" if name else "" - create_filter_interactive(alert={"src_ip": ip}, comment_hint=hint) + add_fp = ( + input("\nGreyNoise classifies this as benign. Add FP filter? [y/N]: ").strip().lower() + ) + if add_fp == "y": + # Lazy import to avoid circular dependency when called from querier + from src.querier.fp_manager import create_filter_interactive + + name = gn.get("name", "") + hint = f"{name} — GreyNoise benign" if name else "" + create_filter_interactive(alert={"src_ip": ip}, comment_hint=hint) _display_urls(ip) return result @@ -105,12 +141,9 @@ def enrich_ip(ip: str, offer_fp: bool = True, urls_only: bool = False) -> dict: for future in as_completed(futures): result[futures[future]] = future.result() - # Display in canonical order abuseipdb.display(ip, result["abuseipdb"]) shodan.display(ip, result["shodan"]) virustotal.display(ip, result["virustotal"]) - - # ---- Step 5: Reference URLs (always) ---- _display_urls(ip) return result From efe11d5ce7c0686bb5932771c3ef017235954d0f Mon Sep 17 00:00:00 2001 From: liamadale Date: Mon, 11 May 2026 21:15:55 -0700 Subject: [PATCH 076/109] perf(querier): cache OpenSearch session and optimize query construction - Reuse the requests.Session (with connection pool) across calls as long as credentials are unchanged, avoiding TLS handshake overhead on every one of the 30+ parallel queries per page load - Add any_ip_filter to build_base_query() so ip_pivot can search an IP in either src or dest role without two separate queries - Precompute _PRIVATE_CIDR_MUST_NOT at import time; replace the per-query loop that appended broken term-on-CIDR clauses with the single correct bool/range clause - Add sort= parameter (default True) so aggregation queries can skip the sort step instead of building it and then calling body.pop("sort") - Set track_total_hits=False to skip the expensive hit-count scan - Replace sorted()[0] with max() in deduplicate_zeek() for clarity and a minor allocation reduction --- src/querier/zeek_modules/base.py | 77 +++++++++++++++++++++++++++----- 1 file changed, 66 insertions(+), 11 deletions(-) diff --git a/src/querier/zeek_modules/base.py b/src/querier/zeek_modules/base.py index 61ba9a6..3e4b758 100644 --- a/src/querier/zeek_modules/base.py +++ b/src/querier/zeek_modules/base.py @@ -99,6 +99,25 @@ class OpenSearchAuthError(RuntimeError): "ff00::/8", # multicast ] +# Precomputed once: a single must_not clause that matches any source IP in a +# private range. `term` does not evaluate CIDR notation on ip-typed fields — +# range queries with explicit network/broadcast bounds are the correct DSL. +_PRIVATE_CIDR_MUST_NOT: dict = { + "bool": { + "should": [ + { + "range": { + "source.ip": { + "gte": str(ipaddress.ip_network(cidr, strict=False).network_address), + "lte": str(ipaddress.ip_network(cidr, strict=False).broadcast_address), + } + } + } + for cidr in _PRIVATE_CIDRS + ] + } +} + # --------------------------------------------------------------------------- # Utility helpers @@ -193,12 +212,21 @@ def _cache_path(args_hash: str) -> str: # OpenSearch session + query # --------------------------------------------------------------------------- +# Module-level session cache: (url, username, password, session). +# Reusing the same Session keeps the underlying TCP/TLS connection pool alive +# across the 30+ parallel cross-protocol calls that happen per page load. +_opensearch_session_cache: tuple[str, str, str, requests.Session] | None = None + -def _opensearch_session() -> tuple: +def _opensearch_session() -> tuple[str, requests.Session]: """Return (base_url, authenticated Session). Raises OpenSearchConnectionError when credentials are not configured. + The Session is cached at module level and reused as long as credentials + remain unchanged, so the connection pool stays warm across calls. """ + global _opensearch_session_cache + opensearch_url = os.environ.get("OPENSEARCH_URL", OPENSEARCH_URL) username = os.environ.get("PISCES_USERNAME", "") password = os.environ.get("PISCES_PASSWORD", "") @@ -208,6 +236,11 @@ def _opensearch_session() -> tuple: "PISCES_USERNAME and PISCES_PASSWORD must be set — check your .env file" ) + if _opensearch_session_cache is not None: + cached_url, cached_user, cached_pass, cached_session = _opensearch_session_cache + if (cached_url, cached_user, cached_pass) == (opensearch_url, username, password): + return opensearch_url, cached_session + session = requests.Session() session.auth = (username, password) session.verify = False @@ -217,6 +250,10 @@ def _opensearch_session() -> tuple: "osd-xsrf": "true", } ) + adapter = requests.adapters.HTTPAdapter(pool_connections=4, pool_maxsize=16) + session.mount("https://", adapter) + session.mount("http://", adapter) + _opensearch_session_cache = (opensearch_url, username, password, session) return opensearch_url, session @@ -369,9 +406,11 @@ def build_base_query( public_only: bool = False, src_ip_filter: str | None = None, dest_ip_filter: str | None = None, + any_ip_filter: str | None = None, direction: str | None = None, time_from: str | None = None, time_to: str | None = None, + sort: bool = True, ) -> tuple: """Build the OpenSearch query body and request params. @@ -397,6 +436,19 @@ def build_base_query( if dest_ip_filter: must_clauses.append({"term": {"destination.ip": dest_ip_filter}}) + if any_ip_filter: + must_clauses.append( + { + "bool": { + "should": [ + {"term": {"source.ip": any_ip_filter}}, + {"term": {"destination.ip": any_ip_filter}}, + ], + "minimum_should_match": 1, + } + } + ) + if direction: must_clauses.append({"term": {"network.direction": direction}}) @@ -404,12 +456,11 @@ def build_base_query( effective_must_not = list(must_not) if public_only: - for cidr in _PRIVATE_CIDRS: - effective_must_not.append({"term": {"source.ip": cidr}}) + effective_must_not.append(_PRIVATE_CIDR_MUST_NOT) - body = { + body: dict = { "size": limit, - "sort": [{"@timestamp": {"order": "desc"}}], + "track_total_hits": False, "query": { "bool": { "filter": must_clauses, @@ -418,6 +469,8 @@ def build_base_query( }, "_source": source_fields, } + if sort: + body["sort"] = [{"@timestamp": {"order": "desc"}}] params = { "path": f"{INDEX}/_search?timeout=30s", @@ -444,7 +497,7 @@ def deduplicate_zeek(records: list, key_fn) -> list: deduped = [] for _key, group in sorted(grouped.items(), key=lambda kv: -len(kv[1])): - rep = sorted(group, key=lambda r: r["timestamp"], reverse=True)[0].copy() + rep = max(group, key=lambda r: r["timestamp"]).copy() rep["freq"] = len(group) rep["sensors"] = sorted({r["sensor"] for r in group if r.get("sensor")}) deduped.append(rep) @@ -478,12 +531,13 @@ def run_query(module, search_params: dict) -> list: extra_must, post_filters = module.build_extra_must(search_params) - # Guard src/dest ip filters for modules that don't have the field in SOURCE_FIELDS — + # Guard src/dest/any ip filters for modules that don't have the field in SOURCE_FIELDS — # a term query on a missing field returns zero results. - src_ip_for_query = search_params.get("src_ip") if "source.ip" in module.SOURCE_FIELDS else None - dest_ip_for_query = ( - search_params.get("dest_ip") if "destination.ip" in module.SOURCE_FIELDS else None - ) + has_src = "source.ip" in module.SOURCE_FIELDS + has_dest = "destination.ip" in module.SOURCE_FIELDS + src_ip_for_query = search_params.get("src_ip") if has_src else None + dest_ip_for_query = search_params.get("dest_ip") if has_dest else None + any_ip_for_query = search_params.get("any_ip") if (has_src or has_dest) else None # Over-fetch when post-filters are active so truncation still yields enough rows. requested_limit = search_params.get("limit", 500) @@ -502,6 +556,7 @@ def run_query(module, search_params: dict) -> list: public_only=search_params.get("public_only", False), src_ip_filter=src_ip_for_query, dest_ip_filter=dest_ip_for_query, + any_ip_filter=any_ip_for_query, direction=search_params.get("direction"), time_from=search_params.get("time_from"), time_to=search_params.get("time_to"), From 686b01315e77723668b80a505e365262d602fe50 Mon Sep 17 00:00:00 2001 From: liamadale Date: Mon, 11 May 2026 21:16:01 -0700 Subject: [PATCH 077/109] fix(querier): correct FilesModule IP filter flag and SuricataAlert summary type FilesModule: Malcolm does not index tx_hosts/rx_hosts arrays, so IP filtering via source.ip/destination.ip is impossible. Set SUPPORTS_IP_FILTER = False so the module is correctly excluded from ip_pivot queries rather than returning empty results silently. SuricataAlertModule: add missing SUMMARY_TYPE = "grouped" so the /api/summary/suricata_alert route dispatches to the grouped aggregation path rather than falling through to the flat summary path. --- src/querier/zeek_modules/files.py | 5 ++--- src/querier/zeek_modules/suricata_alert.py | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/querier/zeek_modules/files.py b/src/querier/zeek_modules/files.py index 2a27251..315327b 100644 --- a/src/querier/zeek_modules/files.py +++ b/src/querier/zeek_modules/files.py @@ -32,9 +32,8 @@ class FilesModule(ZeekModule): "event.dataset", ] - # source.ip is NOT in SOURCE_FIELDS — IPs come from tx_hosts/rx_hosts arrays. - # run_query() guards src_ip_filter against modules without source.ip. - SUPPORTS_IP_FILTER = True # post-filters handle it + # Malcolm does not index tx_hosts/rx_hosts, so IP filtering is impossible. + SUPPORTS_IP_FILTER = False WEB_CATEGORY = "files" WEB_ICON = "fa-file" diff --git a/src/querier/zeek_modules/suricata_alert.py b/src/querier/zeek_modules/suricata_alert.py index 6d487ad..caccb24 100644 --- a/src/querier/zeek_modules/suricata_alert.py +++ b/src/querier/zeek_modules/suricata_alert.py @@ -20,6 +20,7 @@ class SuricataAlertModule(ZeekModule): EXTRA_PARAMS = ["rule_name", "rule_category", "severity", "sid", "exclude_stream", "tag"] SUMMARY_FIELD = "rule.name" SUMMARY_PARAM = "rule_name" + SUMMARY_TYPE = "grouped" DATASETS = ["alert"] SOURCE_FIELDS = [ "@timestamp", From bcd3860d651847e7c246a52e11286e85b15db2b0 Mon Sep 17 00:00:00 2001 From: liamadale Date: Mon, 11 May 2026 21:16:13 -0700 Subject: [PATCH 078/109] perf(web): add single-flight dedup, shared thread pool, and ETag support cache.py: add claim/wait_inflight/release primitives and a raw_key() accessor. When multiple concurrent requests miss the cache for the same key, only one queries OpenSearch; all others block on an Event and read the result from cache once the leader finishes. queries.py: replace per-operation ThreadPoolExecutors (spawned and torn down on every request) with a single module-level POOL sized to min(32, cpu_count * 4). This bounds total thread count and eliminates the overhead of creating and joining a new executor for each fan-out. app.py: emit ETag headers on /api/search responses using the cache key, and return 304 Not Modified when the client presents a matching If-None-Match header and the result is still cached. --- apps/opensearch_web/cache.py | 36 ++++++++++++++++++++++ apps/opensearch_web/queries.py | 55 +++++++++++++++++++++++----------- 2 files changed, 74 insertions(+), 17 deletions(-) diff --git a/apps/opensearch_web/cache.py b/apps/opensearch_web/cache.py index d99ebe6..6066366 100644 --- a/apps/opensearch_web/cache.py +++ b/apps/opensearch_web/cache.py @@ -10,6 +10,7 @@ import hashlib import json import os +import threading import time from threading import Lock @@ -18,12 +19,47 @@ _store: dict[str, tuple[float, list]] = {} _lock = Lock() +# Single-flight: prevents duplicate in-flight queries for the same key. +_inflight: dict[str, threading.Event] = {} +_inflight_lock = Lock() + def _key(log_type: str, search_params: dict) -> str: payload = json.dumps({"log_type": log_type, **search_params}, sort_keys=True) return hashlib.md5(payload.encode()).hexdigest() +def raw_key(log_type: str, search_params: dict) -> str: + """Return the cache key string (used for ETags and single-flight coordination).""" + return _key(log_type, search_params) + + +def claim(key: str) -> threading.Event | None: + """Try to become the leader for a cache miss. Returns an Event if leader, None if follower.""" + with _inflight_lock: + if key in _inflight: + return None + event = threading.Event() + _inflight[key] = event + return event + + +def wait_inflight(key: str) -> None: + """Block until the leader for this key finishes.""" + with _inflight_lock: + event = _inflight.get(key) + if event: + event.wait() + + +def release(key: str) -> None: + """Mark a key as done and wake any threads waiting on it.""" + with _inflight_lock: + event = _inflight.pop(key, None) + if event: + event.set() + + def get(log_type: str, search_params: dict) -> list | None: """Return cached records if still within TTL, else None.""" k = _key(log_type, search_params) diff --git a/apps/opensearch_web/queries.py b/apps/opensearch_web/queries.py index be268b3..5de7eb4 100644 --- a/apps/opensearch_web/queries.py +++ b/apps/opensearch_web/queries.py @@ -1,5 +1,6 @@ """Web-layer query helpers — bridge between HTTP request params and run_query().""" +import os from collections import defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed @@ -12,6 +13,11 @@ run_query, ) +# Shared long-lived pool for all OpenSearch fan-out operations. One pool +# bounds total thread count across all concurrent requests instead of letting +# each route spawn its own unlimited pool. +POOL = ThreadPoolExecutor(max_workers=min(32, (os.cpu_count() or 4) * 4)) + def build_search_params_from_request(request, extra_keys=None) -> dict: """Build the search_params dict that run_query() expects, from an HTTP request.""" @@ -34,13 +40,29 @@ def build_search_params_from_request(request, extra_keys=None) -> dict: def cached_run_query(log_type: str, search_params: dict) -> list: - """run_query with in-memory TTL caching. Falls through to OpenSearch on miss.""" + """run_query with in-memory TTL caching and single-flight dedup. + + If two concurrent requests arrive with the same params and both miss the + cache, only one queries OpenSearch; the other waits and returns the same + result. + """ cached = wcache.get(log_type, search_params) if cached is not None: return cached - records = run_query(MODULES[log_type], search_params) - wcache.put(log_type, search_params, records) - return records + + k = wcache.raw_key(log_type, search_params) + event = wcache.claim(k) + + if event is None: + wcache.wait_inflight(k) + return wcache.get(log_type, search_params) or [] + + try: + records = run_query(MODULES[log_type], search_params) + wcache.put(log_type, search_params, records) + return records + finally: + wcache.release(k) def run_cross_protocol_query(search_params: dict) -> list: @@ -52,19 +74,18 @@ def run_cross_protocol_query(search_params: dict) -> list: ip_modules = {lt: mod for lt, mod in MODULES.items() if mod.SUPPORTS_IP_FILTER} results_by_type: dict = {} first_conn_error: Exception | None = None - with ThreadPoolExecutor(max_workers=len(ip_modules)) as ex: - futures = {ex.submit(cached_run_query, lt, search_params): lt for lt in ip_modules} - for f in as_completed(futures): - lt = futures[f] - try: - results_by_type[lt] = f.result() - except (OpenSearchConnectionError, OpenSearchAuthError) as exc: - if first_conn_error is None: - first_conn_error = exc - results_by_type[lt] = [] - except Exception as exc: - results_by_type[lt] = [] - console.print(f"[yellow]Cross-protocol query failed for {lt}: {exc}[/yellow]") + futures = {POOL.submit(cached_run_query, lt, search_params): lt for lt in ip_modules} + for f in as_completed(futures): + lt = futures[f] + try: + results_by_type[lt] = f.result() + except (OpenSearchConnectionError, OpenSearchAuthError) as exc: + if first_conn_error is None: + first_conn_error = exc + results_by_type[lt] = [] + except Exception as exc: + results_by_type[lt] = [] + console.print(f"[yellow]Cross-protocol query failed for {lt}: {exc}[/yellow]") if first_conn_error is not None: raise first_conn_error From 9431667f554e0eb5b8db4db49c8bb3b3204e7117 Mon Sep 17 00:00:00 2001 From: liamadale Date: Mon, 11 May 2026 21:16:26 -0700 Subject: [PATCH 079/109] feat(web): add src/dest/both IP role toggle to ip_pivot view The ip_pivot page previously hardcoded the pivot IP as source only. Analysts can now switch between three modes via a toggle bar: - Both: matches the IP in source.ip OR destination.ip - As Source: matches source.ip only (previous behaviour) - As Destination: matches destination.ip only The mode is carried in the ip_role query parameter and preserved across all toggle URLs. The "Full view" button and row detail URLs are updated to pass the correct field based on the active role. Table headers and cell content flip between Dst/Src IP+Port accordingly. Empty-state messages also reflect the selected role. The ip_pivot route now runs its module queries in parallel via the shared POOL instead of sequentially, matching the cross-protocol query behaviour. --- apps/opensearch_web/app.py | 227 ++++++++++++-------- apps/opensearch_web/static/pisces.css | 23 ++ apps/opensearch_web/templates/ip_pivot.html | 33 ++- 3 files changed, 186 insertions(+), 97 deletions(-) diff --git a/apps/opensearch_web/app.py b/apps/opensearch_web/app.py index cd0a7b9..ab2933a 100644 --- a/apps/opensearch_web/app.py +++ b/apps/opensearch_web/app.py @@ -1,12 +1,14 @@ """Flask application factory and route definitions for PISCES Web UI.""" import os +from concurrent.futures import as_completed from datetime import datetime, timedelta, timezone -from flask import Flask, abort, render_template, request +from flask import Flask, abort, make_response, render_template, request from apps.opensearch_web import cache as wcache from apps.opensearch_web.queries import ( + POOL, build_search_params_from_request, cached_run_query, run_cross_protocol_query, @@ -153,24 +155,56 @@ def overview(): # ------------------------------------------------------------------ @app.route("/ip/") def ip_pivot(ip: str): - search_params = build_search_params_from_request(request) - search_params["src_ip"] = ip + from urllib.parse import urlencode + + ip_role = request.args.get("ip_role", "both") + if ip_role not in ("src", "dest", "both"): + ip_role = "both" + search_params = build_search_params_from_request(request) + if ip_role == "dest": + search_params["dest_ip"] = ip + search_params["src_ip"] = None + elif ip_role == "src": + search_params["src_ip"] = ip + search_params["dest_ip"] = None + else: # both + search_params["any_ip"] = ip + search_params["src_ip"] = None + search_params["dest_ip"] = None + + # Build toggle URLs preserving all other query params + base_args = {k: v for k, v in request.args.items() if k != "ip_role"} + script_name = request.environ.get("SCRIPT_NAME", "") + src_url = f"{script_name}/ip/{ip}?ip_role=src&{urlencode(base_args)}" + dest_url = f"{script_name}/ip/{ip}?ip_role=dest&{urlencode(base_args)}" + both_url = f"{script_name}/ip/{ip}?ip_role=both&{urlencode(base_args)}" + + ip_modules = [lt for lt, mod in MODULES.items() if mod.SUPPORTS_IP_FILTER] results: dict = {} error = None - for lt, mod in MODULES.items(): - if not mod.SUPPORTS_IP_FILTER: - continue # pe, capture_loss have no src_ip to pivot on - sp = dict(search_params) + first_error: Exception | None = None + futures = {POOL.submit(cached_run_query, lt, dict(search_params)): lt for lt in ip_modules} + for f in as_completed(futures): + lt = futures[f] try: - results[lt] = cached_run_query(lt, sp) + results[lt] = f.result() except (OpenSearchConnectionError, OpenSearchAuthError) as exc: - error = str(exc) - break + if first_error is None: + first_error = exc + results[lt] = [] + except Exception: + results[lt] = [] + if first_error is not None: + error = str(first_error) return render_template( "ip_pivot.html", ip=ip, + ip_role=ip_role, + src_url=src_url, + dest_url=dest_url, + both_url=both_url, results=results, error=error, search_params=search_params, @@ -223,6 +257,14 @@ def api_search(log_type: str): mod = MODULES[log_type] extra_keys = mod.EXTRA_PARAMS search_params = build_search_params_from_request(request, extra_keys) + + etag = f'"{wcache.raw_key(log_type, search_params)}"' + if ( + request.headers.get("If-None-Match") == etag + and wcache.get(log_type, search_params) is not None + ): + return "", 304 + try: records = cached_run_query(log_type, search_params) except (OpenSearchConnectionError, OpenSearchAuthError) as exc: @@ -234,14 +276,18 @@ def api_search(log_type: str): detail_fields=mod.DETAIL_FIELDS, web_columns=mod.WEB_COLUMNS, ) - return render_template( - "partials/log_rows.html", - log_type=log_type, - records=records, - error=None, - detail_fields=mod.DETAIL_FIELDS, - web_columns=mod.WEB_COLUMNS, + resp = make_response( + render_template( + "partials/log_rows.html", + log_type=log_type, + records=records, + error=None, + detail_fields=mod.DETAIL_FIELDS, + web_columns=mod.WEB_COLUMNS, + ) ) + resp.headers["ETag"] = etag + return resp # ------------------------------------------------------------------ # GET /api/detail// — HTMX: expanded record detail @@ -462,29 +508,19 @@ def api_summary(log_type: str): direction=search_params.get("direction"), time_from=search_params.get("time_from"), time_to=search_params.get("time_to"), + sort=False, ) body["size"] = 0 - body.pop("sort", None) body.pop("_source", None) if mod.SUMMARY_TYPE == "grouped": - # Prefix-grouped aggregation: extract prefix from rule.name, - # with nested severity breakdown and top rules per group. + # Aggregate on the full rule.name field; Python groups into prefixes. + # Replaces a painless script that ran on every document server-side. body["aggs"] = { - "prefixes": { + "rules": { "terms": { - "script": { - "source": ( - "def n = doc['rule.name'].value;" - "int i = n.indexOf(' ');" - "if (i < 0) return n;" - "int j = n.indexOf(' ', i + 1);" - "if (j < 0) return n.substring(0, i);" - "return n.substring(0, j);" - ), - "lang": "painless", - }, - "size": 50, + "field": mod.SUMMARY_FIELD, + "size": 500, "order": {"_count": "desc"}, }, "aggs": { @@ -494,13 +530,6 @@ def api_summary(log_type: str): "size": 5, } }, - "top_rules": { - "terms": { - "field": "rule.name", - "size": 10, - "order": {"_count": "desc"}, - } - }, }, } } @@ -517,34 +546,45 @@ def api_summary(log_type: str): raw = query_opensearch(body, params) - is_inline = request.args.get("inline") == "1" - if mod.SUMMARY_TYPE == "grouped": - groups = [] + # Group the flat rule.name buckets into two-word prefixes in Python. + groups_by_prefix: dict[str, dict] = {} if raw: - for b in raw.get("aggregations", {}).get("prefixes", {}).get("buckets", []): + for b in raw.get("aggregations", {}).get("rules", {}).get("buckets", []): + rule_name: str = b["key"] + count: int = b["doc_count"] sev_map = { s["key"]: s["doc_count"] for s in b.get("by_severity", {}).get("buckets", []) } - min_sev = min(sev_map.keys()) if sev_map else 3 - groups.append( - { - "prefix": b["key"], - "count": b["doc_count"], - "min_severity": min_sev, - "sev": sev_map, - "top_rules": [ - {"name": r["key"], "count": r["doc_count"]} - for r in b.get("top_rules", {}).get("buckets", []) - ], + # Mirror the painless prefix logic: up to the second space + i = rule_name.find(" ") + if i < 0: + prefix = rule_name + else: + j = rule_name.find(" ", i + 1) + prefix = rule_name[:i] if j < 0 else rule_name[:j] + + if prefix not in groups_by_prefix: + groups_by_prefix[prefix] = { + "prefix": prefix, + "count": 0, + "sev": {}, + "top_rules": [], } - ) - template = ( - "partials/summary_grouped.html" if is_inline else "partials/summary_grouped.html" - ) + g = groups_by_prefix[prefix] + g["count"] += count + for sev, cnt in sev_map.items(): + g["sev"][sev] = g["sev"].get(sev, 0) + cnt + g["top_rules"].append({"name": rule_name, "count": count}) + + groups = sorted(groups_by_prefix.values(), key=lambda g: -g["count"])[:50] + for g in groups: + g["top_rules"] = sorted(g["top_rules"], key=lambda r: -r["count"])[:10] + g["min_severity"] = min(g["sev"].keys()) if g["sev"] else 3 + return render_template( - template, + "partials/summary_grouped.html", groups=groups, summary_param=mod.SUMMARY_PARAM, ) @@ -589,9 +629,9 @@ def api_sensor_summary(): direction=None, time_from=search_params.get("time_from"), time_to=search_params.get("time_to"), + sort=False, ) body["size"] = 0 - body.pop("sort", None) body.pop("_source", None) body["aggs"] = { "sensors": { @@ -670,8 +710,6 @@ def investigate_view(src_ip: str, dest_ip: str): # ------------------------------------------------------------------ @app.route("/api/investigate/profiles") def api_investigate_profiles(): - from concurrent.futures import ThreadPoolExecutor - from src.enricher.threat_intel import enrich_ip from src.profiler.device_profiler import profile_device from src.profiler.public_ip_profiler import profile_public_ip @@ -711,10 +749,10 @@ def _do_dest(): except Exception as exc: dest_error = str(exc) - with ThreadPoolExecutor(max_workers=2) as pool: - pool.submit(_do_src) - f_dest = pool.submit(_do_dest) - f_dest.result() # wait for both + f_src = POOL.submit(_do_src) + f_dest = POOL.submit(_do_dest) + f_src.result() + f_dest.result() def _enrich_urls(ip: str) -> dict: from src.enricher import abuseipdb, greynoise, shodan, virustotal @@ -756,6 +794,7 @@ def api_investigate_auth(): kerberos, ntlm = query_auth_history(src_ip, dest_ip, sensor, time_range) error = None except Exception as exc: + app.logger.exception("api_investigate_auth failed") kerberos, ntlm, error = [], [], str(exc) return render_template( @@ -782,6 +821,7 @@ def api_investigate_chain(): chain = query_attack_chain(src_ip, sensor, time_range) error = None except Exception as exc: + app.logger.exception("api_investigate_chain failed") chain, error = [], str(exc) return render_template( @@ -820,6 +860,7 @@ def api_investigate_notices(): records.sort(key=lambda r: r.get("timestamp", "")) error = None except Exception as exc: + app.logger.exception("api_investigate_notices failed") records, error = [], str(exc) return render_template( @@ -862,6 +903,7 @@ def api_investigate_suricata(): records.sort(key=lambda r: r.get("timestamp", "")) error = None except Exception as exc: + app.logger.exception("api_investigate_suricata failed") records, error = [], str(exc) return render_template( @@ -898,6 +940,7 @@ def api_investigate_tickets(): dest_tickets = [t for t in dest_tickets if t.get("id") not in src_ids] error = None except Exception as exc: + app.logger.exception("api_investigate_tickets failed") src_tickets, dest_tickets, error = [], [], str(exc) return render_template( @@ -914,9 +957,6 @@ def api_investigate_tickets(): # ------------------------------------------------------------------ @app.route("/api/investigate/timeline") def api_investigate_timeline(): - from concurrent.futures import ThreadPoolExecutor - from concurrent.futures import as_completed as _as_completed - from src.correlator.incident_context import ( IncidentContext, build_timeline, @@ -940,20 +980,21 @@ def _auth() -> tuple[list[dict], list[dict]]: def _chain() -> list[dict]: return query_attack_chain(src_ip, sensor, time_range) - with ThreadPoolExecutor(max_workers=2) as pool: - f_auth = pool.submit(_auth) - f_chain = pool.submit(_chain) - for fut in _as_completed([f_auth, f_chain]): - if fut is f_auth: - try: - kerberos, ntlm = fut.result() - except Exception as exc: - errors["auth"] = str(exc) - else: - try: - chain = fut.result() - except Exception as exc: - errors["chain"] = str(exc) + f_auth = POOL.submit(_auth) + f_chain = POOL.submit(_chain) + for fut in as_completed([f_auth, f_chain]): + if fut is f_auth: + try: + kerberos, ntlm = fut.result() + except Exception as exc: + app.logger.exception("api_investigate_timeline auth failed") + errors["auth"] = str(exc) + else: + try: + chain = fut.result() + except Exception as exc: + app.logger.exception("api_investigate_timeline chain failed") + errors["chain"] = str(exc) ctx = IncidentContext( trigger_type="ip_pair", @@ -980,9 +1021,6 @@ def _chain() -> list[dict]: # ------------------------------------------------------------------ @app.route("/api/investigate/log_counts") def api_investigate_log_counts(): - from concurrent.futures import ThreadPoolExecutor - from concurrent.futures import as_completed as _as_completed - src_ip = request.args.get("src_ip", "") sensor = request.args.get("sensor", "all") time_range = request.args.get("time_range", "now-24h") @@ -1002,14 +1040,13 @@ def api_investigate_log_counts(): } counts: dict[str, int] = {} - with ThreadPoolExecutor(max_workers=len(MODULES)) as pool: - futures = {pool.submit(cached_run_query, lt, dict(search_params)): lt for lt in MODULES} - for fut in _as_completed(futures): - lt = futures[fut] - try: - counts[lt] = len(fut.result()) - except Exception: - counts[lt] = 0 + futures = {POOL.submit(cached_run_query, lt, dict(search_params)): lt for lt in MODULES} + for fut in as_completed(futures): + lt = futures[fut] + try: + counts[lt] = len(fut.result()) + except Exception: + counts[lt] = 0 return render_template( "partials/investigate_log_counts.html", diff --git a/apps/opensearch_web/static/pisces.css b/apps/opensearch_web/static/pisces.css index 7a4acf4..aa09212 100644 --- a/apps/opensearch_web/static/pisces.css +++ b/apps/opensearch_web/static/pisces.css @@ -1346,6 +1346,29 @@ tr.detail-row td { .empty-note { color: var(--on-surface-dim); font-size: 0.82rem; font-style: italic; } +.role-toggle { + display: flex; + align-items: center; + gap: 0.5rem; + margin-bottom: 1rem; +} + +.role-toggle-label { + font-size: 0.8rem; + color: var(--on-surface-dim); +} + +.btn-active { + background: var(--primary); + color: var(--on-primary); + border-color: var(--primary); +} + +.btn-active:hover { + background: var(--primary); + opacity: 0.9; +} + /* ── 13. Utilities ────────────────────────────────────── */ diff --git a/apps/opensearch_web/templates/ip_pivot.html b/apps/opensearch_web/templates/ip_pivot.html index 98b2ab5..a4d44a6 100644 --- a/apps/opensearch_web/templates/ip_pivot.html +++ b/apps/opensearch_web/templates/ip_pivot.html @@ -26,6 +26,19 @@

IP Pivot: {{ ip }}

{% endif %}
+
+ IP role: + + Both + + + As Source + + + As Destination + +
+
@@ -45,7 +58,13 @@

IP Pivot: {{ ip }}

{{ lt }} {{ records|length }} record(s) + {% if ip_role == 'dest' %} + + {% elif ip_role == 'both' %} + + {% else %} + {% endif %} Full view
@@ -64,8 +83,13 @@

IP Pivot: {{ ip }}

# Time Sensor + {% if ip_role == 'dest' %} + Src IP + Src Port + {% else %} Dst IP Dst Port + {% endif %} {% for header, _ in web_columns %} {{ header }} {% endfor %} @@ -76,7 +100,7 @@

IP Pivot: {{ ip }}

{% for rec in records %} {% set idx = loop.index %} IP Pivot: {{ ip }} {% set sensors = rec.get('sensors') %} {% if sensors %}{{ sensors | join(', ') }}{% else %}{{ rec.get('sensor', '—') }}{% endif %} + {% if ip_role == 'dest' %} + {{ rec.get('src_ip', '—') }} + {{ rec.get('src_port', '—') }} + {% else %} {{ rec.get('dest_ip', '—') }} {{ rec.get('dest_port', '—') }} + {% endif %} {% for _, fn in web_columns %} {{ fn(rec) }} {% endfor %} @@ -101,7 +130,7 @@

IP Pivot: {{ ip }}

{% else %} -

No {{ lt }} records for this IP in the selected window.

+

No {{ lt }} records for this IP as {{ ip_role }}.

{% endif %} {% endfor %} From 868595789d1e7540d25f6ff2fc854dc8dd5a7ec6 Mon Sep 17 00:00:00 2001 From: liamadale Date: Mon, 11 May 2026 21:16:29 -0700 Subject: [PATCH 080/109] chore: move pytest out of main dependencies into dev dependencies pytest belongs in the dev/test dependency group, not in the runtime dependency list that gets installed in production environments. --- pyproject.toml | 1 - uv.lock | 2 -- 2 files changed, 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 26991a2..93fb2df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,6 @@ dependencies = [ "plotext>=5.3.2", "cryptography>=46.0.7", "pygments>=2.20.0", - "pytest>=9.0.3", "python-multipart>=0.0.26", ] diff --git a/uv.lock b/uv.lock index 8c35a99..fe8ef77 100644 --- a/uv.lock +++ b/uv.lock @@ -1490,7 +1490,6 @@ dependencies = [ { name = "orjson" }, { name = "plotext" }, { name = "pygments" }, - { name = "pytest" }, { name = "python-dotenv" }, { name = "python-multipart" }, { name = "pyyaml" }, @@ -1538,7 +1537,6 @@ requires-dist = [ { name = "pyasn", marker = "extra == 'all'", specifier = ">=1.6.2" }, { name = "pyasn", marker = "extra == 'offline-enrichment'", specifier = ">=1.6.2" }, { name = "pygments", specifier = ">=2.20.0" }, - { name = "pytest", specifier = ">=9.0.3" }, { name = "python-dotenv", specifier = ">=1.2.2" }, { name = "python-multipart", specifier = ">=0.0.26" }, { name = "pyyaml", specifier = ">=6.0.3" }, From 88261613bda007f37f65ec9ead665661fb205133 Mon Sep 17 00:00:00 2001 From: liamadale Date: Mon, 11 May 2026 21:39:40 -0700 Subject: [PATCH 081/109] fix(web): log exceptions in bare except handlers that were silently swallowing tracebacks Three handlers in the ip_pivot fan-out, profile worker threads, and log-count fan-out were catching Exception without logging, making failures invisible in production. Add app.logger.exception() to each so full tracebacks appear in Flask logs. --- apps/opensearch_web/app.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/opensearch_web/app.py b/apps/opensearch_web/app.py index ab2933a..65a6032 100644 --- a/apps/opensearch_web/app.py +++ b/apps/opensearch_web/app.py @@ -194,6 +194,7 @@ def ip_pivot(ip: str): first_error = exc results[lt] = [] except Exception: + app.logger.exception("ip_pivot query failed for %s", lt) results[lt] = [] if first_error is not None: error = str(first_error) @@ -736,6 +737,7 @@ def _do_src(): src_profile = profile_public_ip(src_ip, time_range=time_range) src_enrichment = enrich_ip(src_ip, offer_fp=False) except Exception as exc: + app.logger.exception("api_investigate_profiles src failed") src_error = str(exc) def _do_dest(): @@ -747,6 +749,7 @@ def _do_dest(): dest_profile = profile_public_ip(dest_ip, time_range=time_range) dest_enrichment = enrich_ip(dest_ip, offer_fp=False) except Exception as exc: + app.logger.exception("api_investigate_profiles dest failed") dest_error = str(exc) f_src = POOL.submit(_do_src) @@ -1046,6 +1049,7 @@ def api_investigate_log_counts(): try: counts[lt] = len(fut.result()) except Exception: + app.logger.exception("log count query failed for %s", lt) counts[lt] = 0 return render_template( From 418322194e8b037f481c9744400acc25cf042eb3 Mon Sep 17 00:00:00 2001 From: liamadale Date: Mon, 11 May 2026 21:39:50 -0700 Subject: [PATCH 082/109] refactor(web): hoist lazy imports, enable static asset caching, remove dead code - Move incident_context imports to module top, removing repeated in-function imports across api_investigate_auth, _chain, and _timeline handlers - Set send_file_max_age_default = 31_536_000 so browsers cache versioned static assets for one year instead of re-fetching on every page load - Delete unreachable second render_template() call in api_investigate_suricata that could never be reached after the first return --- apps/opensearch_web/app.py | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/apps/opensearch_web/app.py b/apps/opensearch_web/app.py index 65a6032..0700c69 100644 --- a/apps/opensearch_web/app.py +++ b/apps/opensearch_web/app.py @@ -20,6 +20,12 @@ make_shared_static_blueprint, ) from apps.shared.jinja_globals import register_shared_helpers +from src.correlator.incident_context import ( + IncidentContext, + build_timeline, + query_attack_chain, + query_auth_history, +) from src.querier.zeek_modules import ( CATEGORY_LABELS, CATEGORY_ORDER, @@ -78,6 +84,7 @@ def _resolve_city(request): # type: ignore[no-untyped-def] def create_app() -> Flask: app = Flask(__name__, static_folder="static", template_folder="templates") + app.send_file_max_age_default = 31_536_000 # 1 year for versioned static assets register_shared_helpers(app) @@ -786,8 +793,6 @@ def _enrich_urls(ip: str) -> dict: # ------------------------------------------------------------------ @app.route("/api/investigate/auth") def api_investigate_auth(): - from src.correlator.incident_context import query_auth_history - src_ip = request.args.get("src_ip", "") dest_ip = request.args.get("dest_ip", "") sensor = request.args.get("sensor", "all") @@ -814,8 +819,6 @@ def api_investigate_auth(): # ------------------------------------------------------------------ @app.route("/api/investigate/chain") def api_investigate_chain(): - from src.correlator.incident_context import query_attack_chain - src_ip = request.args.get("src_ip", "") sensor = request.args.get("sensor", "all") time_range = request.args.get("time_range", "now-24h") @@ -917,14 +920,6 @@ def api_investigate_suricata(): error=error, ) - return render_template( - "partials/investigate_suricata.html", - alerts=records, - src_ip=src_ip, - dest_ip=dest_ip, - error=error, - ) - # ------------------------------------------------------------------ # GET /api/investigate/tickets — HTMX: Mantis tickets for src + dest # ------------------------------------------------------------------ @@ -960,13 +955,6 @@ def api_investigate_tickets(): # ------------------------------------------------------------------ @app.route("/api/investigate/timeline") def api_investigate_timeline(): - from src.correlator.incident_context import ( - IncidentContext, - build_timeline, - query_attack_chain, - query_auth_history, - ) - src_ip = request.args.get("src_ip", "") dest_ip = request.args.get("dest_ip", "") sensor = request.args.get("sensor", "all") From 288f7e38f617b46499d702e7cd10168a8fabbaee Mon Sep 17 00:00:00 2001 From: liamadale Date: Mon, 11 May 2026 21:40:01 -0700 Subject: [PATCH 083/109] perf(querier): reduce redundant work in filter loading, remapping, and post-filtering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit filter_loader: add a 5-second TTL gate so repeated load_filters() calls within a request burst skip the os.walk mtime scan entirely. base: cache the remapped filter clause list by object identity so _remap_clause is not re-run on every run_query() call when filters have not changed. base: gate disk cache writes behind use_cache=True — the web path no longer writes multi-MB JSON to disk unconditionally on every request. base: compose post-filters into a single all() predicate so each record is walked once instead of once per filter. base: short-circuit sensor-set construction in deduplicate_zeek for single- record groups, avoiding a set comprehension on the common case. base: register atexit.register(session.close) on the shared OpenSearch session so the connection pool is cleanly torn down on interpreter exit. --- src/querier/filter_loader.py | 13 ++++++++++--- src/querier/zeek_modules/base.py | 27 ++++++++++++++++++--------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/querier/filter_loader.py b/src/querier/filter_loader.py index 8ba3914..23af5fd 100755 --- a/src/querier/filter_loader.py +++ b/src/querier/filter_loader.py @@ -8,12 +8,15 @@ import os import sys +import time import yaml -# Module-level cache: (filters_dir, municipality) → {"mtime": float, "result": dict} +# Module-level cache: (filters_dir, municipality) → {mtime, result, checked} _filter_cache: dict[tuple, dict] = {} +_MTIME_TTL = 5.0 # seconds — skip the directory walk when cache is this fresh + def _max_mtime(filters_dir: str) -> float: """Return the highest mtime across all *.yaml files under filters_dir.""" @@ -49,9 +52,13 @@ def load_filters( } """ cache_key = (filters_dir, municipality) - current_mtime = _max_mtime(filters_dir) + now = time.monotonic() cached = _filter_cache.get(cache_key) + if cached is not None and (now - cached["checked"]) < _MTIME_TTL: + return cached["result"] + current_mtime = _max_mtime(filters_dir) if cached is not None and cached["mtime"] == current_mtime: + cached["checked"] = now return cached["result"] must_not_clauses: list[dict] = [] @@ -117,7 +124,7 @@ def load_filters( "filter_count": filter_count, "errors": errors, } - _filter_cache[cache_key] = {"mtime": current_mtime, "result": result} + _filter_cache[cache_key] = {"mtime": current_mtime, "result": result, "checked": now} return result diff --git a/src/querier/zeek_modules/base.py b/src/querier/zeek_modules/base.py index 3e4b758..9fa70c5 100644 --- a/src/querier/zeek_modules/base.py +++ b/src/querier/zeek_modules/base.py @@ -6,6 +6,7 @@ (conn, dns, http, …) import from this file and the ZeekModule base class. """ +import atexit import hashlib import ipaddress import json @@ -254,6 +255,7 @@ def _opensearch_session() -> tuple[str, requests.Session]: session.mount("https://", adapter) session.mount("http://", adapter) _opensearch_session_cache = (opensearch_url, username, password, session) + atexit.register(session.close) return opensearch_url, session @@ -378,16 +380,20 @@ def display_profile(raw: dict) -> None: # --------------------------------------------------------------------------- +# Cache: (raw_must_not, remapped) — invalidates when load_filters returns a new list object. +_remap_cache: tuple[list, list] | None = None + + def load_with_remap(filters_dir: str) -> tuple: """Load filters and remap field names. Returns (must_not, fcount, errors).""" + global _remap_cache from src.querier.filter_loader import load_filters filter_result = load_filters(filters_dir) - raw_must_not = filter_result["must_not"] - fcount = filter_result["filter_count"] - errors = filter_result["errors"] - must_not = [_remap_clause(c) for c in raw_must_not] - return must_not, fcount, errors + raw = filter_result["must_not"] + if _remap_cache is None or _remap_cache[0] is not raw: + _remap_cache = (raw, [_remap_clause(c) for c in raw]) + return _remap_cache[1], filter_result["filter_count"], filter_result["errors"] # --------------------------------------------------------------------------- @@ -499,7 +505,10 @@ def deduplicate_zeek(records: list, key_fn) -> list: for _key, group in sorted(grouped.items(), key=lambda kv: -len(kv[1])): rep = max(group, key=lambda r: r["timestamp"]).copy() rep["freq"] = len(group) - rep["sensors"] = sorted({r["sensor"] for r in group if r.get("sensor")}) + if len(group) == 1: + rep["sensors"] = [rep["sensor"]] if rep.get("sensor") else [] + else: + rep["sensors"] = sorted({r["sensor"] for r in group if r.get("sensor")}) deduped.append(rep) return deduped @@ -582,7 +591,7 @@ def run_query(module, search_params: dict) -> list: f" ({search_params.get('time_range', 'now-24h')})...[/dim]" ) raw = query_opensearch(body, params) - if not search_params.get("profile"): + if use_cache: _save_cache(raw, cpath) if search_params.get("profile"): @@ -601,8 +610,8 @@ def run_query(module, search_params: dict) -> list: # Apply post-filters (over-fetch strategy: fetch 3× then truncate). if post_filters: - for pf in post_filters: - records = [r for r in records if pf(r)] + keep = lambda r: all(pf(r) for pf in post_filters) # noqa: E731 + records = [r for r in records if keep(r)] if len(records) < requested_limit: console.print( f"[dim]Showing {len(records)}/{requested_limit} after post-filtering — " From a7c457ffdc0b36d40d713567d9e598d3d56a93a7 Mon Sep 17 00:00:00 2001 From: liamadale Date: Mon, 11 May 2026 21:40:09 -0700 Subject: [PATCH 084/109] perf(enricher): add retry adapter, shared console, and atexit cleanup to all enrichers All four enrichers (GreyNoise, AbuseIPDB, Shodan, VirusTotal) now share the same console instance from src.utils.terminal rather than each constructing a private Console(file=sys.stderr). Each session adapter now includes a urllib3 Retry policy (2 retries, 0.5s backoff, status_forcelist 429/502/503/504) to handle transient rate-limit and gateway errors without surfacing them as failures to the caller. atexit.register(_session.close) is wired on each session so HTTP connection pools are released cleanly on interpreter exit. --- src/enricher/abuseipdb.py | 15 ++++++++++----- src/enricher/greynoise.py | 15 ++++++++++----- src/enricher/shodan.py | 15 ++++++++++----- src/enricher/virustotal.py | 15 ++++++++++----- 4 files changed, 40 insertions(+), 20 deletions(-) diff --git a/src/enricher/abuseipdb.py b/src/enricher/abuseipdb.py index 9f03a70..7d28140 100644 --- a/src/enricher/abuseipdb.py +++ b/src/enricher/abuseipdb.py @@ -4,21 +4,26 @@ Returns raw API data for analyst interpretation — no threshold logic applied. """ +import atexit import os -import sys import requests +import urllib3 from rich import box -from rich.console import Console from rich.table import Table +from src.utils.terminal import console + _BASE_URL = "https://api.abuseipdb.com/api/v2/check" URL = "https://www.abuseipdb.com/check/{ip}" -console = Console(file=sys.stderr) - +_retry = urllib3.util.Retry(total=2, backoff_factor=0.5, status_forcelist=(429, 502, 503, 504)) _session = requests.Session() -_session.mount("https://", requests.adapters.HTTPAdapter(pool_connections=4, pool_maxsize=8)) +_session.mount( + "https://", + requests.adapters.HTTPAdapter(max_retries=_retry, pool_connections=4, pool_maxsize=8), +) +atexit.register(_session.close) def check_ip(ip: str, max_age_days: int = 90) -> dict: diff --git a/src/enricher/greynoise.py b/src/enricher/greynoise.py index 8175c22..5040fc4 100644 --- a/src/enricher/greynoise.py +++ b/src/enricher/greynoise.py @@ -4,21 +4,26 @@ Returns classification: benign | malicious | not_found """ +import atexit import os -import sys import requests +import urllib3 from rich import box -from rich.console import Console from rich.table import Table +from src.utils.terminal import console + _BASE_URL = "https://api.greynoise.io/v3/community" URL = "https://viz.greynoise.io/ip/{ip}" -console = Console(file=sys.stderr) - +_retry = urllib3.util.Retry(total=2, backoff_factor=0.5, status_forcelist=(429, 502, 503, 504)) _session = requests.Session() -_session.mount("https://", requests.adapters.HTTPAdapter(pool_connections=4, pool_maxsize=8)) +_session.mount( + "https://", + requests.adapters.HTTPAdapter(max_retries=_retry, pool_connections=4, pool_maxsize=8), +) +atexit.register(_session.close) def check_ip(ip: str) -> dict: diff --git a/src/enricher/shodan.py b/src/enricher/shodan.py index f5e6616..da45d71 100644 --- a/src/enricher/shodan.py +++ b/src/enricher/shodan.py @@ -4,21 +4,26 @@ Returns open ports, org, country, ISP, OS, hostnames, and known vulns. """ +import atexit import os -import sys import requests +import urllib3 from rich import box -from rich.console import Console from rich.table import Table +from src.utils.terminal import console + _BASE_URL = "https://api.shodan.io/shodan/host" URL = "https://www.shodan.io/search?query={ip}" -console = Console(file=sys.stderr) - +_retry = urllib3.util.Retry(total=2, backoff_factor=0.5, status_forcelist=(429, 502, 503, 504)) _session = requests.Session() -_session.mount("https://", requests.adapters.HTTPAdapter(pool_connections=4, pool_maxsize=8)) +_session.mount( + "https://", + requests.adapters.HTTPAdapter(max_retries=_retry, pool_connections=4, pool_maxsize=8), +) +atexit.register(_session.close) def check_ip(ip: str) -> dict: diff --git a/src/enricher/virustotal.py b/src/enricher/virustotal.py index faf34ff..43d3196 100644 --- a/src/enricher/virustotal.py +++ b/src/enricher/virustotal.py @@ -8,23 +8,28 @@ display_hash()— Rich table for hash results """ +import atexit import os -import sys import requests +import urllib3 from rich import box -from rich.console import Console from rich.table import Table +from src.utils.terminal import console + _BASE_URL = "https://www.virustotal.com/api/v3/ip_addresses" _HASH_BASE_URL = "https://www.virustotal.com/api/v3/files" URL = "https://www.virustotal.com/gui/ip-address/{ip}" HASH_URL = "https://www.virustotal.com/gui/file/{hash}" -console = Console(file=sys.stderr) - +_retry = urllib3.util.Retry(total=2, backoff_factor=0.5, status_forcelist=(429, 502, 503, 504)) _session = requests.Session() -_session.mount("https://", requests.adapters.HTTPAdapter(pool_connections=4, pool_maxsize=8)) +_session.mount( + "https://", + requests.adapters.HTTPAdapter(max_retries=_retry, pool_connections=4, pool_maxsize=8), +) +atexit.register(_session.close) def check_ip(ip: str) -> dict: From 4f43e46ee95224359de81f4032066f99288deac5 Mon Sep 17 00:00:00 2001 From: liamadale Date: Mon, 11 May 2026 21:57:58 -0700 Subject: [PATCH 085/109] refactor(querier): split monolithic base.py into focused modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit base.py grew to 1100+ lines handling HTTP I/O, query construction, deduplication, CLI interaction, and the ZeekModule ABC. Split into: - client.py — httpx.Client / AsyncClient, query_opensearch{,_async} - builder.py — build_base_query, FIELD_MAP, TIME_RANGES, is_private - runner.py — run_query, run_query_async, deduplicate_zeek, load_with_remap - module.py — ZeekModule ABC - cli_loop.py — interactive_loop and CLI-only helpers base.py is now a thin backwards-compat shim so all protocol modules continue to work without changes to their relative imports. --- src/querier/builder.py | 203 +++++ src/querier/cli_loop.py | 492 ++++++++++++ src/querier/client.py | 172 +++++ src/querier/module.py | 113 +++ src/querier/runner.py | 253 +++++++ src/querier/zeek_modules/base.py | 1210 +++--------------------------- 6 files changed, 1327 insertions(+), 1116 deletions(-) create mode 100644 src/querier/builder.py create mode 100644 src/querier/cli_loop.py create mode 100644 src/querier/client.py create mode 100644 src/querier/module.py create mode 100644 src/querier/runner.py diff --git a/src/querier/builder.py b/src/querier/builder.py new file mode 100644 index 0000000..b24d287 --- /dev/null +++ b/src/querier/builder.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""OpenSearch DSL query construction: field remapping and query body building.""" + +import ipaddress + +from src.querier.client import INDEX + +# Field name translation: Kibana convention → Malcolm/Zeek field +FIELD_MAP = { + "src_ip": "source.ip", + "dest_ip": "destination.ip", + "src_port": "source.port", + "dest_port": "destination.port", + "app_proto": "network.protocol", + "clientID": "host.name", +} + +TIME_RANGES = [ + "now-15m", + "now-30m", + "now-1h", + "now-3h", + "now-6h", + "now-12h", + "now-24h", + "now-2d", + "now-3d", + "now-7d", + "now-14d", + "now-30d", +] + +# Non-routable CIDRs excluded by --public-only. +_PRIVATE_CIDRS = [ + # IPv4 + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + "127.0.0.0/8", + "169.254.0.0/16", # link-local / APIPA + # IPv6 + "::1/128", # loopback + "fe80::/10", # link-local + "fc00::/7", # unique-local (fd00::/8 etc.) + "ff00::/8", # multicast +] + +# Precomputed once: a single must_not clause that matches any source IP in a +# private range. `term` does not evaluate CIDR notation on ip-typed fields — +# range queries with explicit network/broadcast bounds are the correct DSL. +_PRIVATE_CIDR_MUST_NOT: dict = { + "bool": { + "should": [ + { + "range": { + "source.ip": { + "gte": str(ipaddress.ip_network(cidr, strict=False).network_address), + "lte": str(ipaddress.ip_network(cidr, strict=False).broadcast_address), + } + } + } + for cidr in _PRIVATE_CIDRS + ] + } +} + + +def is_private(ip: str) -> bool: + """Return True if *ip* falls within any RFC-1918 / non-routable range.""" + try: + addr = ipaddress.ip_address(ip) + return any(addr in ipaddress.ip_network(cidr) for cidr in _PRIVATE_CIDRS) + except ValueError: + return False + + +def _remap_clause(clause: dict) -> dict: + """Recursively remap Kibana field names to Malcolm/Zeek field names. + + Walks a DSL clause dict and renames any key found in FIELD_MAP. + Handles term, terms, range, match_phrase, bool (recurses into + must/must_not/should/filter). Pure function — returns a new dict. + """ + if not isinstance(clause, dict): + return clause + + result = {} + for key, value in clause.items(): + if key in ( + "term", + "terms", + "range", + "match_phrase", + "match", + "wildcard", + "prefix", + "regexp", + ): + remapped_inner = {} + for field, field_val in value.items(): + new_field = FIELD_MAP.get(field, field) + remapped_inner[new_field] = field_val + result[key] = remapped_inner + elif key == "bool": + new_bool = {} + for bool_key, bool_val in value.items(): + if bool_key in ("must", "must_not", "should", "filter"): + if isinstance(bool_val, list): + new_bool[bool_key] = [_remap_clause(c) for c in bool_val] + else: + new_bool[bool_key] = _remap_clause(bool_val) + else: + new_bool[bool_key] = bool_val + result[key] = new_bool + else: + result[key] = value + + return result + + +def build_base_query( + must_not: list, + extra_must: list, + source_fields: list, + limit: int, + time_range: str, + sensors: list | None, + datasets: list, + public_only: bool = False, + src_ip_filter: str | None = None, + dest_ip_filter: str | None = None, + any_ip_filter: str | None = None, + direction: str | None = None, + time_from: str | None = None, + time_to: str | None = None, + sort: bool = True, +) -> tuple: + """Build the OpenSearch query body and request params. + + datasets: list of event.dataset values, or ["all"] to omit the filter. + time_from/time_to: absolute ISO timestamps; when both are set they + override the relative *time_range* parameter. + """ + if time_from and time_to: + ts_clause = {"range": {"@timestamp": {"gte": time_from, "lte": time_to}}} + else: + ts_clause = {"range": {"@timestamp": {"gte": time_range, "lte": "now"}}} + must_clauses: list = [ts_clause] + + if datasets and datasets != ["all"]: + must_clauses.append({"terms": {"event.dataset": datasets}}) + + if sensors: + must_clauses.append({"terms": {"host.name": sensors}}) + + if src_ip_filter: + must_clauses.append({"term": {"source.ip": src_ip_filter}}) + + if dest_ip_filter: + must_clauses.append({"term": {"destination.ip": dest_ip_filter}}) + + if any_ip_filter: + must_clauses.append( + { + "bool": { + "should": [ + {"term": {"source.ip": any_ip_filter}}, + {"term": {"destination.ip": any_ip_filter}}, + ], + "minimum_should_match": 1, + } + } + ) + + if direction: + must_clauses.append({"term": {"network.direction": direction}}) + + must_clauses.extend(extra_must) + + effective_must_not = list(must_not) + if public_only: + effective_must_not.append(_PRIVATE_CIDR_MUST_NOT) + + body: dict = { + "size": limit, + "track_total_hits": False, + "query": { + "bool": { + "filter": must_clauses, + "must_not": effective_must_not, + } + }, + "_source": source_fields, + } + if sort: + body["sort"] = [{"@timestamp": {"order": "desc"}}] + + params = { + "path": f"{INDEX}/_search?timeout=30s", + "method": "POST", + } + + return body, params diff --git a/src/querier/cli_loop.py b/src/querier/cli_loop.py new file mode 100644 index 0000000..b209bdc --- /dev/null +++ b/src/querier/cli_loop.py @@ -0,0 +1,492 @@ +#!/usr/bin/env python3 +"""Interactive CLI loop, profile display, and OpenSearch diagnostic helpers.""" + +import json +import readline # noqa: F401 — enables arrow-key history in input() +from collections import defaultdict + +import httpx +from rich import box +from rich.table import Table + +from src.querier.client import ( + INDEX, + OpenSearchAuthError, + OpenSearchConnectionError, + _opensearch_session, + console, + query_opensearch, +) +from src.querier.runner import run_query +from src.utils.terminal import confirm_exit, prompt + +# --------------------------------------------------------------------------- +# Profile display +# --------------------------------------------------------------------------- + + +def _walk_clauses(node: dict, acc: list) -> None: + """DFS walk of an ES profile query node, collecting (type, description, time_ms).""" + if not isinstance(node, dict): + return + time_ms = node.get("time_in_nanos", 0) / 1_000_000 + acc.append((node.get("type", ""), node.get("description", ""), time_ms)) + for child in node.get("children", []): + _walk_clauses(child, acc) + + +def display_profile(raw: dict) -> None: + """Render OpenSearch profile data: per-shard timing and slowest query clauses.""" + profile = raw.get("profile") + if not profile: + console.print("[yellow]No profile data in response.[/yellow]") + return + + shards = profile.get("shards", []) + if not shards: + console.print("[yellow]Profile present but no shard data.[/yellow]") + return + + # --- Shard summary table --- + shard_table = Table(title="Profile — per-shard timing", box=box.SIMPLE_HEAVY) + shard_table.add_column("Shard", style="cyan", no_wrap=True) + shard_table.add_column("Query (ms)", justify="right") + shard_table.add_column("Fetch (ms)", justify="right") + + total_query_ms = 0.0 + total_fetch_ms = 0.0 + all_clauses: list = [] + + for shard in shards: + shard_id = shard.get("id", "?") + query_ms = 0.0 + fetch_ms = 0.0 + + for search in shard.get("searches", []): + for q in search.get("query", []): + query_ms += q.get("time_in_nanos", 0) / 1_000_000 + _walk_clauses(q, all_clauses) + for collector in search.get("collector", []): + fetch_ms += collector.get("time_in_nanos", 0) / 1_000_000 + + total_query_ms += query_ms + total_fetch_ms += fetch_ms + shard_table.add_row(shard_id, f"{query_ms:.2f}", f"{fetch_ms:.2f}") + + shard_table.add_section() + shard_table.add_row( + "[bold]Total[/bold]", + f"[bold]{total_query_ms:.2f}[/bold]", + f"[bold]{total_fetch_ms:.2f}[/bold]", + ) + console.print(shard_table) + + # --- Slowest clauses table (top 15, aggregated across shards) --- + clause_totals: dict = defaultdict(float) + clause_types: dict = {} + for ctype, desc, ms in all_clauses: + key = desc or ctype + clause_totals[key] += ms + clause_types[key] = ctype + + top = sorted(clause_totals.items(), key=lambda kv: -kv[1])[:15] + if top: + clause_table = Table( + title="Profile — slowest clauses (all shards combined)", box=box.SIMPLE_HEAVY + ) + clause_table.add_column("Type", style="dim", no_wrap=True) + clause_table.add_column("Description", overflow="fold") + clause_table.add_column("Total (ms)", justify="right") + for desc, ms in top: + clause_table.add_row(clause_types.get(desc, ""), desc, f"{ms:.2f}") + console.print(clause_table) + + +# --------------------------------------------------------------------------- +# Search-again prompt +# --------------------------------------------------------------------------- + + +def _search_again_prompt(current: dict, module) -> dict: + """Prompt for new search parameters, shared + module-specific.""" + from src.querier.builder import TIME_RANGES + + console.print( + "\n[bold cyan]New Search Parameters[/bold cyan] [dim](Enter to keep current)[/dim]" + ) + console.print("[dim]Time ranges: " + " ".join(TIME_RANGES) + "[/dim]") + + def _ask(label: str, current_val) -> str: + """Prompt for a value; returns user input or preserves current (empty string for None).""" + display = "" if current_val is None else str(current_val) + try: + val = prompt(f" {label} [{display}]: ").strip() + return val if val else display + except KeyboardInterrupt: + console.print("") + return display + + new = dict(current) + new["time_range"] = _ask("Time range", current.get("time_range", "now-24h")) + if module.SENSOR_PARAM is not None: + new[module.SENSOR_PARAM] = _ask( + "Sensor (comma-sep or 'all')", current.get(module.SENSOR_PARAM, "all") + ) + + if module.SUPPORTS_IP_FILTER: + pub_raw = _ask("Public only (y/n)", "y" if current.get("public_only") else "n") + new["public_only"] = pub_raw.lower() in ("y", "yes") + + src_raw = _ask("Src IP filter (blank to clear)", current.get("src_ip")) + new["src_ip"] = src_raw if src_raw else None + + dir_raw = _ask( + "Direction filter (inbound/outbound/internal/external/blank)", + current.get("direction"), + ) + new["direction"] = dir_raw if dir_raw else None + + limit_raw = _ask("Limit", current.get("limit", 500)) + try: + new["limit"] = int(limit_raw) + except ValueError: + new["limit"] = current.get("limit", 500) + + # Module-specific search params + module.add_search_params_prompt(new, _ask) + + return new + + +# --------------------------------------------------------------------------- +# Interactive loop +# --------------------------------------------------------------------------- + + +def interactive_loop(records: list, search_params: dict, module, query_fn=None) -> None: + """Interactive post-display prompt — dispatch to module for protocol actions. + + query_fn: callable with signature (module, search_params) -> list. + Defaults to run_query when None. + """ + _query_fn = query_fn if query_fn is not None else run_query + + from src.enricher.threat_intel import enrich_ip + from src.mantis.mantis_search import ( + display_results as display_mantis, + ) + from src.mantis.mantis_search import ( + search as mantis_search, + ) + from src.mantis.mantis_search import ( + sensor_to_project, + ) + + last_record: dict | None = None + + while True: + console.print("") + if last_record: + r = last_record["record"] + hint = module.describe_record(r) + console.print(f"[dim]↩ Last: #{last_record['idx']} {hint}[/dim]") + console.print( + "[bold cyan]Action[/bold cyan] — enter record #" + " / \\[r]e-search / \\[p]rint (CTRL+C to exit):" + ) + try: + raw = prompt(" > ").strip().lower() + except KeyboardInterrupt: + if confirm_exit(): + break + continue + + if raw in ("r", "research", "search"): + search_params = _search_again_prompt(search_params, module) + new_records = _query_fn(module, search_params) + if new_records: + records = new_records + module.display(records) + continue + + if raw in ("p", "print"): + module.display(records) + continue + + if not raw: + continue + + readline.add_history(raw) + + try: + idx = int(raw) - 1 + if idx < 0: + raise ValueError + record = records[idx] + except (ValueError, IndexError): + console.print("[red]Invalid selection.[/red]") + continue + + src_ip = record.get("src_ip", "") + module.display_detail(record, idx + 1) + + # Build action menu dynamically based on module capabilities. + action_parts = [] + if module.SUPPORTS_ENRICHMENT: + action_parts.append("\\[e]nrich") + if module.SUPPORTS_FP: + action_parts.append("\\[f]alse positive") + action_parts += ["\\[m]antis search", "\\[s]kip"] + console.print(" " + " ".join(action_parts)) + + try: + action = prompt(" Action: ").strip().lower() + except KeyboardInterrupt: + console.print("[dim]Cancelled.[/dim]") + continue + + key = action[:1] + if key: + readline.add_history(action) + + if key == "e" and module.SUPPORTS_ENRICHMENT: + hash_val = record.get("sha256") or record.get("md5") or record.get("file_hash") + if hash_val and not src_ip: + # Hash-only enrichment (e.g. pe module — no IP). + from src.enricher.virustotal import check_hash, display_hash + + console.print(f"[dim]Querying VirusTotal for {hash_val[:16]}…[/dim]") + display_hash(hash_val, check_hash(hash_val)) + elif hash_val: + # Both hash and IP available — ask which. + console.print(" \\[h]ash (VirusTotal file lookup) \\[i]p enrichment") + try: + sub = prompt(" Choice: ").strip().lower() + except KeyboardInterrupt: + console.print("[dim]Cancelled.[/dim]") + sub = "" + if sub.startswith("h"): + from src.enricher.virustotal import check_hash, display_hash + + console.print(f"[dim]Querying VirusTotal for {hash_val[:16]}…[/dim]") + display_hash(hash_val, check_hash(hash_val)) + else: + enrich_ip(src_ip) + else: + enrich_ip(src_ip) + elif key == "f" and module.SUPPORTS_FP: + module.fp_action(record) + elif key == "m": + dest_ip = record.get("dest_ip", "") + queries = dict.fromkeys(ip for ip in [src_ip, dest_ip] if ip) + city = sensor_to_project(record.get("sensor", "all")) + if city: + console.print(f"[dim]Filtering Mantis to project '{city}'[/dim]") + combined: list = [] + seen_ids: set = set() + for q in queries: + console.print(f"[dim]Searching Mantis for '{q}'...[/dim]") + for r in mantis_search(q, city=city): + if r["id"] not in seen_ids: + combined.append(r) + seen_ids.add(r["id"]) + display_mantis(combined) + last_record = {"idx": idx + 1, "record": record} + + +# --------------------------------------------------------------------------- +# Diagnostic helpers +# --------------------------------------------------------------------------- + + +def list_sensors(time_range: str = "now-7d") -> None: + """Aggregate on host.name and print all known sensors.""" + body = { + "size": 0, + "query": { + "bool": { + "filter": [ + {"range": {"@timestamp": {"gte": time_range, "lte": "now"}}}, + {"exists": {"field": "event.dataset"}}, + ] + } + }, + "aggs": { + "sensors": { + "terms": { + "field": "host.name", + "size": 500, + "order": {"_count": "desc"}, + } + } + }, + } + params = {"path": f"{INDEX}/_search", "method": "POST"} + + console.print(f"[dim]Querying host.name values ({time_range})...[/dim]") + try: + raw = query_opensearch(body, params) + except (OpenSearchConnectionError, OpenSearchAuthError) as exc: + console.print(f"[red]{exc}[/red]") + return + + buckets = raw.get("aggregations", {}).get("sensors", {}).get("buckets", []) + if not buckets: + console.print("[yellow]No sensors found in the given time range.[/yellow]") + return + + table = Table( + title=f"Malcolm sensors (past {time_range.replace('now-', '')})", + box=box.SIMPLE_HEAVY, + ) + table.add_column("host.name", style="cyan") + table.add_column("Record count", justify="right") + + for bucket in buckets: + table.add_row(bucket["key"], str(bucket["doc_count"])) + + console.print(table) + console.print( + f"[dim]{len(buckets)} sensor(s) found. " + f"Pass one or more to --sensor as a comma-separated list.[/dim]" + ) + + +def list_log_types(time_range: str = "now-7d") -> None: + """Aggregate on event.dataset and print all Zeek log types present.""" + body = { + "size": 0, + "query": { + "bool": { + "filter": [ + {"range": {"@timestamp": {"gte": time_range, "lte": "now"}}}, + {"exists": {"field": "event.dataset"}}, + ] + } + }, + "aggs": { + "log_types": { + "terms": { + "field": "event.dataset", + "size": 200, + "order": {"_count": "desc"}, + } + } + }, + } + params = {"path": f"{INDEX}/_search", "method": "POST"} + + console.print(f"[dim]Querying event.dataset values ({time_range})...[/dim]") + try: + raw = query_opensearch(body, params) + except (OpenSearchConnectionError, OpenSearchAuthError) as exc: + console.print(f"[red]{exc}[/red]") + return + + buckets = raw.get("aggregations", {}).get("log_types", {}).get("buckets", []) + if not buckets: + console.print("[yellow]No log types found in the given time range.[/yellow]") + return + + table = Table( + title=f"Zeek log types in Malcolm (past {time_range.replace('now-', '')})", + box=box.SIMPLE_HEAVY, + ) + table.add_column("event.dataset (log type)", style="cyan") + table.add_column("Record count", justify="right") + + for bucket in buckets: + table.add_row(bucket["key"], str(bucket["doc_count"])) + + console.print(table) + console.print( + f"[dim]{len(buckets)} log type(s) found. Pass one to --log-type (or 'all').[/dim]" + ) + + +def list_indices() -> None: + """List all indices in the cluster sorted by doc count.""" + try: + base_url, session = _opensearch_session() + except (OpenSearchConnectionError, OpenSearchAuthError) as exc: + console.print(f"[red]{exc}[/red]") + return + + try: + resp = session.post( + base_url + "/api/console/proxy", + params={ + "path": ( + "_cat/indices?format=json&s=docs.count:desc" + "&h=index,docs.count,store.size,health" + ), + "method": "GET", + }, + timeout=30, + ) + except httpx.RequestError as exc: + console.print(f"[red]Cannot reach OpenSearch at {base_url}: {exc}[/red]") + return + + if not resp.is_success: + console.print(f"[red]OpenSearch error {resp.status_code}: {resp.text[:300]}[/red]") + return + + indices = resp.json() + if not indices: + console.print("[yellow]No indices found.[/yellow]") + return + + table = Table(title="OpenSearch indices (sorted by doc count)", box=box.SIMPLE_HEAVY) + table.add_column("Index", style="cyan") + table.add_column("Docs", justify="right") + table.add_column("Size", justify="right") + table.add_column("Health", justify="center") + + for entry in indices: + health = entry.get("health", "") + health_color = {"green": "green", "yellow": "yellow", "red": "red"}.get(health, "white") + table.add_row( + entry.get("index", ""), + entry.get("docs.count", "—"), + entry.get("store.size", "—"), + f"[{health_color}]{health}[/{health_color}]", + ) + + console.print(table) + console.print( + f"[dim]Current INDEX pattern: '{INDEX}' — update the INDEX constant if needed.[/dim]" + ) + + +def match_all_sample(time_range: str = "now-24h", limit: int = 3) -> None: + """Fire a plain match_all to confirm data exists and show actual field names.""" + body = { + "size": limit, + "sort": [{"@timestamp": {"order": "desc"}}], + "query": {"bool": {"must": [{"range": {"@timestamp": {"gte": time_range, "lte": "now"}}}]}}, + } + params = {"path": f"{INDEX}/_search", "method": "POST"} + + console.print(f"[dim]match_all against '{INDEX}' ({time_range}, limit {limit})...[/dim]") + try: + raw = query_opensearch(body, params) + except (OpenSearchConnectionError, OpenSearchAuthError) as exc: + console.print(f"[red]{exc}[/red]") + return + + total = raw.get("hits", {}).get("total", {}) + total_val = total.get("value", 0) if isinstance(total, dict) else total + console.print(f"[bold]Total hits:[/bold] {total_val}") + + hits = raw.get("hits", {}).get("hits", []) + if not hits: + console.print( + "[yellow]No hits — index pattern may not match any indices," + " or no data in this time range.[/yellow]" + ) + return + + for i, hit in enumerate(hits, 1): + console.print(f"\n[bold cyan]── Hit {i} ──[/bold cyan]") + console.print(json.dumps(hit.get("_source", {}), indent=2, default=str)) diff --git a/src/querier/client.py b/src/querier/client.py new file mode 100644 index 0000000..b3c9618 --- /dev/null +++ b/src/querier/client.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +"""OpenSearch connection management: session lifecycle and raw query execution.""" + +import atexit +import os +import sys + +import httpx +from rich.console import Console + +console = Console(file=sys.stderr) + +OPENSEARCH_URL = os.environ.get("OPENSEARCH_URL", "https://pisces-opensearch.cyberrangepoulsbo.com") +INDEX = "arkime_sessions3-*" + +_DEFAULT_HEADERS = { + "Content-Type": "application/json", + "osd-xsrf": "true", +} + + +class OpenSearchConnectionError(RuntimeError): + """Raised when OpenSearch is unreachable or the URL / credentials are not configured.""" + + +class OpenSearchAuthError(RuntimeError): + """Raised when OpenSearch rejects the supplied credentials (HTTP 401).""" + + +# --------------------------------------------------------------------------- +# Sync client (CLI + per-request web calls) +# --------------------------------------------------------------------------- + +# Module-level client cache: (url, username, password, client). +_client_cache: tuple[str, str, str, httpx.Client] | None = None + + +def _opensearch_client() -> tuple[str, httpx.Client]: + """Return (base_url, authenticated httpx.Client). + + The Client is cached at module level and reused as long as credentials + remain unchanged, so the connection pool stays warm across calls. + Raises OpenSearchConnectionError when credentials are not configured. + """ + global _client_cache + + opensearch_url = os.environ.get("OPENSEARCH_URL", OPENSEARCH_URL) + username = os.environ.get("PISCES_USERNAME", "") + password = os.environ.get("PISCES_PASSWORD", "") + + if not username or not password: + raise OpenSearchConnectionError( + "PISCES_USERNAME and PISCES_PASSWORD must be set — check your .env file" + ) + + if _client_cache is not None: + cached_url, cached_user, cached_pass, cached_client = _client_cache + if (cached_url, cached_user, cached_pass) == (opensearch_url, username, password): + return opensearch_url, cached_client + + client = httpx.Client( + auth=(username, password), + verify=False, + headers=_DEFAULT_HEADERS, + limits=httpx.Limits(max_connections=20, max_keepalive_connections=16), + timeout=30.0, + ) + _client_cache = (opensearch_url, username, password, client) + atexit.register(client.close) + return opensearch_url, client + + +# Keep backwards-compatible alias used by list_indices in cli_loop.py +_opensearch_session = _opensearch_client + + +def query_opensearch(body: dict, params: dict) -> dict: + """Submit a synchronous query to OpenSearch. + + Raises OpenSearchConnectionError or OpenSearchAuthError on failure. + """ + base_url, client = _opensearch_client() + + try: + resp = client.post( + base_url + "/api/console/proxy", + params=params, + json=body, + ) + except httpx.RequestError as exc: + raise OpenSearchConnectionError( + f"Cannot reach OpenSearch at {base_url} — are you on the VPN? ({exc})" + ) from exc + + if resp.status_code == 401: + raise OpenSearchAuthError( + "OpenSearch rejected the credentials — check PISCES_USERNAME/PASSWORD" + ) + + if not resp.is_success: + raise OpenSearchConnectionError( + f"OpenSearch returned HTTP {resp.status_code}: {resp.text[:300]}" + ) + + return resp.json() + + +# --------------------------------------------------------------------------- +# Async client (cross-protocol fan-out on the web path) +# --------------------------------------------------------------------------- + +_async_client_cache: tuple[str, str, str, httpx.AsyncClient] | None = None + + +async def _get_async_client() -> tuple[str, httpx.AsyncClient]: + """Return (base_url, long-lived AsyncClient) — one per process per credential set.""" + global _async_client_cache + + opensearch_url = os.environ.get("OPENSEARCH_URL", OPENSEARCH_URL) + username = os.environ.get("PISCES_USERNAME", "") + password = os.environ.get("PISCES_PASSWORD", "") + + if not username or not password: + raise OpenSearchConnectionError( + "PISCES_USERNAME and PISCES_PASSWORD must be set — check your .env file" + ) + + if _async_client_cache is not None: + cached_url, cached_user, cached_pass, cached_client = _async_client_cache + if (cached_url, cached_user, cached_pass) == (opensearch_url, username, password): + return opensearch_url, cached_client + + async_client = httpx.AsyncClient( + auth=(username, password), + verify=False, + headers=_DEFAULT_HEADERS, + limits=httpx.Limits(max_connections=40, max_keepalive_connections=32), + timeout=30.0, + ) + _async_client_cache = (opensearch_url, username, password, async_client) + return opensearch_url, async_client + + +async def query_opensearch_async(body: dict, params: dict) -> dict: + """Async variant of query_opensearch — for use in the web fan-out path. + + Raises OpenSearchConnectionError or OpenSearchAuthError on failure. + """ + base_url, client = await _get_async_client() + + try: + resp = await client.post( + base_url + "/api/console/proxy", + params=params, + json=body, + ) + except httpx.RequestError as exc: + raise OpenSearchConnectionError( + f"Cannot reach OpenSearch at {base_url} — are you on the VPN? ({exc})" + ) from exc + + if resp.status_code == 401: + raise OpenSearchAuthError( + "OpenSearch rejected the credentials — check PISCES_USERNAME/PASSWORD" + ) + + if not resp.is_success: + raise OpenSearchConnectionError( + f"OpenSearch returned HTTP {resp.status_code}: {resp.text[:300]}" + ) + + return resp.json() diff --git a/src/querier/module.py b/src/querier/module.py new file mode 100644 index 0000000..9e7ed2f --- /dev/null +++ b/src/querier/module.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""ZeekModule abstract base class — protocol module interface.""" + +from src.querier.client import console + + +class ZeekModule: + """Protocol module interface. Subclass and override methods as needed.""" + + DATASETS: list = ["all"] + SOURCE_FIELDS: list = [] + DETAIL_FIELDS: list = [] # List of (label: str, value_fn: Callable[[dict], str]) + SENSOR_PARAM: str | None = "sensor" # Set to None to skip the sensor prompt in re-search + SUPPORTS_IP_FILTER: bool = True # Set False for metadata-only modules (pe, capture_loss) + SUPPORTS_ENRICHMENT: bool = True # Set False for modules with no IPs or hashes to enrich + SUPPORTS_FP: bool = True # Set False for diagnostic modules (capture_loss) + WEB_CATEGORY: str = "core" # Category group for the web UI sidebar + WEB_ICON: str = "fa-question" # Font Awesome icon class for the web UI sidebar + WEB_COLUMNS: list = [] # List of (header: str, value_fn: Callable[[dict], str]) for web table + EXTRA_PARAMS: list[str] = [] # Protocol-specific search_params keys forwarded from HTTP request + SUMMARY_FIELD: str | None = None # OpenSearch field to aggregate on for the browse modal + SUMMARY_PARAM: str | None = None # EXTRA_PARAMS key that filters on SUMMARY_FIELD + SUMMARY_TYPE: str = "flat" # "flat" = single field agg, "grouped" = scripted prefix + severity + + def build_extra_must(self, search_params: dict) -> tuple: + """Return (must_clauses, post_filters) built from search_params. + + must_clauses: list of OpenSearch DSL clause dicts added to the query must. + post_filters: list of callables (record: dict) -> bool applied after parsing. + When non-empty, run_query() uses 3× over-fetch automatically. + """ + return [], [] + + def prepare_hits(self, hits: list) -> None: + """Pre-parse hook called on raw hits before parse_hit() loop. + + Override for batch lookups (e.g. x509 community_id pivot, pe fuid→hash lookup). + Default is a no-op. + """ + + def parse_hit(self, src: dict) -> dict: + """Convert an OpenSearch _source dict to a normalised record dict. + + Must include at minimum: timestamp, sensor, src_ip, dest_ip, + dest_port, src_port, _raw. + """ + raise NotImplementedError + + def dedup_key(self, record: dict) -> tuple: + """Return the grouping key tuple for deduplicate_zeek.""" + raise NotImplementedError + + def display(self, records: list) -> None: + """Render records as a Rich table.""" + raise NotImplementedError + + def display_detail(self, record: dict, idx: int) -> None: + """Render a Rich Panel with every DETAIL_FIELDS field for the selected record.""" + from rich.panel import Panel + from rich.table import Table + + grid = Table.grid(padding=(0, 2)) + grid.add_column(style="dim", no_wrap=True, min_width=12) + grid.add_column(overflow="fold") + for label, fn in self.DETAIL_FIELDS: + grid.add_row(label, fn(record)) + console.print( + Panel( + grid, + title=f"[bold]#{idx}[/bold] {self.describe_record(record)}", + expand=False, + ) + ) + + def add_args(self, parser) -> None: + """Add protocol-specific argparse arguments to the shared parser.""" + pass + + def describe_record(self, record: dict) -> str: + """One-line summary used in the interactive loop hint line.""" + src = record.get("src_ip", "?") + dst = record.get("dest_ip", "?") + port = record.get("dest_port", "?") + return f"{src} → {dst}:{port}" + + def fp_signature(self, record: dict) -> str: + """Signature string embedded in the FP alert dict.""" + return "zeek/unknown" + + def fp_action(self, record: dict) -> None: + """Handle the [f]alse positive action. Override for custom behaviour.""" + from src.querier.fp_manager import create_filter_interactive + + fp_alert = { + "src_ip": record.get("src_ip"), + "dest_ip": record.get("dest_ip"), + "dest_port": record.get("dest_port"), + "alert": { + "signature": self.fp_signature(record), + "severity": 3, + }, + "clientID": (record.get("sensors") or [record.get("sensor", "")])[0], + } + create_filter_interactive(alert=fp_alert) + + def add_search_params_prompt(self, new: dict, _ask) -> None: + """Prompt for protocol-specific re-search parameters. + + Override to append module-specific fields to `new`. + `_ask(label, current_val)` returns the user-entered string or the + current value's string form if the user pressed Enter. + """ + pass diff --git a/src/querier/runner.py b/src/querier/runner.py new file mode 100644 index 0000000..d081ea7 --- /dev/null +++ b/src/querier/runner.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +"""Core query execution: deduplication, post-filtering, and run_query entrypoint.""" + +import hashlib +import json +import os +from collections import defaultdict + +from src.querier.builder import _remap_clause, build_base_query +from src.querier.client import console, query_opensearch, query_opensearch_async +from src.utils.cache import cache_path as _cache_path_util +from src.utils.cache import load_cache, save_cache +from src.utils.format import fmt_bytes, fmt_dur + +# Backwards-compatible aliases — zeek modules import these names from .base +_fmt_bytes = fmt_bytes +_fmt_dur = fmt_dur + +# Project root — four dirname() calls up from src/querier/runner.py +_BASE = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +FILTERS_DIR = os.path.join(_BASE, "filters") + +# Fetch this many times the requested limit when post-filters are active. +_OVERFETCH_MULTIPLIER = 3 + + +def _first(val): + """Return val as-is if scalar, or the first element if it's a list.""" + if isinstance(val, list): + return val[0] if val else None + return val + + +def _sensor_str(rec: dict) -> str: + """Format the sensor(s) column for display.""" + sensors = rec.get("sensors") + vals = sensors if sensors else ([rec["sensor"]] if rec.get("sensor") else []) + return ", ".join(v.removeprefix("hedgehog-") for v in vals) + + +def _cache_path(args_hash: str) -> str: + return _cache_path_util(f"opensearch_{args_hash}.json") + + +_save_cache = save_cache +_load_cache = load_cache + +# Cache: (raw_must_not, remapped) — invalidates when load_filters returns a new list object. +_remap_cache: tuple[list, list] | None = None + + +def load_with_remap(filters_dir: str) -> tuple: + """Load filters and remap field names. Returns (must_not, fcount, errors).""" + global _remap_cache + from src.querier.filter_loader import load_filters + + filter_result = load_filters(filters_dir) + raw = filter_result["must_not"] + if _remap_cache is None or _remap_cache[0] is not raw: + _remap_cache = (raw, [_remap_clause(c) for c in raw]) + return _remap_cache[1], filter_result["filter_count"], filter_result["errors"] + + +def deduplicate_zeek(records: list, key_fn) -> list: + """Deduplicate records by key_fn, keeping the most recent per group. + + Sorts output by descending frequency so highest-volume flows appear first. + """ + grouped: dict = defaultdict(list) + for rec in records: + key = key_fn(rec) + grouped[key].append(rec) + + deduped = [] + for _key, group in sorted(grouped.items(), key=lambda kv: -len(kv[1])): + rep = max(group, key=lambda r: r["timestamp"]).copy() + rep["freq"] = len(group) + if len(group) == 1: + rep["sensors"] = [rep["sensor"]] if rep.get("sensor") else [] + else: + rep["sensors"] = sorted({r["sensor"] for r in group if r.get("sensor")}) + deduped.append(rep) + + return deduped + + +def run_query(module, search_params: dict) -> list: + """Execute a full query cycle: load filters, build query, fetch, parse, dedup.""" + if search_params.get("no_filters"): + must_not: list = [] + console.print("[yellow]--no-filters: all false positive filters disabled[/yellow]") + else: + must_not, fcount, errors = load_with_remap(FILTERS_DIR) + console.print("[dim]Loading false positive filters...[/dim]") + console.print( + f"[dim]Loaded {fcount} filter file(s) → {len(must_not)} must_not clause(s)[/dim]" + ) + for err in errors: + console.print(f"[yellow]Filter warning: {err}[/yellow]") + + sensors: list | None = None + sensor_val = search_params.get("sensor", "all") + if sensor_val and str(sensor_val).lower() != "all": + sensors = [s.strip() for s in str(sensor_val).split(",")] + + extra_must, post_filters = module.build_extra_must(search_params) + + # Guard src/dest/any ip filters for modules that don't have the field in SOURCE_FIELDS — + # a term query on a missing field returns zero results. + has_src = "source.ip" in module.SOURCE_FIELDS + has_dest = "destination.ip" in module.SOURCE_FIELDS + src_ip_for_query = search_params.get("src_ip") if has_src else None + dest_ip_for_query = search_params.get("dest_ip") if has_dest else None + any_ip_for_query = search_params.get("any_ip") if (has_src or has_dest) else None + + # Over-fetch when post-filters are active so truncation still yields enough rows. + requested_limit = search_params.get("limit", 500) + query_limit = ( + min(requested_limit * _OVERFETCH_MULTIPLIER, 5000) if post_filters else requested_limit + ) + + body, params = build_base_query( + must_not=must_not, + extra_must=extra_must, + source_fields=module.SOURCE_FIELDS, + limit=query_limit, + time_range=search_params.get("time_range", "now-24h"), + sensors=sensors, + datasets=module.DATASETS, + public_only=search_params.get("public_only", False), + src_ip_filter=src_ip_for_query, + dest_ip_filter=dest_ip_for_query, + any_ip_filter=any_ip_for_query, + direction=search_params.get("direction"), + time_from=search_params.get("time_from"), + time_to=search_params.get("time_to"), + ) + + if search_params.get("profile"): + body["profile"] = True + + # Cache is meaningless for profile runs (timing data is request-specific). + use_cache = search_params.get("use_cache", False) and not search_params.get("profile") + raw = None + cache_key = hashlib.md5(json.dumps(body, sort_keys=True).encode()).hexdigest()[:10] + cpath = _cache_path(cache_key) + + if use_cache: + raw = _load_cache(cpath) + if raw: + console.print(f"[dim]Using cached response: {cpath}[/dim]") + + if raw is None: + console.print( + f"[dim]Querying OpenSearch / Malcolm" + f" ({search_params.get('time_range', 'now-24h')})...[/dim]" + ) + raw = query_opensearch(body, params) + if use_cache: + _save_cache(raw, cpath) + + if search_params.get("profile"): + from src.querier.cli_loop import display_profile + + display_profile(raw) + + hits = raw.get("hits", {}).get("hits", []) + if not hits: + console.print("[yellow]No records returned.[/yellow]") + return [] + + # Pre-parse hook — e.g. x509 community_id batch lookup, pe fuid→hash lookup. + module.prepare_hits(hits) + + records = [module.parse_hit(hit.get("_source", {})) for hit in hits] + records = [r for r in records if r] + + # Apply post-filters (over-fetch strategy: fetch 3× then truncate). + if post_filters: + keep = lambda r: all(pf(r) for pf in post_filters) # noqa: E731 + records = [r for r in records if keep(r)] + if len(records) < requested_limit: + console.print( + f"[dim]Showing {len(records)}/{requested_limit} after post-filtering — " + f"increase time range for more results.[/dim]" + ) + records = records[:requested_limit] + + return deduplicate_zeek(records, module.dedup_key) + + +async def run_query_async(module, search_params: dict) -> list: + """Async variant of run_query — uses httpx.AsyncClient for the web fan-out path. + + Skips profile support (web path only). The cache check, filter loading, + query building, parsing, and deduplication are identical to the sync version. + """ + if search_params.get("no_filters"): + must_not: list = [] + else: + must_not, _fcount, _errors = load_with_remap(FILTERS_DIR) + + sensors: list | None = None + sensor_val = search_params.get("sensor", "all") + if sensor_val and str(sensor_val).lower() != "all": + sensors = [s.strip() for s in str(sensor_val).split(",")] + + extra_must, post_filters = module.build_extra_must(search_params) + + has_src = "source.ip" in module.SOURCE_FIELDS + has_dest = "destination.ip" in module.SOURCE_FIELDS + src_ip_for_query = search_params.get("src_ip") if has_src else None + dest_ip_for_query = search_params.get("dest_ip") if has_dest else None + any_ip_for_query = search_params.get("any_ip") if (has_src or has_dest) else None + + requested_limit = search_params.get("limit", 500) + query_limit = ( + min(requested_limit * _OVERFETCH_MULTIPLIER, 5000) if post_filters else requested_limit + ) + + body, params = build_base_query( + must_not=must_not, + extra_must=extra_must, + source_fields=module.SOURCE_FIELDS, + limit=query_limit, + time_range=search_params.get("time_range", "now-24h"), + sensors=sensors, + datasets=module.DATASETS, + public_only=search_params.get("public_only", False), + src_ip_filter=src_ip_for_query, + dest_ip_filter=dest_ip_for_query, + any_ip_filter=any_ip_for_query, + direction=search_params.get("direction"), + time_from=search_params.get("time_from"), + time_to=search_params.get("time_to"), + ) + + raw = await query_opensearch_async(body, params) + + hits = raw.get("hits", {}).get("hits", []) + if not hits: + return [] + + module.prepare_hits(hits) + records = [module.parse_hit(hit.get("_source", {})) for hit in hits] + records = [r for r in records if r] + + if post_filters: + keep = lambda r: all(pf(r) for pf in post_filters) # noqa: E731 + records = [r for r in records if keep(r)] + records = records[:requested_limit] + + return deduplicate_zeek(records, module.dedup_key) diff --git a/src/querier/zeek_modules/base.py b/src/querier/zeek_modules/base.py index 9fa70c5..5390762 100644 --- a/src/querier/zeek_modules/base.py +++ b/src/querier/zeek_modules/base.py @@ -1,1123 +1,101 @@ #!/usr/bin/env python3 -"""Shared infrastructure for PISCES OpenSearch / Malcolm / Zeek querier modules. - -All constants, utility functions, OpenSearch interaction, query building, -deduplication, and interactive loop logic live here. Protocol modules -(conn, dns, http, …) import from this file and the ZeekModule base class. -""" - -import atexit -import hashlib -import ipaddress -import json -import os -import readline # noqa: F401 — enables arrow-key history in input() -import sys -from collections import defaultdict - -import requests -import urllib3 -from rich import box -from rich.console import Console -from rich.table import Table - -from src.utils.cache import cache_path as _cache_path_util -from src.utils.cache import load_cache, save_cache -from src.utils.format import fmt_bytes, fmt_dur -from src.utils.terminal import confirm_exit, prompt - -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - -console = Console(file=sys.stderr) +"""Backwards-compatibility shim — re-exports everything from the focused querier modules. +Zeek protocol modules import from this file via relative imports (`from .base import …`). +New code should import directly from the focused modules in src/querier/: -class OpenSearchConnectionError(RuntimeError): - """Raised when OpenSearch is unreachable or the URL / credentials are not configured.""" - - -class OpenSearchAuthError(RuntimeError): - """Raised when OpenSearch rejects the supplied credentials (HTTP 401).""" - - -# Backwards-compatible aliases — zeek modules import these names from .base -_fmt_bytes = fmt_bytes -_fmt_dur = fmt_dur + from src.querier.client import query_opensearch, console + from src.querier.builder import build_base_query, FIELD_MAP + from src.querier.runner import run_query, deduplicate_zeek + from src.querier.cli_loop import interactive_loop + from src.querier.module import ZeekModule +""" -# Project root — four dirname() calls up from src/querier/zeek_modules/base.py -_BASE = os.path.dirname( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from src.querier.builder import ( + _PRIVATE_CIDR_MUST_NOT, + _PRIVATE_CIDRS, + FIELD_MAP, + TIME_RANGES, + _remap_clause, + build_base_query, + is_private, +) +from src.querier.cli_loop import ( + _search_again_prompt, + _walk_clauses, + display_profile, + interactive_loop, + list_indices, + list_log_types, + list_sensors, + match_all_sample, +) +from src.querier.client import ( + INDEX, + OPENSEARCH_URL, + OpenSearchAuthError, + OpenSearchConnectionError, + _opensearch_session, + console, + query_opensearch, +) +from src.querier.module import ZeekModule +from src.querier.runner import ( + _OVERFETCH_MULTIPLIER, + FILTERS_DIR, + _cache_path, + _first, + _fmt_bytes, + _fmt_dur, + _load_cache, + _remap_cache, + _save_cache, + _sensor_str, + deduplicate_zeek, + load_with_remap, + run_query, ) -FILTERS_DIR = os.path.join(_BASE, "filters") - -# --------------------------------------------------------------------------- -# Constants -# --------------------------------------------------------------------------- - -# Fetch this many times the requested limit when post-filters are active so -# truncation still yields enough rows after Python-side filtering discards hits. -_OVERFETCH_MULTIPLIER = 3 - -OPENSEARCH_URL = os.environ.get("OPENSEARCH_URL", "https://pisces-opensearch.cyberrangepoulsbo.com") -INDEX = "arkime_sessions3-*" - -# Field name translation: existing YAML filter field → Malcolm/Zeek field -FIELD_MAP = { - "src_ip": "source.ip", - "dest_ip": "destination.ip", - "src_port": "source.port", - "dest_port": "destination.port", - "app_proto": "network.protocol", - "clientID": "host.name", -} - -TIME_RANGES = [ - "now-15m", - "now-30m", - "now-1h", - "now-3h", - "now-6h", - "now-12h", - "now-24h", - "now-2d", - "now-3d", - "now-7d", - "now-14d", - "now-30d", -] -# Non-routable CIDRs excluded by --public-only. -_PRIVATE_CIDRS = [ - # IPv4 - "10.0.0.0/8", - "172.16.0.0/12", - "192.168.0.0/16", - "127.0.0.0/8", - "169.254.0.0/16", # link-local / APIPA - # IPv6 - "::1/128", # loopback - "fe80::/10", # link-local - "fc00::/7", # unique-local (fd00::/8 etc.) - "ff00::/8", # multicast +__all__ = [ + # client + "INDEX", + "OPENSEARCH_URL", + "OpenSearchAuthError", + "OpenSearchConnectionError", + "_opensearch_session", + "console", + "query_opensearch", + # builder + "FIELD_MAP", + "TIME_RANGES", + "_PRIVATE_CIDRS", + "_PRIVATE_CIDR_MUST_NOT", + "_remap_clause", + "build_base_query", + "is_private", + # runner + "FILTERS_DIR", + "_OVERFETCH_MULTIPLIER", + "_cache_path", + "_first", + "_fmt_bytes", + "_fmt_dur", + "_load_cache", + "_remap_cache", + "_save_cache", + "_sensor_str", + "deduplicate_zeek", + "load_with_remap", + "run_query", + # cli_loop + "_search_again_prompt", + "_walk_clauses", + "display_profile", + "interactive_loop", + "list_indices", + "list_log_types", + "list_sensors", + "match_all_sample", + # module + "ZeekModule", ] - -# Precomputed once: a single must_not clause that matches any source IP in a -# private range. `term` does not evaluate CIDR notation on ip-typed fields — -# range queries with explicit network/broadcast bounds are the correct DSL. -_PRIVATE_CIDR_MUST_NOT: dict = { - "bool": { - "should": [ - { - "range": { - "source.ip": { - "gte": str(ipaddress.ip_network(cidr, strict=False).network_address), - "lte": str(ipaddress.ip_network(cidr, strict=False).broadcast_address), - } - } - } - for cidr in _PRIVATE_CIDRS - ] - } -} - - -# --------------------------------------------------------------------------- -# Utility helpers -# --------------------------------------------------------------------------- - - -def _first(val): - """Return val as-is if scalar, or the first element if it's a list.""" - if isinstance(val, list): - return val[0] if val else None - return val - - -def is_private(ip: str) -> bool: - try: - addr = ipaddress.ip_address(ip) - return any(addr in ipaddress.ip_network(cidr) for cidr in _PRIVATE_CIDRS) - except ValueError: - return False - - -def _sensor_str(rec: dict) -> str: - """Format the sensor(s) column for display.""" - sensors = rec.get("sensors") - vals = sensors if sensors else ([rec["sensor"]] if rec.get("sensor") else []) - return ", ".join(v.removeprefix("hedgehog-") for v in vals) - - -# --------------------------------------------------------------------------- -# Field remapping -# --------------------------------------------------------------------------- - - -def _remap_clause(clause: dict) -> dict: - """Recursively remap Kibana field names to Malcolm/Zeek field names. - - Walks a DSL clause dict and renames any key found in FIELD_MAP. - Handles term, terms, range, match_phrase, bool (recurses into - must/must_not/should/filter). Pure function — returns a new dict. - """ - if not isinstance(clause, dict): - return clause - - result = {} - for key, value in clause.items(): - if key in ( - "term", - "terms", - "range", - "match_phrase", - "match", - "wildcard", - "prefix", - "regexp", - ): - remapped_inner = {} - for field, field_val in value.items(): - new_field = FIELD_MAP.get(field, field) - remapped_inner[new_field] = field_val - result[key] = remapped_inner - elif key == "bool": - new_bool = {} - for bool_key, bool_val in value.items(): - if bool_key in ("must", "must_not", "should", "filter"): - if isinstance(bool_val, list): - new_bool[bool_key] = [_remap_clause(c) for c in bool_val] - else: - new_bool[bool_key] = _remap_clause(bool_val) - else: - new_bool[bool_key] = bool_val - result[key] = new_bool - else: - result[key] = value - - return result - - -# --------------------------------------------------------------------------- -# Cache helpers -# --------------------------------------------------------------------------- - - -def _cache_path(args_hash: str) -> str: - return _cache_path_util(f"opensearch_{args_hash}.json") - - -_save_cache = save_cache -_load_cache = load_cache - - -# --------------------------------------------------------------------------- -# OpenSearch session + query -# --------------------------------------------------------------------------- - -# Module-level session cache: (url, username, password, session). -# Reusing the same Session keeps the underlying TCP/TLS connection pool alive -# across the 30+ parallel cross-protocol calls that happen per page load. -_opensearch_session_cache: tuple[str, str, str, requests.Session] | None = None - - -def _opensearch_session() -> tuple[str, requests.Session]: - """Return (base_url, authenticated Session). - - Raises OpenSearchConnectionError when credentials are not configured. - The Session is cached at module level and reused as long as credentials - remain unchanged, so the connection pool stays warm across calls. - """ - global _opensearch_session_cache - - opensearch_url = os.environ.get("OPENSEARCH_URL", OPENSEARCH_URL) - username = os.environ.get("PISCES_USERNAME", "") - password = os.environ.get("PISCES_PASSWORD", "") - - if not username or not password: - raise OpenSearchConnectionError( - "PISCES_USERNAME and PISCES_PASSWORD must be set — check your .env file" - ) - - if _opensearch_session_cache is not None: - cached_url, cached_user, cached_pass, cached_session = _opensearch_session_cache - if (cached_url, cached_user, cached_pass) == (opensearch_url, username, password): - return opensearch_url, cached_session - - session = requests.Session() - session.auth = (username, password) - session.verify = False - session.headers.update( - { - "Content-Type": "application/json", - "osd-xsrf": "true", - } - ) - adapter = requests.adapters.HTTPAdapter(pool_connections=4, pool_maxsize=16) - session.mount("https://", adapter) - session.mount("http://", adapter) - _opensearch_session_cache = (opensearch_url, username, password, session) - atexit.register(session.close) - return opensearch_url, session - - -def query_opensearch(body: dict, params: dict) -> dict: - """Submit a query to OpenSearch. - - Raises OpenSearchConnectionError or OpenSearchAuthError on failure. - """ - base_url, session = _opensearch_session() - - try: - resp = session.post( - base_url + "/api/console/proxy", - params=params, - json=body, - timeout=30, - ) - except requests.RequestException as exc: - raise OpenSearchConnectionError( - f"Cannot reach OpenSearch at {base_url} — are you on the VPN? ({exc})" - ) from exc - - if resp.status_code == 401: - raise OpenSearchAuthError( - "OpenSearch rejected the credentials — check PISCES_USERNAME/PASSWORD" - ) - - if not resp.ok: - raise OpenSearchConnectionError( - f"OpenSearch returned HTTP {resp.status_code}: {resp.text[:300]}" - ) - - return resp.json() - - -# --------------------------------------------------------------------------- -# Profile display -# --------------------------------------------------------------------------- - - -def _walk_clauses(node: dict, acc: list) -> None: - """DFS walk of an ES profile query node, collecting (type, description, time_ms).""" - if not isinstance(node, dict): - return - time_ms = node.get("time_in_nanos", 0) / 1_000_000 - acc.append((node.get("type", ""), node.get("description", ""), time_ms)) - for child in node.get("children", []): - _walk_clauses(child, acc) - - -def display_profile(raw: dict) -> None: - """Render OpenSearch profile data: per-shard timing and slowest query clauses.""" - profile = raw.get("profile") - if not profile: - console.print("[yellow]No profile data in response.[/yellow]") - return - - shards = profile.get("shards", []) - if not shards: - console.print("[yellow]Profile present but no shard data.[/yellow]") - return - - # --- Shard summary table --- - shard_table = Table(title="Profile — per-shard timing", box=box.SIMPLE_HEAVY) - shard_table.add_column("Shard", style="cyan", no_wrap=True) - shard_table.add_column("Query (ms)", justify="right") - shard_table.add_column("Fetch (ms)", justify="right") - - total_query_ms = 0.0 - total_fetch_ms = 0.0 - all_clauses: list = [] - - for shard in shards: - shard_id = shard.get("id", "?") - query_ms = 0.0 - fetch_ms = 0.0 - - for search in shard.get("searches", []): - for q in search.get("query", []): - query_ms += q.get("time_in_nanos", 0) / 1_000_000 - _walk_clauses(q, all_clauses) - for collector in search.get("collector", []): - fetch_ms += collector.get("time_in_nanos", 0) / 1_000_000 - - total_query_ms += query_ms - total_fetch_ms += fetch_ms - shard_table.add_row(shard_id, f"{query_ms:.2f}", f"{fetch_ms:.2f}") - - shard_table.add_section() - shard_table.add_row( - "[bold]Total[/bold]", - f"[bold]{total_query_ms:.2f}[/bold]", - f"[bold]{total_fetch_ms:.2f}[/bold]", - ) - console.print(shard_table) - - # --- Slowest clauses table (top 15, aggregated across shards) --- - from collections import defaultdict - - clause_totals: dict = defaultdict(float) - clause_types: dict = {} - for ctype, desc, ms in all_clauses: - key = desc or ctype - clause_totals[key] += ms - clause_types[key] = ctype - - top = sorted(clause_totals.items(), key=lambda kv: -kv[1])[:15] - if top: - clause_table = Table( - title="Profile — slowest clauses (all shards combined)", box=box.SIMPLE_HEAVY - ) - clause_table.add_column("Type", style="dim", no_wrap=True) - clause_table.add_column("Description", overflow="fold") - clause_table.add_column("Total (ms)", justify="right") - for desc, ms in top: - clause_table.add_row(clause_types.get(desc, ""), desc, f"{ms:.2f}") - console.print(clause_table) - - -# --------------------------------------------------------------------------- -# Filter loading -# --------------------------------------------------------------------------- - - -# Cache: (raw_must_not, remapped) — invalidates when load_filters returns a new list object. -_remap_cache: tuple[list, list] | None = None - - -def load_with_remap(filters_dir: str) -> tuple: - """Load filters and remap field names. Returns (must_not, fcount, errors).""" - global _remap_cache - from src.querier.filter_loader import load_filters - - filter_result = load_filters(filters_dir) - raw = filter_result["must_not"] - if _remap_cache is None or _remap_cache[0] is not raw: - _remap_cache = (raw, [_remap_clause(c) for c in raw]) - return _remap_cache[1], filter_result["filter_count"], filter_result["errors"] - - -# --------------------------------------------------------------------------- -# Query building -# --------------------------------------------------------------------------- - - -def build_base_query( - must_not: list, - extra_must: list, - source_fields: list, - limit: int, - time_range: str, - sensors: list | None, - datasets: list, - public_only: bool = False, - src_ip_filter: str | None = None, - dest_ip_filter: str | None = None, - any_ip_filter: str | None = None, - direction: str | None = None, - time_from: str | None = None, - time_to: str | None = None, - sort: bool = True, -) -> tuple: - """Build the OpenSearch query body and request params. - - datasets: list of event.dataset values, or ["all"] to omit the filter. - time_from/time_to: absolute ISO timestamps; when both are set they - override the relative *time_range* parameter. - """ - if time_from and time_to: - ts_clause = {"range": {"@timestamp": {"gte": time_from, "lte": time_to}}} - else: - ts_clause = {"range": {"@timestamp": {"gte": time_range, "lte": "now"}}} - must_clauses: list = [ts_clause] - - if datasets and datasets != ["all"]: - must_clauses.append({"terms": {"event.dataset": datasets}}) - - if sensors: - must_clauses.append({"terms": {"host.name": sensors}}) - - if src_ip_filter: - must_clauses.append({"term": {"source.ip": src_ip_filter}}) - - if dest_ip_filter: - must_clauses.append({"term": {"destination.ip": dest_ip_filter}}) - - if any_ip_filter: - must_clauses.append( - { - "bool": { - "should": [ - {"term": {"source.ip": any_ip_filter}}, - {"term": {"destination.ip": any_ip_filter}}, - ], - "minimum_should_match": 1, - } - } - ) - - if direction: - must_clauses.append({"term": {"network.direction": direction}}) - - must_clauses.extend(extra_must) - - effective_must_not = list(must_not) - if public_only: - effective_must_not.append(_PRIVATE_CIDR_MUST_NOT) - - body: dict = { - "size": limit, - "track_total_hits": False, - "query": { - "bool": { - "filter": must_clauses, - "must_not": effective_must_not, - } - }, - "_source": source_fields, - } - if sort: - body["sort"] = [{"@timestamp": {"order": "desc"}}] - - params = { - "path": f"{INDEX}/_search?timeout=30s", - "method": "POST", - } - - return body, params - - -# --------------------------------------------------------------------------- -# Deduplication -# --------------------------------------------------------------------------- - - -def deduplicate_zeek(records: list, key_fn) -> list: - """Deduplicate records by key_fn, keeping the most recent per group. - - Sorts output by descending frequency so highest-volume flows appear first. - """ - grouped: dict = defaultdict(list) - for rec in records: - key = key_fn(rec) - grouped[key].append(rec) - - deduped = [] - for _key, group in sorted(grouped.items(), key=lambda kv: -len(kv[1])): - rep = max(group, key=lambda r: r["timestamp"]).copy() - rep["freq"] = len(group) - if len(group) == 1: - rep["sensors"] = [rep["sensor"]] if rep.get("sensor") else [] - else: - rep["sensors"] = sorted({r["sensor"] for r in group if r.get("sensor")}) - deduped.append(rep) - - return deduped - - -# --------------------------------------------------------------------------- -# run_query -# --------------------------------------------------------------------------- - - -def run_query(module, search_params: dict) -> list: - """Execute a full query cycle: load filters, build query, fetch, parse, dedup.""" - if search_params.get("no_filters"): - must_not: list = [] - console.print("[yellow]--no-filters: all false positive filters disabled[/yellow]") - else: - must_not, fcount, errors = load_with_remap(FILTERS_DIR) - console.print("[dim]Loading false positive filters...[/dim]") - console.print( - f"[dim]Loaded {fcount} filter file(s) → {len(must_not)} must_not clause(s)[/dim]" - ) - for err in errors: - console.print(f"[yellow]Filter warning: {err}[/yellow]") - - sensors: list | None = None - sensor_val = search_params.get("sensor", "all") - if sensor_val and str(sensor_val).lower() != "all": - sensors = [s.strip() for s in str(sensor_val).split(",")] - - extra_must, post_filters = module.build_extra_must(search_params) - - # Guard src/dest/any ip filters for modules that don't have the field in SOURCE_FIELDS — - # a term query on a missing field returns zero results. - has_src = "source.ip" in module.SOURCE_FIELDS - has_dest = "destination.ip" in module.SOURCE_FIELDS - src_ip_for_query = search_params.get("src_ip") if has_src else None - dest_ip_for_query = search_params.get("dest_ip") if has_dest else None - any_ip_for_query = search_params.get("any_ip") if (has_src or has_dest) else None - - # Over-fetch when post-filters are active so truncation still yields enough rows. - requested_limit = search_params.get("limit", 500) - query_limit = ( - min(requested_limit * _OVERFETCH_MULTIPLIER, 5000) if post_filters else requested_limit - ) - - body, params = build_base_query( - must_not=must_not, - extra_must=extra_must, - source_fields=module.SOURCE_FIELDS, - limit=query_limit, - time_range=search_params.get("time_range", "now-24h"), - sensors=sensors, - datasets=module.DATASETS, - public_only=search_params.get("public_only", False), - src_ip_filter=src_ip_for_query, - dest_ip_filter=dest_ip_for_query, - any_ip_filter=any_ip_for_query, - direction=search_params.get("direction"), - time_from=search_params.get("time_from"), - time_to=search_params.get("time_to"), - ) - - if search_params.get("profile"): - body["profile"] = True - - # Cache is meaningless for profile runs (timing data is request-specific). - use_cache = search_params.get("use_cache", False) and not search_params.get("profile") - raw = None - cache_key = hashlib.md5(json.dumps(body, sort_keys=True).encode()).hexdigest()[:10] - cpath = _cache_path(cache_key) - - if use_cache: - raw = _load_cache(cpath) - if raw: - console.print(f"[dim]Using cached response: {cpath}[/dim]") - - if raw is None: - console.print( - f"[dim]Querying OpenSearch / Malcolm" - f" ({search_params.get('time_range', 'now-24h')})...[/dim]" - ) - raw = query_opensearch(body, params) - if use_cache: - _save_cache(raw, cpath) - - if search_params.get("profile"): - display_profile(raw) - - hits = raw.get("hits", {}).get("hits", []) - if not hits: - console.print("[yellow]No records returned.[/yellow]") - return [] - - # Pre-parse hook — e.g. x509 community_id batch lookup, pe fuid→hash lookup. - module.prepare_hits(hits) - - records = [module.parse_hit(hit.get("_source", {})) for hit in hits] - records = [r for r in records if r] - - # Apply post-filters (over-fetch strategy: fetch 3× then truncate). - if post_filters: - keep = lambda r: all(pf(r) for pf in post_filters) # noqa: E731 - records = [r for r in records if keep(r)] - if len(records) < requested_limit: - console.print( - f"[dim]Showing {len(records)}/{requested_limit} after post-filtering — " - f"increase time range for more results.[/dim]" - ) - records = records[:requested_limit] - - return deduplicate_zeek(records, module.dedup_key) - - -# --------------------------------------------------------------------------- -# Interactive loop -# --------------------------------------------------------------------------- - - -def interactive_loop(records: list, search_params: dict, module, query_fn=None) -> None: - """Interactive post-display prompt — dispatch to module for protocol actions. - - query_fn: callable with signature (module, search_params) -> list. - Defaults to run_query when None. - """ - _query_fn = query_fn if query_fn is not None else run_query - - from src.enricher.threat_intel import enrich_ip - from src.mantis.mantis_search import ( - display_results as display_mantis, - ) - from src.mantis.mantis_search import ( - search as mantis_search, - ) - from src.mantis.mantis_search import ( - sensor_to_project, - ) - - last_record: dict | None = None - - while True: - console.print("") - if last_record: - r = last_record["record"] - hint = module.describe_record(r) - console.print(f"[dim]↩ Last: #{last_record['idx']} {hint}[/dim]") - console.print( - "[bold cyan]Action[/bold cyan] — enter record #" - " / \\[r]e-search / \\[p]rint (CTRL+C to exit):" - ) - try: - raw = prompt(" > ").strip().lower() - except KeyboardInterrupt: - if confirm_exit(): - break - continue - - if raw in ("r", "research", "search"): - search_params = _search_again_prompt(search_params, module) - new_records = _query_fn(module, search_params) - if new_records: - records = new_records - module.display(records) - continue - - if raw in ("p", "print"): - module.display(records) - continue - - if not raw: - continue - - readline.add_history(raw) - - try: - idx = int(raw) - 1 - if idx < 0: - raise ValueError - record = records[idx] - except (ValueError, IndexError): - console.print("[red]Invalid selection.[/red]") - continue - - src_ip = record.get("src_ip", "") - module.display_detail(record, idx + 1) - - # Build action menu dynamically based on module capabilities. - action_parts = [] - if module.SUPPORTS_ENRICHMENT: - action_parts.append("\\[e]nrich") - if module.SUPPORTS_FP: - action_parts.append("\\[f]alse positive") - action_parts += ["\\[m]antis search", "\\[s]kip"] - console.print(" " + " ".join(action_parts)) - - try: - action = prompt(" Action: ").strip().lower() - except KeyboardInterrupt: - console.print("[dim]Cancelled.[/dim]") - continue - - key = action[:1] - if key: - readline.add_history(action) - - if key == "e" and module.SUPPORTS_ENRICHMENT: - hash_val = record.get("sha256") or record.get("md5") or record.get("file_hash") - if hash_val and not src_ip: - # Hash-only enrichment (e.g. pe module — no IP). - from src.enricher.virustotal import check_hash, display_hash - - console.print(f"[dim]Querying VirusTotal for {hash_val[:16]}…[/dim]") - display_hash(hash_val, check_hash(hash_val)) - elif hash_val: - # Both hash and IP available — ask which. - console.print(" \\[h]ash (VirusTotal file lookup) \\[i]p enrichment") - try: - sub = prompt(" Choice: ").strip().lower() - except KeyboardInterrupt: - console.print("[dim]Cancelled.[/dim]") - sub = "" - if sub.startswith("h"): - from src.enricher.virustotal import check_hash, display_hash - - console.print(f"[dim]Querying VirusTotal for {hash_val[:16]}…[/dim]") - display_hash(hash_val, check_hash(hash_val)) - else: - enrich_ip(src_ip) - else: - enrich_ip(src_ip) - elif key == "f" and module.SUPPORTS_FP: - module.fp_action(record) - elif key == "m": - dest_ip = record.get("dest_ip", "") - queries = dict.fromkeys(ip for ip in [src_ip, dest_ip] if ip) - city = sensor_to_project(record.get("sensor", "all")) - if city: - console.print(f"[dim]Filtering Mantis to project '{city}'[/dim]") - combined: list = [] - seen_ids: set = set() - for q in queries: - console.print(f"[dim]Searching Mantis for '{q}'...[/dim]") - for r in mantis_search(q, city=city): - if r["id"] not in seen_ids: - combined.append(r) - seen_ids.add(r["id"]) - display_mantis(combined) - last_record = {"idx": idx + 1, "record": record} - - -# --------------------------------------------------------------------------- -# Search-again prompt -# --------------------------------------------------------------------------- - - -def _search_again_prompt(current: dict, module) -> dict: - """Prompt for new search parameters, shared + module-specific.""" - console.print( - "\n[bold cyan]New Search Parameters[/bold cyan] [dim](Enter to keep current)[/dim]" - ) - console.print("[dim]Time ranges: " + " ".join(TIME_RANGES) + "[/dim]") - - def _ask(label: str, current_val) -> str: - """Prompt for a value; returns user input or preserves current (empty string for None).""" - display = "" if current_val is None else str(current_val) - try: - val = prompt(f" {label} [{display}]: ").strip() - return val if val else display - except KeyboardInterrupt: - console.print("") - return display - - new = dict(current) - new["time_range"] = _ask("Time range", current.get("time_range", "now-24h")) - if module.SENSOR_PARAM is not None: - new[module.SENSOR_PARAM] = _ask( - "Sensor (comma-sep or 'all')", current.get(module.SENSOR_PARAM, "all") - ) - - if module.SUPPORTS_IP_FILTER: - pub_raw = _ask("Public only (y/n)", "y" if current.get("public_only") else "n") - new["public_only"] = pub_raw.lower() in ("y", "yes") - - src_raw = _ask("Src IP filter (blank to clear)", current.get("src_ip")) - new["src_ip"] = src_raw if src_raw else None - - dir_raw = _ask( - "Direction filter (inbound/outbound/internal/external/blank)", - current.get("direction"), - ) - new["direction"] = dir_raw if dir_raw else None - - limit_raw = _ask("Limit", current.get("limit", 500)) - try: - new["limit"] = int(limit_raw) - except ValueError: - new["limit"] = current.get("limit", 500) - - # Module-specific search params - module.add_search_params_prompt(new, _ask) - - return new - - -# --------------------------------------------------------------------------- -# Diagnostic helpers -# --------------------------------------------------------------------------- - - -def list_sensors(time_range: str = "now-7d") -> None: - """Aggregate on host.name and print all known sensors.""" - body = { - "size": 0, - "query": { - "bool": { - "filter": [ - {"range": {"@timestamp": {"gte": time_range, "lte": "now"}}}, - {"exists": {"field": "event.dataset"}}, - ] - } - }, - "aggs": { - "sensors": { - "terms": { - "field": "host.name", - "size": 500, - "order": {"_count": "desc"}, - } - } - }, - } - params = {"path": f"{INDEX}/_search", "method": "POST"} - - console.print(f"[dim]Querying host.name values ({time_range})...[/dim]") - try: - raw = query_opensearch(body, params) - except (OpenSearchConnectionError, OpenSearchAuthError) as exc: - console.print(f"[red]{exc}[/red]") - return - - buckets = raw.get("aggregations", {}).get("sensors", {}).get("buckets", []) - if not buckets: - console.print("[yellow]No sensors found in the given time range.[/yellow]") - return - - table = Table( - title=f"Malcolm sensors (past {time_range.replace('now-', '')})", - box=box.SIMPLE_HEAVY, - ) - table.add_column("host.name", style="cyan") - table.add_column("Record count", justify="right") - - for bucket in buckets: - table.add_row(bucket["key"], str(bucket["doc_count"])) - - console.print(table) - console.print( - f"[dim]{len(buckets)} sensor(s) found. " - f"Pass one or more to --sensor as a comma-separated list.[/dim]" - ) - - -def list_log_types(time_range: str = "now-7d") -> None: - """Aggregate on event.dataset and print all Zeek log types present.""" - body = { - "size": 0, - "query": { - "bool": { - "filter": [ - {"range": {"@timestamp": {"gte": time_range, "lte": "now"}}}, - {"exists": {"field": "event.dataset"}}, - ] - } - }, - "aggs": { - "log_types": { - "terms": { - "field": "event.dataset", - "size": 200, - "order": {"_count": "desc"}, - } - } - }, - } - params = {"path": f"{INDEX}/_search", "method": "POST"} - - console.print(f"[dim]Querying event.dataset values ({time_range})...[/dim]") - try: - raw = query_opensearch(body, params) - except (OpenSearchConnectionError, OpenSearchAuthError) as exc: - console.print(f"[red]{exc}[/red]") - return - - buckets = raw.get("aggregations", {}).get("log_types", {}).get("buckets", []) - if not buckets: - console.print("[yellow]No log types found in the given time range.[/yellow]") - return - - table = Table( - title=f"Zeek log types in Malcolm (past {time_range.replace('now-', '')})", - box=box.SIMPLE_HEAVY, - ) - table.add_column("event.dataset (log type)", style="cyan") - table.add_column("Record count", justify="right") - - for bucket in buckets: - table.add_row(bucket["key"], str(bucket["doc_count"])) - - console.print(table) - console.print( - f"[dim]{len(buckets)} log type(s) found. Pass one to --log-type (or 'all').[/dim]" - ) - - -def list_indices() -> None: - """List all indices in the cluster sorted by doc count.""" - try: - base_url, session = _opensearch_session() - except (OpenSearchConnectionError, OpenSearchAuthError) as exc: - console.print(f"[red]{exc}[/red]") - return - - try: - resp = session.post( - base_url + "/api/console/proxy", - params={ - "path": ( - "_cat/indices?format=json&s=docs.count:desc" - "&h=index,docs.count,store.size,health" - ), - "method": "GET", - }, - timeout=30, - ) - except requests.RequestException as exc: - console.print(f"[red]Cannot reach OpenSearch at {base_url}: {exc}[/red]") - return - - if not resp.ok: - console.print(f"[red]OpenSearch error {resp.status_code}: {resp.text[:300]}[/red]") - return - - indices = resp.json() - if not indices: - console.print("[yellow]No indices found.[/yellow]") - return - - table = Table(title="OpenSearch indices (sorted by doc count)", box=box.SIMPLE_HEAVY) - table.add_column("Index", style="cyan") - table.add_column("Docs", justify="right") - table.add_column("Size", justify="right") - table.add_column("Health", justify="center") - - for entry in indices: - health = entry.get("health", "") - health_color = {"green": "green", "yellow": "yellow", "red": "red"}.get(health, "white") - table.add_row( - entry.get("index", ""), - entry.get("docs.count", "—"), - entry.get("store.size", "—"), - f"[{health_color}]{health}[/{health_color}]", - ) - - console.print(table) - console.print( - f"[dim]Current INDEX pattern: '{INDEX}' — update the INDEX constant if needed.[/dim]" - ) - - -def match_all_sample(time_range: str = "now-24h", limit: int = 3) -> None: - """Fire a plain match_all to confirm data exists and show actual field names.""" - body = { - "size": limit, - "sort": [{"@timestamp": {"order": "desc"}}], - "query": {"bool": {"must": [{"range": {"@timestamp": {"gte": time_range, "lte": "now"}}}]}}, - } - params = {"path": f"{INDEX}/_search", "method": "POST"} - - console.print(f"[dim]match_all against '{INDEX}' ({time_range}, limit {limit})...[/dim]") - try: - raw = query_opensearch(body, params) - except (OpenSearchConnectionError, OpenSearchAuthError) as exc: - console.print(f"[red]{exc}[/red]") - return - - total = raw.get("hits", {}).get("total", {}) - total_val = total.get("value", 0) if isinstance(total, dict) else total - console.print(f"[bold]Total hits:[/bold] {total_val}") - - hits = raw.get("hits", {}).get("hits", []) - if not hits: - console.print( - "[yellow]No hits — index pattern may not match any indices," - " or no data in this time range.[/yellow]" - ) - return - - for i, hit in enumerate(hits, 1): - console.print(f"\n[bold cyan]── Hit {i} ──[/bold cyan]") - console.print(json.dumps(hit.get("_source", {}), indent=2, default=str)) - - -# --------------------------------------------------------------------------- -# ZeekModule base class -# --------------------------------------------------------------------------- - - -class ZeekModule: - """Protocol module interface. Subclass and override methods as needed.""" - - DATASETS: list = ["all"] - SOURCE_FIELDS: list = [] - DETAIL_FIELDS: list = [] # List of (label: str, value_fn: Callable[[dict], str]) - SENSOR_PARAM: str | None = "sensor" # Set to None to skip the sensor prompt in re-search - SUPPORTS_IP_FILTER: bool = True # Set False for metadata-only modules (pe, capture_loss) - SUPPORTS_ENRICHMENT: bool = True # Set False for modules with no IPs or hashes to enrich - SUPPORTS_FP: bool = True # Set False for diagnostic modules (capture_loss) - WEB_CATEGORY: str = "core" # Category group for the web UI sidebar - WEB_ICON: str = "fa-question" # Font Awesome icon class for the web UI sidebar - WEB_COLUMNS: list = [] # List of (header: str, value_fn: Callable[[dict], str]) for web table - EXTRA_PARAMS: list[str] = [] # Protocol-specific search_params keys forwarded from HTTP request - SUMMARY_FIELD: str | None = None # OpenSearch field to aggregate on for the browse modal - SUMMARY_PARAM: str | None = None # EXTRA_PARAMS key that filters on SUMMARY_FIELD - SUMMARY_TYPE: str = "flat" # "flat" = single field agg, "grouped" = scripted prefix + severity - - def build_extra_must(self, search_params: dict) -> tuple: - """Return (must_clauses, post_filters) built from search_params. - - must_clauses: list of OpenSearch DSL clause dicts added to the query must. - post_filters: list of callables (record: dict) -> bool applied after parsing. - When non-empty, run_query() uses 3× over-fetch automatically. - """ - return [], [] - - def prepare_hits(self, hits: list) -> None: - """Pre-parse hook called on raw hits before parse_hit() loop. - - Override for batch lookups (e.g. x509 community_id pivot, pe fuid→hash lookup). - Default is a no-op. - """ - - def parse_hit(self, src: dict) -> dict: - """Convert an OpenSearch _source dict to a normalised record dict. - - Must include at minimum: timestamp, sensor, src_ip, dest_ip, - dest_port, src_port, _raw. - """ - raise NotImplementedError - - def dedup_key(self, record: dict) -> tuple: - """Return the grouping key tuple for deduplicate_zeek.""" - raise NotImplementedError - - def display(self, records: list) -> None: - """Render records as a Rich table.""" - raise NotImplementedError - - def display_detail(self, record: dict, idx: int) -> None: - """Render a Rich Panel with every DETAIL_FIELDS field for the selected record.""" - from rich.panel import Panel - - grid = Table.grid(padding=(0, 2)) - grid.add_column(style="dim", no_wrap=True, min_width=12) - grid.add_column(overflow="fold") - for label, fn in self.DETAIL_FIELDS: - grid.add_row(label, fn(record)) - console.print( - Panel( - grid, - title=f"[bold]#{idx}[/bold] {self.describe_record(record)}", - expand=False, - ) - ) - - def add_args(self, parser) -> None: - """Add protocol-specific argparse arguments to the shared parser.""" - pass - - def describe_record(self, record: dict) -> str: - """One-line summary used in the interactive loop hint line.""" - src = record.get("src_ip", "?") - dst = record.get("dest_ip", "?") - port = record.get("dest_port", "?") - return f"{src} → {dst}:{port}" - - def fp_signature(self, record: dict) -> str: - """Signature string embedded in the FP alert dict.""" - return "zeek/unknown" - - def fp_action(self, record: dict) -> None: - """Handle the [f]alse positive action. Override for custom behaviour.""" - from src.querier.fp_manager import create_filter_interactive - - fp_alert = { - "src_ip": record.get("src_ip"), - "dest_ip": record.get("dest_ip"), - "dest_port": record.get("dest_port"), - "alert": { - "signature": self.fp_signature(record), - "severity": 3, - }, - "clientID": (record.get("sensors") or [record.get("sensor", "")])[0], - } - create_filter_interactive(alert=fp_alert) - - def add_search_params_prompt(self, new: dict, _ask) -> None: - """Prompt for protocol-specific re-search parameters. - - Override to append module-specific fields to `new`. - `_ask(label, current_val)` returns the user-entered string or the - current value's string form if the user pressed Enter. - """ - pass From ecadce4ce97b92505c06a2ea1d92a935c9d1f387 Mon Sep 17 00:00:00 2001 From: liamadale Date: Mon, 11 May 2026 21:58:06 -0700 Subject: [PATCH 086/109] feat(filter-loader): validate category/subcategory against categories.yaml Added _load_categories() which parses filters/categories.yaml into a {category: set(subcategories)} registry. load_filters() now checks each filter file's category and subcategory fields against this registry and appends a descriptive error for any stale or unrecognised values. This catches broken references at load time rather than silently accepting filter files that reference removed categories. --- src/querier/filter_loader.py | 39 ++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/querier/filter_loader.py b/src/querier/filter_loader.py index 23af5fd..8b6cf5d 100755 --- a/src/querier/filter_loader.py +++ b/src/querier/filter_loader.py @@ -33,6 +33,27 @@ def _max_mtime(filters_dir: str) -> float: return max_t +def _load_categories(filters_dir: str) -> dict[str, set[str]]: + """Load categories.yaml and return {category: set(subcategories)}. + + Returns an empty dict if the file is absent or unparseable (non-fatal). + """ + cat_path = os.path.join(filters_dir, "categories.yaml") + try: + with open(cat_path) as fh: + raw = yaml.safe_load(fh) + except (OSError, yaml.YAMLError): + return {} + + if not isinstance(raw, dict): + return {} + registry: dict[str, set[str]] = {} + for cat, meta in raw.get("categories", {}).items(): + subs = meta.get("subcategories", []) if isinstance(meta, dict) else [] + registry[cat] = set(subs) + return registry + + def load_filters( filters_dir: str, municipality: str | None = None, @@ -72,6 +93,8 @@ def load_filters( "errors": [f"filters_dir not found: {filters_dir}"], } + categories = _load_categories(filters_dir) + for root, _dirs, files in os.walk(filters_dir): for fname in sorted(files): if not fname.endswith(".yaml") and not fname.endswith(".yml"): @@ -95,6 +118,22 @@ def load_filters( errors.append(f"{fpath}: expected a YAML mapping at top level") continue + # Validate category/subcategory against the registry when present. + if categories: + cat = data.get("category") + sub = data.get("subcategory") + if cat is not None and cat not in categories: + errors.append( + f"{fpath}: unknown category '{cat}' (valid: {sorted(categories)})" + ) + elif cat is not None and sub is not None: + valid_subs = categories[cat] + if valid_subs and sub not in valid_subs: + errors.append( + f"{fpath}: unknown subcategory '{sub}' for category '{cat}' " + f"(valid: {sorted(valid_subs)})" + ) + # Respect enabled flag (default True) if not data.get("enabled", True): continue From 8ffecb14bc6aabe135d246e7824c18cf9f05417c Mon Sep 17 00:00:00 2001 From: liamadale Date: Mon, 11 May 2026 21:58:14 -0700 Subject: [PATCH 087/109] chore: add console entry points and httpx/flask[async] dependencies Added 14 pisces-* console scripts to [project.scripts] covering all CLI tools and standalone web servers, so installed users can launch them without knowing the module path. Added main() callables to all run.py launchers and run_all.py to satisfy the entry point protocol. Dependencies: httpx>=0.28.1 (sync + async OpenSearch client), flask[async] (brings in asgiref for async view support). --- apps/dashboard_web/run.py | 9 +++++++-- apps/hub/run.py | 10 +++++++--- apps/mantis_explorer/run.py | 11 +++++++++-- apps/opensearch_web/run.py | 9 +++++++-- apps/threat_model/run.py | 8 +++++++- pyproject.toml | 21 ++++++++++++++++++++- run_all.py | 8 +++++++- uv.lock | 20 ++++++++++++++++++-- 8 files changed, 82 insertions(+), 14 deletions(-) diff --git a/apps/dashboard_web/run.py b/apps/dashboard_web/run.py index e14cd6e..f2fbbb5 100644 --- a/apps/dashboard_web/run.py +++ b/apps/dashboard_web/run.py @@ -21,7 +21,9 @@ app = create_app() -if __name__ == "__main__": + +def main() -> None: + """Entry point for `pisces-dashboard` console script.""" parser = argparse.ArgumentParser(description="PISCES Dashboard") parser.add_argument("--host", default="0.0.0.0", help="Bind address (default: 0.0.0.0)") parser.add_argument("--port", type=int, default=5004, help="Port (default: 5004)") @@ -29,5 +31,8 @@ "--debug", action="store_true", default=False, help="Enable Flask debug mode" ) args = parser.parse_args() - app.run(host=args.host, port=args.port, debug=args.debug, threaded=True) + + +if __name__ == "__main__": + main() diff --git a/apps/hub/run.py b/apps/hub/run.py index d366d09..8c5a10c 100644 --- a/apps/hub/run.py +++ b/apps/hub/run.py @@ -15,12 +15,16 @@ from apps.hub.app import create_app -if __name__ == "__main__": + +def main() -> None: + """Entry point for `pisces-hub` console script.""" parser = argparse.ArgumentParser(description="PISCES Hub Portal") parser.add_argument("--host", default="0.0.0.0") parser.add_argument("--port", type=int, default=5000) parser.add_argument("--debug", action="store_true", default=False) args = parser.parse_args() + create_app().run(host=args.host, port=args.port, debug=args.debug, threaded=True) - app = create_app() - app.run(host=args.host, port=args.port, debug=args.debug, threaded=True) + +if __name__ == "__main__": + main() diff --git a/apps/mantis_explorer/run.py b/apps/mantis_explorer/run.py index 36ca070..97128f5 100644 --- a/apps/mantis_explorer/run.py +++ b/apps/mantis_explorer/run.py @@ -3,8 +3,15 @@ from apps.mantis_explorer.app import create_app from apps.mantis_explorer.data import SLUG_TO_ORG, get_report -if __name__ == "__main__": +app = create_app() + + +def main() -> None: + """Entry point for `pisces-mantis-explorer` console script.""" report = get_report("", "") print(f"Mantis Explorer: {len(SLUG_TO_ORG)} institutions, {len(report)} students") - app = create_app() app.run(host="0.0.0.0", port=5005, debug=False, threaded=True) + + +if __name__ == "__main__": + main() diff --git a/apps/opensearch_web/run.py b/apps/opensearch_web/run.py index 5f96dd0..28aa2c6 100755 --- a/apps/opensearch_web/run.py +++ b/apps/opensearch_web/run.py @@ -21,7 +21,9 @@ app = create_app() -if __name__ == "__main__": + +def main() -> None: + """Entry point for `pisces-opensearch` console script.""" parser = argparse.ArgumentParser(description="PISCES Web UI") parser.add_argument("--host", default="0.0.0.0", help="Bind address (default: 0.0.0.0)") parser.add_argument("--port", type=int, default=5001, help="Port (default: 5001)") @@ -29,5 +31,8 @@ "--debug", action="store_true", default=False, help="Enable Flask debug mode" ) args = parser.parse_args() - app.run(host=args.host, port=args.port, debug=args.debug, threaded=True) + + +if __name__ == "__main__": + main() diff --git a/apps/threat_model/run.py b/apps/threat_model/run.py index bf20c93..64bbdd8 100755 --- a/apps/threat_model/run.py +++ b/apps/threat_model/run.py @@ -14,7 +14,9 @@ app = create_app() -if __name__ == "__main__": + +def main() -> None: + """Entry point for `pisces-threat-model` console script.""" print( f"Loaded: {len(TICKETS_BY_ID):,} tickets | " f"{len(MALICIOUS_ROWS):,} malicious IPs | " @@ -22,3 +24,7 @@ ) print("Threat Modeling → http://0.0.0.0:5003/") app.run(host="0.0.0.0", port=5003, debug=False, threaded=True) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 93fb2df..53166cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,15 +8,34 @@ dependencies = [ "pyyaml>=6.0.3", "rich>=15.0.0", "beautifulsoup4>=4.14.3", - "flask>=3.0", + "flask[async]>=3.0", "geoip2>=5.2.0", "orjson>=3.11.8", "plotext>=5.3.2", "cryptography>=46.0.7", "pygments>=2.20.0", "python-multipart>=0.0.26", + "httpx>=0.28.1", ] +[project.scripts] +# CLI tools +pisces-query = "src.querier.opensearch_querier:main" +pisces-enrich = "src.enricher.threat_intel:main" +pisces-mantis = "src.mantis.mantis_search:main" +pisces-mantis-index = "src.mantis.mantis_index:main" +pisces-mantis-report = "src.mantis.activity_report:main" +pisces-threat-model = "src.mantis.mantis_threat_model:main" +pisces-fleet-scan = "src.profiler.fleet_scanner:main" +pisces-profile = "src.profiler.device_profiler:main" +# Web servers (standalone) +pisces-all = "run_all:main" +pisces-opensearch = "apps.opensearch_web.run:main" +pisces-dashboard = "apps.dashboard_web.run:main" +pisces-hub = "apps.hub.run:main" +pisces-threat-model-web = "apps.threat_model.run:main" +pisces-mantis-explorer = "apps.mantis_explorer.run:main" + [project.optional-dependencies] # AI assistant integration via MCP servers (Claude Code, Claude Desktop, kiro-cli) mcp = ["mcp[cli]>=1.0.0"] diff --git a/run_all.py b/run_all.py index 9becd38..f859687 100755 --- a/run_all.py +++ b/run_all.py @@ -36,7 +36,9 @@ }, ) -if __name__ == "__main__": + +def main() -> None: + """Entry point for `pisces-all` console script.""" parser = argparse.ArgumentParser(description="PISCES Combined Web UI") parser.add_argument("--host", default="0.0.0.0") parser.add_argument("--port", type=int, default=5000) @@ -50,3 +52,7 @@ use_debugger=args.debug, threaded=True, ) + + +if __name__ == "__main__": + main() diff --git a/uv.lock b/uv.lock index fe8ef77..3eaee05 100644 --- a/uv.lock +++ b/uv.lock @@ -140,6 +140,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/38/0e/27be9fdef66e72d64c0cdc3cc2823101b80585f8119b5c112c2e8f5f7dab/anyio-4.12.1-py3-none-any.whl", hash = "sha256:d405828884fc140aa80a3c667b8beed277f1dfedec42ba031bd6ac3db606ab6c", size = 113592, upload-time = "2026-01-06T11:45:19.497Z" }, ] +[[package]] +name = "asgiref" +version = "3.11.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/63/40/f03da1264ae8f7cfdbf9146542e5e7e8100a4c66ab48e791df9a03d3f6c0/asgiref-3.11.1.tar.gz", hash = "sha256:5f184dc43b7e763efe848065441eac62229c9f7b0475f41f80e207a114eda4ce", size = 38550, upload-time = "2026-02-03T13:30:14.33Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/0a/a72d10ed65068e115044937873362e6e32fab1b7dce0046aeb224682c989/asgiref-3.11.1-py3-none-any.whl", hash = "sha256:e8667a091e69529631969fd45dc268fa79b99c92c5fcdda727757e52146ec133", size = 24345, upload-time = "2026-02-03T13:30:13.039Z" }, +] + [[package]] name = "attrs" version = "25.4.0" @@ -653,6 +662,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/9c/34f6962f9b9e9c71f6e5ed806e0d0ff03c9d1b0b2340088a0cf4bce09b18/flask-3.1.3-py3-none-any.whl", hash = "sha256:f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c", size = 103424, upload-time = "2026-02-19T05:00:56.027Z" }, ] +[package.optional-dependencies] +async = [ + { name = "asgiref" }, +] + [[package]] name = "frozenlist" version = "1.8.0" @@ -1485,8 +1499,9 @@ source = { virtual = "." } dependencies = [ { name = "beautifulsoup4" }, { name = "cryptography" }, - { name = "flask" }, + { name = "flask", extra = ["async"] }, { name = "geoip2" }, + { name = "httpx" }, { name = "orjson" }, { name = "plotext" }, { name = "pygments" }, @@ -1528,8 +1543,9 @@ dev = [ requires-dist = [ { name = "beautifulsoup4", specifier = ">=4.14.3" }, { name = "cryptography", specifier = ">=46.0.7" }, - { name = "flask", specifier = ">=3.0" }, + { name = "flask", extras = ["async"], specifier = ">=3.0" }, { name = "geoip2", specifier = ">=5.2.0" }, + { name = "httpx", specifier = ">=0.28.1" }, { name = "mcp", extras = ["cli"], marker = "extra == 'all'", specifier = ">=1.0.0" }, { name = "mcp", extras = ["cli"], marker = "extra == 'mcp'", specifier = ">=1.0.0" }, { name = "orjson", specifier = ">=3.11.8" }, From 3e4b8d61f581f1570c11e75245505fcbc41f3160 Mon Sep 17 00:00:00 2001 From: liamadale Date: Mon, 11 May 2026 21:58:23 -0700 Subject: [PATCH 088/109] feat(enricher): add prewarm_enrichment_cache for background cache warming MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit prewarm_enrichment_cache(ips, max_ips=50) filters out already-cached and private IPs, then submits the remainder to a daemon ThreadPoolExecutor (max 4 workers) as fire-and-forget enrich_ip() calls. Errors are suppressed — pre-warming is best-effort and must never block callers. The 50-IP cap prevents hitting API rate limits on large result sets. --- src/enricher/threat_intel.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/enricher/threat_intel.py b/src/enricher/threat_intel.py index 8dfbe4f..dc447df 100755 --- a/src/enricher/threat_intel.py +++ b/src/enricher/threat_intel.py @@ -51,6 +51,30 @@ def _cache_put(ip: str, result: dict) -> None: _enrich_cache[ip] = (time.time(), result) +def prewarm_enrichment_cache(ips: list[str], max_ips: int = 50) -> None: + """Submit background enrichment for IPs not already in the cache. + + Runs as fire-and-forget in daemon threads so it never blocks the caller. + Limited to the first *max_ips* uncached IPs to avoid hammering rate-limited APIs. + """ + from src.querier.builder import is_private + + uncached = [ip for ip in ips if ip and not is_private(ip) and _cache_get(ip) is None][:max_ips] + + if not uncached: + return + + def _warm(ip: str) -> None: + try: + enrich_ip(ip, offer_fp=False) + except Exception: + pass # best-effort; errors are silently skipped for pre-warming + + with ThreadPoolExecutor(max_workers=4, thread_name_prefix="enrich-prewarm") as pool: + for ip in uncached: + pool.submit(_warm, ip) + + def enrich_ip(ip: str, offer_fp: bool = True, urls_only: bool = False) -> dict: """Run the full enrichment pipeline for a single IP. From 9234e5128d626b28cc71823eca4a3a3cdcaef2e4 Mon Sep 17 00:00:00 2001 From: liamadale Date: Mon, 11 May 2026 21:58:29 -0700 Subject: [PATCH 089/109] perf(web): replace thread-pool fan-out with asyncio.gather on overview route run_cross_protocol_query_async uses asyncio.gather across all IP-capable log types with a shared httpx.AsyncClient, replacing the previous ThreadPoolExecutor approach. This avoids per-request thread creation and allows the connection pool to be reused across the concurrent sub-queries. The overview route is now async and calls prewarm_enrichment_cache after results arrive, so enrichment data is ready in the background by the time the user clicks an IP. --- apps/opensearch_web/app.py | 13 +++++-- apps/opensearch_web/queries.py | 69 ++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/apps/opensearch_web/app.py b/apps/opensearch_web/app.py index 0700c69..1ed3d03 100644 --- a/apps/opensearch_web/app.py +++ b/apps/opensearch_web/app.py @@ -11,7 +11,7 @@ POOL, build_search_params_from_request, cached_run_query, - run_cross_protocol_query, + run_cross_protocol_query_async, ) from apps.shared.blueprints import ( make_cache_blueprint, @@ -136,14 +136,21 @@ def inject_nav_data() -> dict: # GET / — cross-protocol IP matrix # ------------------------------------------------------------------ @app.route("/") - def overview(): + async def overview(): + from src.enricher.threat_intel import prewarm_enrichment_cache + search_params = build_search_params_from_request(request) error = None rows = [] try: - rows = run_cross_protocol_query(search_params) + rows = await run_cross_protocol_query_async(search_params) except (OpenSearchConnectionError, OpenSearchAuthError) as exc: error = str(exc) + + # Pre-warm the enrichment cache for all source IPs while the page renders. + if rows: + prewarm_enrichment_cache([r["src_ip"] for r in rows]) + # Build per-category module lists, excluding non-IP modules (pe, capture_loss) ip_modules_by_category = { cat: [lt for lt in lts if MODULES[lt].SUPPORTS_IP_FILTER] diff --git a/apps/opensearch_web/queries.py b/apps/opensearch_web/queries.py index 5de7eb4..96d2238 100644 --- a/apps/opensearch_web/queries.py +++ b/apps/opensearch_web/queries.py @@ -1,10 +1,12 @@ """Web-layer query helpers — bridge between HTTP request params and run_query().""" +import asyncio import os from collections import defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed from apps.opensearch_web import cache as wcache +from src.querier.runner import run_query_async from src.querier.zeek_modules import MODULES from src.querier.zeek_modules.base import ( OpenSearchAuthError, @@ -104,3 +106,70 @@ def run_cross_protocol_query(search_params: dict) -> list: [{"src_ip": ip, **data} for ip, data in ip_data.items()], key=lambda x: -x["total"], ) + + +async def cached_run_query_async(log_type: str, search_params: dict) -> list: + """Async variant of cached_run_query — uses httpx.AsyncClient for web fan-out.""" + cached = wcache.get(log_type, search_params) + if cached is not None: + return cached + + k = wcache.raw_key(log_type, search_params) + event = wcache.claim(k) + + if event is None: + # Another coroutine is already fetching this — yield control until it's done. + loop = asyncio.get_event_loop() + await loop.run_in_executor(None, wcache.wait_inflight, k) + return wcache.get(log_type, search_params) or [] + + try: + records = await run_query_async(MODULES[log_type], search_params) + wcache.put(log_type, search_params, records) + return records + finally: + wcache.release(k) + + +async def run_cross_protocol_query_async(search_params: dict) -> list: + """Async fan-out across all IP-capable log types using httpx.AsyncClient. + + Replaces the ThreadPoolExecutor-based run_cross_protocol_query with + asyncio.gather, which avoids thread creation overhead and shares a single + persistent HTTP/2-capable connection pool via httpx.AsyncClient. + """ + ip_modules = {lt: mod for lt, mod in MODULES.items() if mod.SUPPORTS_IP_FILTER} + first_conn_error: Exception | None = None + + tasks = {lt: cached_run_query_async(lt, search_params) for lt in ip_modules} + task_results = await asyncio.gather(*tasks.values(), return_exceptions=True) + + results_by_type: dict = {} + for lt, result in zip(tasks.keys(), task_results): + if isinstance(result, (OpenSearchConnectionError, OpenSearchAuthError)): + if first_conn_error is None: + first_conn_error = result + results_by_type[lt] = [] + elif isinstance(result, Exception): + results_by_type[lt] = [] + console.print(f"[yellow]Cross-protocol query failed for {lt}: {result}[/yellow]") + else: + results_by_type[lt] = result + + if first_conn_error is not None: + raise first_conn_error + + ip_data: dict = defaultdict(lambda: {"per_protocol": {lt: 0 for lt in ip_modules}, "total": 0}) + for lt, records in results_by_type.items(): + for rec in records: + ip = rec.get("src_ip", "") + if not ip or ip == "—": + continue + freq = rec.get("freq", 1) + ip_data[ip]["per_protocol"][lt] += freq + ip_data[ip]["total"] += freq + + return sorted( + [{"src_ip": ip, **data} for ip, data in ip_data.items()], + key=lambda x: -x["total"], + ) From 729e20b1f005b4744ed616f11730104945077d80 Mon Sep 17 00:00:00 2001 From: liamadale Date: Mon, 11 May 2026 22:02:53 -0700 Subject: [PATCH 090/109] refactor(mantis): consolidate urllib3 warning suppression into __init__ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three modules — mantis_index, mantis_search, and activity_report — each imported urllib3 and called disable_warnings independently. Moved the single call into src/mantis/__init__.py so it fires once on package import, removing the duplication without changing behavior. --- src/mantis/__init__.py | 3 +++ src/mantis/activity_report.py | 3 --- src/mantis/mantis_index.py | 2 -- src/mantis/mantis_search.py | 2 -- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/mantis/__init__.py b/src/mantis/__init__.py index e69de29..edde1a9 100644 --- a/src/mantis/__init__.py +++ b/src/mantis/__init__.py @@ -0,0 +1,3 @@ +import urllib3 + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) diff --git a/src/mantis/activity_report.py b/src/mantis/activity_report.py index 5de7891..1e19d3d 100644 --- a/src/mantis/activity_report.py +++ b/src/mantis/activity_report.py @@ -34,15 +34,12 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) -import urllib3 from dotenv import load_dotenv from rich import box from rich.console import Console from rich.rule import Rule from rich.table import Table -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - _BASE = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) DEFAULT_INDEX = os.path.join(_BASE, "data", "tickets", "indexed", "tickets_index.json") diff --git a/src/mantis/mantis_index.py b/src/mantis/mantis_index.py index 349b3ea..0fcd9bf 100755 --- a/src/mantis/mantis_index.py +++ b/src/mantis/mantis_index.py @@ -17,7 +17,6 @@ from concurrent.futures import ThreadPoolExecutor, as_completed import requests -import urllib3 from dotenv import load_dotenv from rich.console import Console from rich.progress import ( @@ -36,7 +35,6 @@ from src.utils.dns import setup_dns console = Console() -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) _BASE = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) _FETCH_WORKERS = 8 diff --git a/src/mantis/mantis_search.py b/src/mantis/mantis_search.py index 2871068..ea81030 100755 --- a/src/mantis/mantis_search.py +++ b/src/mantis/mantis_search.py @@ -19,7 +19,6 @@ import sys import requests -import urllib3 from dotenv import load_dotenv from rich import box from rich.console import Console @@ -31,7 +30,6 @@ from src.utils.dns import setup_dns console = Console(file=sys.stderr) -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) def sensor_to_project(sensor_val: str) -> str | None: From 3af92e8a132abb19947aef9d74859a28759c2cba Mon Sep 17 00:00:00 2001 From: liamadale Date: Mon, 11 May 2026 22:02:59 -0700 Subject: [PATCH 091/109] chore(deps): drop cryptography, move geoip2 to offline-enrichment extra MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cryptography had no import anywhere in src/, apps/, or mcp/ — it was an accidental transitive dependency that snuck into the explicit list. geoip2 is only lazily imported inside ticket_enrichment/offline.py with a graceful-degradation fallback, so it does not belong in the mandatory dependency list. Moved it to the offline-enrichment optional extra (and all) alongside pyasn, which has the same usage pattern. Updated the extra's comment to document the GeoLite2-City.mmdb requirement as well. --- pyproject.toml | 9 +++++---- uv.lock | 8 ++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 53166cb..ef872c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,10 +9,8 @@ dependencies = [ "rich>=15.0.0", "beautifulsoup4>=4.14.3", "flask[async]>=3.0", - "geoip2>=5.2.0", "orjson>=3.11.8", "plotext>=5.3.2", - "cryptography>=46.0.7", "pygments>=2.20.0", "python-multipart>=0.0.26", "httpx>=0.28.1", @@ -45,10 +43,12 @@ nlp = [ "spacy>=3.7", ] -# Offline ASN/BGP reputation lookup (requires a BGP dump at data/asn_table.dat) -# Download a dump via: python -m pyasn.scripts.pyasn_util_download +# Offline ASN/BGP reputation lookup and GeoIP (requires data files) +# ASN: python -m pyasn.scripts.pyasn_util_download → data/asn_table.dat +# GeoIP: download GeoLite2-City.mmdb from MaxMind → data/ offline-enrichment = [ "pyasn>=1.6.2", + "geoip2>=5.2.0", ] # Everything @@ -56,6 +56,7 @@ all = [ "mcp[cli]>=1.0.0", "spacy>=3.7", "pyasn>=1.6.2", + "geoip2>=5.2.0", ] [dependency-groups] diff --git a/uv.lock b/uv.lock index 3eaee05..a65dcfa 100644 --- a/uv.lock +++ b/uv.lock @@ -1498,9 +1498,7 @@ version = "1.0.0" source = { virtual = "." } dependencies = [ { name = "beautifulsoup4" }, - { name = "cryptography" }, { name = "flask", extra = ["async"] }, - { name = "geoip2" }, { name = "httpx" }, { name = "orjson" }, { name = "plotext" }, @@ -1514,6 +1512,7 @@ dependencies = [ [package.optional-dependencies] all = [ + { name = "geoip2" }, { name = "mcp", extra = ["cli"] }, { name = "pyasn" }, { name = "spacy" }, @@ -1525,6 +1524,7 @@ nlp = [ { name = "spacy" }, ] offline-enrichment = [ + { name = "geoip2" }, { name = "pyasn" }, ] @@ -1542,9 +1542,9 @@ dev = [ [package.metadata] requires-dist = [ { name = "beautifulsoup4", specifier = ">=4.14.3" }, - { name = "cryptography", specifier = ">=46.0.7" }, { name = "flask", extras = ["async"], specifier = ">=3.0" }, - { name = "geoip2", specifier = ">=5.2.0" }, + { name = "geoip2", marker = "extra == 'all'", specifier = ">=5.2.0" }, + { name = "geoip2", marker = "extra == 'offline-enrichment'", specifier = ">=5.2.0" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "mcp", extras = ["cli"], marker = "extra == 'all'", specifier = ">=1.0.0" }, { name = "mcp", extras = ["cli"], marker = "extra == 'mcp'", specifier = ">=1.0.0" }, From 6adfb2a88cc89df78480c0c7a8368443e915b887 Mon Sep 17 00:00:00 2001 From: liamadale Date: Tue, 12 May 2026 10:00:27 -0700 Subject: [PATCH 092/109] fix(mcp/opensearch): push dest_ip filter into ES query instead of post-filtering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _apply_dest_ip_filter() was running as a client-side pass over already- truncated result sets, meaning queries with a dest_ip constraint and a limit of N would silently return fewer than N records — or none — even when matching documents existed in the index. Fix: pass dest_ip through _base_params() so it reaches run_query() and becomes a bool.filter term clause in the Elasticsearch request body, consistent with how src_ip has always been handled. Remove _apply_dest_ip_filter and all 16 call sites across every protocol tool. Two regression tests added to tests/test_zeek_base.py: - dest_ip_filter alone appears in the ES filter clauses - src_ip_filter and dest_ip_filter coexist correctly in the same query --- mcp/opensearch/server.py | 95 +++++++++++++++++++++++----------------- tests/test_zeek_base.py | 38 ++++++++++++++++ 2 files changed, 92 insertions(+), 41 deletions(-) diff --git a/mcp/opensearch/server.py b/mcp/opensearch/server.py index 8a60d44..be7ad27 100644 --- a/mcp/opensearch/server.py +++ b/mcp/opensearch/server.py @@ -79,6 +79,7 @@ def _base_params( src_ip: Optional[str], direction: Optional[str], no_filters: bool, + dest_ip: Optional[str] = None, ) -> dict: params: dict = { "time_range": time_range, @@ -90,18 +91,13 @@ def _base_params( } if src_ip: params["src_ip"] = src_ip + if dest_ip: + params["dest_ip"] = dest_ip if direction: params["direction"] = direction return params -def _apply_dest_ip_filter(records: list, dest_ip: Optional[str]) -> list: - """Post-filter records by destination IP (base.py only natively filters src_ip).""" - if not dest_ip: - return records - return [r for r in records if r.get("dest_ip") == dest_ip] - - # --------------------------------------------------------------------------- # 10 Zeek protocol tools # --------------------------------------------------------------------------- @@ -124,9 +120,10 @@ def search_conn( Common fields: src_ip, dest_ip, dest_port, proto, bytes, duration, sensor. """ try: - params = _base_params(time_range, sensor, limit, public_only, src_ip, direction, no_filters) + params = _base_params( + time_range, sensor, limit, public_only, src_ip, direction, no_filters, dest_ip + ) records = run_query(MODULES["conn"], params) - records = _apply_dest_ip_filter(records, dest_ip) return _ok({"count": len(records), "records": _serialise_records(records)}) except Exception as exc: return _err(str(exc)) @@ -154,7 +151,9 @@ def search_dns( dns_qtype: Query type to filter by, e.g. "A", "MX", "TXT". """ try: - params = _base_params(time_range, sensor, limit, public_only, src_ip, direction, no_filters) + params = _base_params( + time_range, sensor, limit, public_only, src_ip, direction, no_filters, dest_ip + ) if dns_query: params["dns_query"] = dns_query if dns_rcode: @@ -162,7 +161,6 @@ def search_dns( if dns_qtype: params["qtype"] = dns_qtype records = run_query(MODULES["dns"], params) - records = _apply_dest_ip_filter(records, dest_ip) return _ok({"count": len(records), "records": _serialise_records(records)}) except Exception as exc: return _err(str(exc)) @@ -192,7 +190,9 @@ def search_http( status_code: HTTP response status code to filter by. """ try: - params = _base_params(time_range, sensor, limit, public_only, src_ip, direction, no_filters) + params = _base_params( + time_range, sensor, limit, public_only, src_ip, direction, no_filters, dest_ip + ) if http_method: params["http_method"] = http_method if http_host: @@ -202,7 +202,6 @@ def search_http( if status_code is not None: params["status_code"] = status_code records = run_query(MODULES["http"], params) - records = _apply_dest_ip_filter(records, dest_ip) return _ok({"count": len(records), "records": _serialise_records(records)}) except Exception as exc: return _err(str(exc)) @@ -228,13 +227,14 @@ def search_ssl( ssl_invalid_only: If True, return only connections with invalid/self-signed certs. """ try: - params = _base_params(time_range, sensor, limit, public_only, src_ip, direction, no_filters) + params = _base_params( + time_range, sensor, limit, public_only, src_ip, direction, no_filters, dest_ip + ) if ssl_sni: params["ssl_sni"] = ssl_sni if ssl_invalid_only: params["ssl_invalid_only"] = True records = run_query(MODULES["ssl"], params) - records = _apply_dest_ip_filter(records, dest_ip) return _ok({"count": len(records), "records": _serialise_records(records)}) except Exception as exc: return _err(str(exc)) @@ -262,7 +262,9 @@ def search_smtp( smtp_subject: Subject line substring to filter by. """ try: - params = _base_params(time_range, sensor, limit, public_only, src_ip, direction, no_filters) + params = _base_params( + time_range, sensor, limit, public_only, src_ip, direction, no_filters, dest_ip + ) if smtp_mail_from: params["smtp_mail_from"] = smtp_mail_from if smtp_rcpt_to: @@ -270,7 +272,6 @@ def search_smtp( if smtp_subject: params["smtp_subject"] = smtp_subject records = run_query(MODULES["smtp"], params) - records = _apply_dest_ip_filter(records, dest_ip) return _ok({"count": len(records), "records": _serialise_records(records)}) except Exception as exc: return _err(str(exc)) @@ -296,13 +297,14 @@ def search_rdp( rdp_cookie: RDP cookie/username string to filter by. """ try: - params = _base_params(time_range, sensor, limit, public_only, src_ip, direction, no_filters) + params = _base_params( + time_range, sensor, limit, public_only, src_ip, direction, no_filters, dest_ip + ) if rdp_result: params["rdp_result"] = rdp_result if rdp_cookie: params["rdp_cookie"] = rdp_cookie records = run_query(MODULES["rdp"], params) - records = _apply_dest_ip_filter(records, dest_ip) return _ok({"count": len(records), "records": _serialise_records(records)}) except Exception as exc: return _err(str(exc)) @@ -328,13 +330,14 @@ def search_smb( smb_action: SMB action verb to filter by, e.g. "SMB::FILE_OPEN". """ try: - params = _base_params(time_range, sensor, limit, public_only, src_ip, direction, no_filters) + params = _base_params( + time_range, sensor, limit, public_only, src_ip, direction, no_filters, dest_ip + ) if smb_share: params["smb_share"] = smb_share if smb_action: params["smb_action"] = smb_action records = run_query(MODULES["smb"], params) - records = _apply_dest_ip_filter(records, dest_ip) return _ok({"count": len(records), "records": _serialise_records(records)}) except Exception as exc: return _err(str(exc)) @@ -360,13 +363,14 @@ def search_ssh( ssh_auth_result: Auth result string to filter by, e.g. "failure", "success". """ try: - params = _base_params(time_range, sensor, limit, public_only, src_ip, direction, no_filters) + params = _base_params( + time_range, sensor, limit, public_only, src_ip, direction, no_filters, dest_ip + ) if ssh_failed_only: params["ssh_failed_only"] = True if ssh_auth_result is not None: params["ssh_auth_result"] = ssh_auth_result records = run_query(MODULES["ssh"], params) - records = _apply_dest_ip_filter(records, dest_ip) return _ok({"count": len(records), "records": _serialise_records(records)}) except Exception as exc: return _err(str(exc)) @@ -393,11 +397,12 @@ def search_notice( notice_note: Notice type to filter by, e.g. "Scan::Port_Scan". """ try: - params = _base_params(time_range, sensor, limit, public_only, src_ip, direction, no_filters) + params = _base_params( + time_range, sensor, limit, public_only, src_ip, direction, no_filters, dest_ip + ) if notice_note: params["notice_note"] = notice_note records = run_query(MODULES["notice"], params) - records = _apply_dest_ip_filter(records, dest_ip) return _ok({"count": len(records), "records": _serialise_records(records)}) except Exception as exc: return _err(str(exc)) @@ -424,11 +429,12 @@ def search_weird( weird_name: Weird event name to filter by, e.g. "bad_HTTP_reply". """ try: - params = _base_params(time_range, sensor, limit, public_only, src_ip, direction, no_filters) + params = _base_params( + time_range, sensor, limit, public_only, src_ip, direction, no_filters, dest_ip + ) if weird_name: params["weird_name"] = weird_name records = run_query(MODULES["weird"], params) - records = _apply_dest_ip_filter(records, dest_ip) return _ok({"count": len(records), "records": _serialise_records(records)}) except Exception as exc: return _err(str(exc)) @@ -466,7 +472,9 @@ def search_suricata_alert( tag: Filter by tag, e.g. "CISA_KEV", "Exploit", "RAT". """ try: - params = _base_params(time_range, sensor, limit, public_only, src_ip, direction, no_filters) + params = _base_params( + time_range, sensor, limit, public_only, src_ip, direction, no_filters, dest_ip + ) if rule_name: params["rule_name"] = rule_name if rule_category: @@ -480,7 +488,6 @@ def search_suricata_alert( if tag: params["tag"] = tag records = run_query(MODULES["suricata_alert"], params) - records = _apply_dest_ip_filter(records, dest_ip) return _ok({"count": len(records), "records": _serialise_records(records)}) except Exception as exc: return _err(str(exc)) @@ -512,7 +519,9 @@ def search_radius( failed_only: Show only failed authentication attempts. """ try: - params = _base_params(time_range, sensor, limit, public_only, src_ip, None, no_filters) + params = _base_params( + time_range, sensor, limit, public_only, src_ip, None, no_filters, dest_ip + ) if username: params["username"] = username if mac: @@ -520,7 +529,6 @@ def search_radius( if failed_only: params["failed_only"] = failed_only records = run_query(MODULES["radius"], params) - records = _apply_dest_ip_filter(records, dest_ip) return _ok({"count": len(records), "records": _serialise_records(records)}) except Exception as exc: return _err(str(exc)) @@ -547,7 +555,9 @@ def search_sip( user_agent: Filter by User-Agent (substring match). """ try: - params = _base_params(time_range, sensor, limit, public_only, src_ip, None, no_filters) + params = _base_params( + time_range, sensor, limit, public_only, src_ip, None, no_filters, dest_ip + ) if method: params["method"] = method if status_code: @@ -555,7 +565,6 @@ def search_sip( if user_agent: params["user_agent"] = user_agent records = run_query(MODULES["sip"], params) - records = _apply_dest_ip_filter(records, dest_ip) return _ok({"count": len(records), "records": _serialise_records(records)}) except Exception as exc: return _err(str(exc)) @@ -578,11 +587,12 @@ def search_tunnel( tunnel_type: Filter by tunnel type (Tunnel::IP, Tunnel::GRE, etc.). """ try: - params = _base_params(time_range, sensor, limit, public_only, src_ip, None, no_filters) + params = _base_params( + time_range, sensor, limit, public_only, src_ip, None, no_filters, dest_ip + ) if tunnel_type: params["tunnel_type"] = tunnel_type records = run_query(MODULES["tunnel"], params) - records = _apply_dest_ip_filter(records, dest_ip) return _ok({"count": len(records), "records": _serialise_records(records)}) except Exception as exc: return _err(str(exc)) @@ -607,13 +617,14 @@ def search_ntp( version: Filter by NTP version (exact match, integer). """ try: - params = _base_params(time_range, sensor, limit, public_only, src_ip, None, no_filters) + params = _base_params( + time_range, sensor, limit, public_only, src_ip, None, no_filters, dest_ip + ) if mode is not None: params["mode"] = mode if version is not None: params["version"] = version records = run_query(MODULES["ntp"], params) - records = _apply_dest_ip_filter(records, dest_ip) return _ok({"count": len(records), "records": _serialise_records(records)}) except Exception as exc: return _err(str(exc)) @@ -638,13 +649,14 @@ def search_modbus( exceptions_only: Show only records with exception codes. """ try: - params = _base_params(time_range, sensor, limit, public_only, src_ip, None, no_filters) + params = _base_params( + time_range, sensor, limit, public_only, src_ip, None, no_filters, dest_ip + ) if function: params["function"] = function if exceptions_only: params["exceptions_only"] = True records = run_query(MODULES["modbus"], params) - records = _apply_dest_ip_filter(records, dest_ip) return _ok({"count": len(records), "records": _serialise_records(records)}) except Exception as exc: return _err(str(exc)) @@ -667,11 +679,12 @@ def search_dnp3( function: Filter by DNP3 function request (substring match). """ try: - params = _base_params(time_range, sensor, limit, public_only, src_ip, None, no_filters) + params = _base_params( + time_range, sensor, limit, public_only, src_ip, None, no_filters, dest_ip + ) if function: params["function"] = function records = run_query(MODULES["dnp3"], params) - records = _apply_dest_ip_filter(records, dest_ip) return _ok({"count": len(records), "records": _serialise_records(records)}) except Exception as exc: return _err(str(exc)) diff --git a/tests/test_zeek_base.py b/tests/test_zeek_base.py index ada79b9..b6e50e1 100644 --- a/tests/test_zeek_base.py +++ b/tests/test_zeek_base.py @@ -183,6 +183,44 @@ def test_build_base_query_src_ip_filter() -> None: assert ip_clause["term"]["source.ip"] == "198.51.100.1" +def test_build_base_query_dest_ip_filter() -> None: + """dest_ip_filter must appear in the ES filter clauses, not as a post-filter.""" + body, _ = build_base_query( + must_not=[], + extra_must=[], + source_fields=["source.ip", "destination.ip"], + limit=10, + time_range="now-1h", + sensors=None, + datasets=["conn"], + dest_ip_filter="8.8.8.8", + ) + must = body["query"]["bool"]["filter"] + ip_clause = next((c for c in must if "term" in c and "destination.ip" in c["term"]), None) + assert ip_clause is not None, "dest_ip_filter must be pushed into the ES filter clause" + assert ip_clause["term"]["destination.ip"] == "8.8.8.8" + + +def test_build_base_query_src_and_dest_ip_filter() -> None: + """src and dest IP filters must both appear in the ES filter clauses simultaneously.""" + body, _ = build_base_query( + must_not=[], + extra_must=[], + source_fields=["source.ip", "destination.ip"], + limit=10, + time_range="now-1h", + sensors=None, + datasets=["conn"], + src_ip_filter="1.2.3.4", + dest_ip_filter="8.8.8.8", + ) + must = body["query"]["bool"]["filter"] + src_clause = next((c for c in must if "term" in c and "source.ip" in c["term"]), None) + dest_clause = next((c for c in must if "term" in c and "destination.ip" in c["term"]), None) + assert src_clause is not None and src_clause["term"]["source.ip"] == "1.2.3.4" + assert dest_clause is not None and dest_clause["term"]["destination.ip"] == "8.8.8.8" + + def test_build_base_query_public_only_adds_must_not() -> None: body, _ = build_base_query( must_not=[], From 9b5fdf885a9961a069dfeddd058829af4039440f Mon Sep 17 00:00:00 2001 From: liamadale Date: Tue, 12 May 2026 10:20:56 -0700 Subject: [PATCH 093/109] feat(mcp/querier): add port/proto filters, multi-value IP/sensor, and absolute timestamps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three Tier 2 query capability improvements wired from the MCP tool signatures down through builder.py and runner.py to the ES query: - §4.3: dest_port/src_port/proto filters added to search_conn; dest_port added to search_http, search_ssl, search_rdp, search_smb, search_ssh. Values are passed through to build_base_query as term/terms clauses on destination.port, source.port, and network.transport. Port and proto filters are only applied when the module's SOURCE_FIELDS includes the corresponding ES field. - §4.4: src_ip, dest_ip, and sensor now accept list[str]. A list generates a terms clause; a scalar string retains the existing term clause. sensor list bypasses the legacy comma-split path. - §4.6: time_from/time_to absolute ISO 8601 timestamps exposed on all 16 per-protocol search tools. When provided they override time_range and are passed directly to the existing range clause in build_base_query. Tests added for all three features in tests/test_zeek_base.py. --- mcp/opensearch/server.py | 403 ++++++++++++++++++++++++++++++++------- src/querier/builder.py | 35 +++- src/querier/runner.py | 34 +++- tests/test_zeek_base.py | 155 +++++++++++++++ 4 files changed, 548 insertions(+), 79 deletions(-) diff --git a/mcp/opensearch/server.py b/mcp/opensearch/server.py index be7ad27..4814d1d 100644 --- a/mcp/opensearch/server.py +++ b/mcp/opensearch/server.py @@ -73,13 +73,15 @@ def _err(msg: str) -> str: def _base_params( time_range: str, - sensor: str, + sensor: str | list[str], limit: int, public_only: bool, - src_ip: Optional[str], + src_ip: str | list[str] | None, direction: Optional[str], no_filters: bool, - dest_ip: Optional[str] = None, + dest_ip: str | list[str] | None = None, + time_from: Optional[str] = None, + time_to: Optional[str] = None, ) -> dict: params: dict = { "time_range": time_range, @@ -95,6 +97,10 @@ def _base_params( params["dest_ip"] = dest_ip if direction: params["direction"] = direction + if time_from: + params["time_from"] = time_from + if time_to: + params["time_to"] = time_to return params @@ -106,23 +112,50 @@ def _base_params( @mcp.tool() def search_conn( time_range: str = "now-24h", - sensor: str = "all", + sensor: str | list[str] = "all", limit: int = 500, public_only: bool = False, - src_ip: Optional[str] = None, - dest_ip: Optional[str] = None, + src_ip: str | list[str] | None = None, + dest_ip: str | list[str] | None = None, direction: Optional[str] = None, no_filters: bool = False, + dest_port: Optional[int] = None, + src_port: Optional[int] = None, + proto: Optional[str] = None, + time_from: Optional[str] = None, + time_to: Optional[str] = None, ) -> str: """Search Zeek conn (connection) logs from Malcolm/OpenSearch. Returns deduplicated connection records sorted by frequency. Common fields: src_ip, dest_ip, dest_port, proto, bytes, duration, sensor. + + Args: + dest_port: Destination port to filter by, e.g. 443. + src_port: Source port to filter by. + proto: Transport protocol to filter by, e.g. "tcp", "udp", "icmp". + time_from: Absolute start timestamp (ISO 8601), e.g. "2026-04-19T00:00:00Z". + time_to: Absolute end timestamp (ISO 8601). Overrides time_range when both are set. """ try: params = _base_params( - time_range, sensor, limit, public_only, src_ip, direction, no_filters, dest_ip + time_range, + sensor, + limit, + public_only, + src_ip, + direction, + no_filters, + dest_ip, + time_from, + time_to, ) + if dest_port is not None: + params["dest_port"] = dest_port + if src_port is not None: + params["src_port"] = src_port + if proto: + params["proto"] = proto records = run_query(MODULES["conn"], params) return _ok({"count": len(records), "records": _serialise_records(records)}) except Exception as exc: @@ -132,16 +165,18 @@ def search_conn( @mcp.tool() def search_dns( time_range: str = "now-24h", - sensor: str = "all", + sensor: str | list[str] = "all", limit: int = 500, public_only: bool = False, - src_ip: Optional[str] = None, - dest_ip: Optional[str] = None, + src_ip: str | list[str] | None = None, + dest_ip: str | list[str] | None = None, direction: Optional[str] = None, no_filters: bool = False, dns_query: Optional[str] = None, dns_rcode: Optional[str] = None, dns_qtype: Optional[str] = None, + time_from: Optional[str] = None, + time_to: Optional[str] = None, ) -> str: """Search Zeek DNS logs from Malcolm/OpenSearch. @@ -149,10 +184,21 @@ def search_dns( dns_query: Domain name to filter by (substring match). dns_rcode: Response code to filter by, e.g. "NXDOMAIN". dns_qtype: Query type to filter by, e.g. "A", "MX", "TXT". + time_from: Absolute start timestamp (ISO 8601). Overrides time_range when both are set. + time_to: Absolute end timestamp (ISO 8601). """ try: params = _base_params( - time_range, sensor, limit, public_only, src_ip, direction, no_filters, dest_ip + time_range, + sensor, + limit, + public_only, + src_ip, + direction, + no_filters, + dest_ip, + time_from, + time_to, ) if dns_query: params["dns_query"] = dns_query @@ -169,17 +215,20 @@ def search_dns( @mcp.tool() def search_http( time_range: str = "now-24h", - sensor: str = "all", + sensor: str | list[str] = "all", limit: int = 500, public_only: bool = False, - src_ip: Optional[str] = None, - dest_ip: Optional[str] = None, + src_ip: str | list[str] | None = None, + dest_ip: str | list[str] | None = None, direction: Optional[str] = None, no_filters: bool = False, http_method: Optional[str] = None, http_host: Optional[str] = None, http_uri: Optional[str] = None, status_code: Optional[int] = None, + dest_port: Optional[int] = None, + time_from: Optional[str] = None, + time_to: Optional[str] = None, ) -> str: """Search Zeek HTTP logs from Malcolm/OpenSearch. @@ -188,10 +237,22 @@ def search_http( http_host: Virtual host header to filter by. http_uri: URI path substring to filter by. status_code: HTTP response status code to filter by. + dest_port: Destination port to filter by (default 80/8080, but not enforced). + time_from: Absolute start timestamp (ISO 8601). Overrides time_range when both are set. + time_to: Absolute end timestamp (ISO 8601). """ try: params = _base_params( - time_range, sensor, limit, public_only, src_ip, direction, no_filters, dest_ip + time_range, + sensor, + limit, + public_only, + src_ip, + direction, + no_filters, + dest_ip, + time_from, + time_to, ) if http_method: params["http_method"] = http_method @@ -201,6 +262,8 @@ def search_http( params["http_uri"] = http_uri if status_code is not None: params["status_code"] = status_code + if dest_port is not None: + params["dest_port"] = dest_port records = run_query(MODULES["http"], params) return _ok({"count": len(records), "records": _serialise_records(records)}) except Exception as exc: @@ -210,30 +273,47 @@ def search_http( @mcp.tool() def search_ssl( time_range: str = "now-24h", - sensor: str = "all", + sensor: str | list[str] = "all", limit: int = 500, public_only: bool = False, - src_ip: Optional[str] = None, - dest_ip: Optional[str] = None, + src_ip: str | list[str] | None = None, + dest_ip: str | list[str] | None = None, direction: Optional[str] = None, no_filters: bool = False, ssl_sni: Optional[str] = None, ssl_invalid_only: bool = False, + dest_port: Optional[int] = None, + time_from: Optional[str] = None, + time_to: Optional[str] = None, ) -> str: """Search Zeek SSL/TLS logs from Malcolm/OpenSearch. Args: ssl_sni: Server Name Indication hostname to filter by. ssl_invalid_only: If True, return only connections with invalid/self-signed certs. + dest_port: Destination port to filter by (commonly 443, 8443). + time_from: Absolute start timestamp (ISO 8601). Overrides time_range when both are set. + time_to: Absolute end timestamp (ISO 8601). """ try: params = _base_params( - time_range, sensor, limit, public_only, src_ip, direction, no_filters, dest_ip + time_range, + sensor, + limit, + public_only, + src_ip, + direction, + no_filters, + dest_ip, + time_from, + time_to, ) if ssl_sni: params["ssl_sni"] = ssl_sni if ssl_invalid_only: params["ssl_invalid_only"] = True + if dest_port is not None: + params["dest_port"] = dest_port records = run_query(MODULES["ssl"], params) return _ok({"count": len(records), "records": _serialise_records(records)}) except Exception as exc: @@ -243,16 +323,18 @@ def search_ssl( @mcp.tool() def search_smtp( time_range: str = "now-24h", - sensor: str = "all", + sensor: str | list[str] = "all", limit: int = 500, public_only: bool = False, - src_ip: Optional[str] = None, - dest_ip: Optional[str] = None, + src_ip: str | list[str] | None = None, + dest_ip: str | list[str] | None = None, direction: Optional[str] = None, no_filters: bool = False, smtp_mail_from: Optional[str] = None, smtp_rcpt_to: Optional[str] = None, smtp_subject: Optional[str] = None, + time_from: Optional[str] = None, + time_to: Optional[str] = None, ) -> str: """Search Zeek SMTP logs from Malcolm/OpenSearch. @@ -260,10 +342,21 @@ def search_smtp( smtp_mail_from: Sender address to filter by. smtp_rcpt_to: Recipient address to filter by. smtp_subject: Subject line substring to filter by. + time_from: Absolute start timestamp (ISO 8601). Overrides time_range when both are set. + time_to: Absolute end timestamp (ISO 8601). """ try: params = _base_params( - time_range, sensor, limit, public_only, src_ip, direction, no_filters, dest_ip + time_range, + sensor, + limit, + public_only, + src_ip, + direction, + no_filters, + dest_ip, + time_from, + time_to, ) if smtp_mail_from: params["smtp_mail_from"] = smtp_mail_from @@ -280,30 +373,47 @@ def search_smtp( @mcp.tool() def search_rdp( time_range: str = "now-24h", - sensor: str = "all", + sensor: str | list[str] = "all", limit: int = 500, public_only: bool = False, - src_ip: Optional[str] = None, - dest_ip: Optional[str] = None, + src_ip: str | list[str] | None = None, + dest_ip: str | list[str] | None = None, direction: Optional[str] = None, no_filters: bool = False, rdp_result: Optional[str] = None, rdp_cookie: Optional[str] = None, + dest_port: Optional[int] = None, + time_from: Optional[str] = None, + time_to: Optional[str] = None, ) -> str: """Search Zeek RDP logs from Malcolm/OpenSearch. Args: rdp_result: RDP result string to filter by, e.g. "encrypted". rdp_cookie: RDP cookie/username string to filter by. + dest_port: Destination port to filter by (commonly 3389). + time_from: Absolute start timestamp (ISO 8601). Overrides time_range when both are set. + time_to: Absolute end timestamp (ISO 8601). """ try: params = _base_params( - time_range, sensor, limit, public_only, src_ip, direction, no_filters, dest_ip + time_range, + sensor, + limit, + public_only, + src_ip, + direction, + no_filters, + dest_ip, + time_from, + time_to, ) if rdp_result: params["rdp_result"] = rdp_result if rdp_cookie: params["rdp_cookie"] = rdp_cookie + if dest_port is not None: + params["dest_port"] = dest_port records = run_query(MODULES["rdp"], params) return _ok({"count": len(records), "records": _serialise_records(records)}) except Exception as exc: @@ -313,30 +423,47 @@ def search_rdp( @mcp.tool() def search_smb( time_range: str = "now-24h", - sensor: str = "all", + sensor: str | list[str] = "all", limit: int = 500, public_only: bool = False, - src_ip: Optional[str] = None, - dest_ip: Optional[str] = None, + src_ip: str | list[str] | None = None, + dest_ip: str | list[str] | None = None, direction: Optional[str] = None, no_filters: bool = False, smb_share: Optional[str] = None, smb_action: Optional[str] = None, + dest_port: Optional[int] = None, + time_from: Optional[str] = None, + time_to: Optional[str] = None, ) -> str: """Search Zeek SMB logs from Malcolm/OpenSearch. Args: smb_share: SMB share name to filter by. smb_action: SMB action verb to filter by, e.g. "SMB::FILE_OPEN". + dest_port: Destination port to filter by (commonly 445). + time_from: Absolute start timestamp (ISO 8601). Overrides time_range when both are set. + time_to: Absolute end timestamp (ISO 8601). """ try: params = _base_params( - time_range, sensor, limit, public_only, src_ip, direction, no_filters, dest_ip + time_range, + sensor, + limit, + public_only, + src_ip, + direction, + no_filters, + dest_ip, + time_from, + time_to, ) if smb_share: params["smb_share"] = smb_share if smb_action: params["smb_action"] = smb_action + if dest_port is not None: + params["dest_port"] = dest_port records = run_query(MODULES["smb"], params) return _ok({"count": len(records), "records": _serialise_records(records)}) except Exception as exc: @@ -346,30 +473,47 @@ def search_smb( @mcp.tool() def search_ssh( time_range: str = "now-24h", - sensor: str = "all", + sensor: str | list[str] = "all", limit: int = 500, public_only: bool = False, - src_ip: Optional[str] = None, - dest_ip: Optional[str] = None, + src_ip: str | list[str] | None = None, + dest_ip: str | list[str] | None = None, direction: Optional[str] = None, no_filters: bool = False, ssh_failed_only: bool = False, ssh_auth_result: Optional[str] = None, + dest_port: Optional[int] = None, + time_from: Optional[str] = None, + time_to: Optional[str] = None, ) -> str: """Search Zeek SSH logs from Malcolm/OpenSearch. Args: ssh_failed_only: If True, return only failed authentication attempts. ssh_auth_result: Auth result string to filter by, e.g. "failure", "success". + dest_port: Destination port to filter by (commonly 22). + time_from: Absolute start timestamp (ISO 8601). Overrides time_range when both are set. + time_to: Absolute end timestamp (ISO 8601). """ try: params = _base_params( - time_range, sensor, limit, public_only, src_ip, direction, no_filters, dest_ip + time_range, + sensor, + limit, + public_only, + src_ip, + direction, + no_filters, + dest_ip, + time_from, + time_to, ) if ssh_failed_only: params["ssh_failed_only"] = True if ssh_auth_result is not None: params["ssh_auth_result"] = ssh_auth_result + if dest_port is not None: + params["dest_port"] = dest_port records = run_query(MODULES["ssh"], params) return _ok({"count": len(records), "records": _serialise_records(records)}) except Exception as exc: @@ -379,14 +523,16 @@ def search_ssh( @mcp.tool() def search_notice( time_range: str = "now-24h", - sensor: str = "all", + sensor: str | list[str] = "all", limit: int = 500, public_only: bool = False, - src_ip: Optional[str] = None, - dest_ip: Optional[str] = None, + src_ip: str | list[str] | None = None, + dest_ip: str | list[str] | None = None, direction: Optional[str] = None, no_filters: bool = False, notice_note: Optional[str] = None, + time_from: Optional[str] = None, + time_to: Optional[str] = None, ) -> str: """Search Zeek Notice logs from Malcolm/OpenSearch. @@ -395,10 +541,21 @@ def search_notice( Args: notice_note: Notice type to filter by, e.g. "Scan::Port_Scan". + time_from: Absolute start timestamp (ISO 8601). Overrides time_range when both are set. + time_to: Absolute end timestamp (ISO 8601). """ try: params = _base_params( - time_range, sensor, limit, public_only, src_ip, direction, no_filters, dest_ip + time_range, + sensor, + limit, + public_only, + src_ip, + direction, + no_filters, + dest_ip, + time_from, + time_to, ) if notice_note: params["notice_note"] = notice_note @@ -411,14 +568,16 @@ def search_notice( @mcp.tool() def search_weird( time_range: str = "now-24h", - sensor: str = "all", + sensor: str | list[str] = "all", limit: int = 500, public_only: bool = False, - src_ip: Optional[str] = None, - dest_ip: Optional[str] = None, + src_ip: str | list[str] | None = None, + dest_ip: str | list[str] | None = None, direction: Optional[str] = None, no_filters: bool = False, weird_name: Optional[str] = None, + time_from: Optional[str] = None, + time_to: Optional[str] = None, ) -> str: """Search Zeek Weird logs from Malcolm/OpenSearch. @@ -427,10 +586,21 @@ def search_weird( Args: weird_name: Weird event name to filter by, e.g. "bad_HTTP_reply". + time_from: Absolute start timestamp (ISO 8601). Overrides time_range when both are set. + time_to: Absolute end timestamp (ISO 8601). """ try: params = _base_params( - time_range, sensor, limit, public_only, src_ip, direction, no_filters, dest_ip + time_range, + sensor, + limit, + public_only, + src_ip, + direction, + no_filters, + dest_ip, + time_from, + time_to, ) if weird_name: params["weird_name"] = weird_name @@ -443,11 +613,11 @@ def search_weird( @mcp.tool() def search_suricata_alert( time_range: str = "now-24h", - sensor: str = "all", + sensor: str | list[str] = "all", limit: int = 500, public_only: bool = False, - src_ip: Optional[str] = None, - dest_ip: Optional[str] = None, + src_ip: str | list[str] | None = None, + dest_ip: str | list[str] | None = None, direction: Optional[str] = None, no_filters: bool = False, rule_name: Optional[str] = None, @@ -456,6 +626,8 @@ def search_suricata_alert( sid: Optional[int] = None, exclude_stream: bool = False, tag: Optional[str] = None, + time_from: Optional[str] = None, + time_to: Optional[str] = None, ) -> str: """Search Suricata IDS alert records from Malcolm/OpenSearch. @@ -470,10 +642,21 @@ def search_suricata_alert( sid: Suricata rule ID (SID) to filter by. exclude_stream: Exclude noisy SURICATA STREAM/QUIC protocol anomaly rules. tag: Filter by tag, e.g. "CISA_KEV", "Exploit", "RAT". + time_from: Absolute start timestamp (ISO 8601). Overrides time_range when both are set. + time_to: Absolute end timestamp (ISO 8601). """ try: params = _base_params( - time_range, sensor, limit, public_only, src_ip, direction, no_filters, dest_ip + time_range, + sensor, + limit, + public_only, + src_ip, + direction, + no_filters, + dest_ip, + time_from, + time_to, ) if rule_name: params["rule_name"] = rule_name @@ -501,15 +684,17 @@ def search_suricata_alert( @mcp.tool() def search_radius( time_range: str = "now-24h", - sensor: str = "all", + sensor: str | list[str] = "all", limit: int = 500, public_only: bool = False, - src_ip: Optional[str] = None, - dest_ip: Optional[str] = None, + src_ip: str | list[str] | None = None, + dest_ip: str | list[str] | None = None, no_filters: bool = False, username: Optional[str] = None, mac: Optional[str] = None, failed_only: bool = False, + time_from: Optional[str] = None, + time_to: Optional[str] = None, ) -> str: """Search Zeek RADIUS authentication logs — VPN/802.1X auth records. @@ -517,10 +702,21 @@ def search_radius( username: Filter by username (substring match). mac: Filter by MAC address (exact match). failed_only: Show only failed authentication attempts. + time_from: Absolute start timestamp (ISO 8601). Overrides time_range when both are set. + time_to: Absolute end timestamp (ISO 8601). """ try: params = _base_params( - time_range, sensor, limit, public_only, src_ip, None, no_filters, dest_ip + time_range, + sensor, + limit, + public_only, + src_ip, + None, + no_filters, + dest_ip, + time_from, + time_to, ) if username: params["username"] = username @@ -537,15 +733,17 @@ def search_radius( @mcp.tool() def search_sip( time_range: str = "now-24h", - sensor: str = "all", + sensor: str | list[str] = "all", limit: int = 500, public_only: bool = False, - src_ip: Optional[str] = None, - dest_ip: Optional[str] = None, + src_ip: str | list[str] | None = None, + dest_ip: str | list[str] | None = None, no_filters: bool = False, method: Optional[str] = None, status_code: Optional[str] = None, user_agent: Optional[str] = None, + time_from: Optional[str] = None, + time_to: Optional[str] = None, ) -> str: """Search Zeek SIP/VoIP session logs. @@ -553,10 +751,21 @@ def search_sip( method: Filter by SIP method (INVITE, REGISTER, OPTIONS, etc.). status_code: Filter by SIP status code (exact match). user_agent: Filter by User-Agent (substring match). + time_from: Absolute start timestamp (ISO 8601). Overrides time_range when both are set. + time_to: Absolute end timestamp (ISO 8601). """ try: params = _base_params( - time_range, sensor, limit, public_only, src_ip, None, no_filters, dest_ip + time_range, + sensor, + limit, + public_only, + src_ip, + None, + no_filters, + dest_ip, + time_from, + time_to, ) if method: params["method"] = method @@ -573,22 +782,35 @@ def search_sip( @mcp.tool() def search_tunnel( time_range: str = "now-24h", - sensor: str = "all", + sensor: str | list[str] = "all", limit: int = 500, public_only: bool = False, - src_ip: Optional[str] = None, - dest_ip: Optional[str] = None, + src_ip: str | list[str] | None = None, + dest_ip: str | list[str] | None = None, no_filters: bool = False, tunnel_type: Optional[str] = None, + time_from: Optional[str] = None, + time_to: Optional[str] = None, ) -> str: """Search Zeek tunnel logs — protocol encapsulation / covert channel detection. Args: tunnel_type: Filter by tunnel type (Tunnel::IP, Tunnel::GRE, etc.). + time_from: Absolute start timestamp (ISO 8601). Overrides time_range when both are set. + time_to: Absolute end timestamp (ISO 8601). """ try: params = _base_params( - time_range, sensor, limit, public_only, src_ip, None, no_filters, dest_ip + time_range, + sensor, + limit, + public_only, + src_ip, + None, + no_filters, + dest_ip, + time_from, + time_to, ) if tunnel_type: params["tunnel_type"] = tunnel_type @@ -601,24 +823,37 @@ def search_tunnel( @mcp.tool() def search_ntp( time_range: str = "now-24h", - sensor: str = "all", + sensor: str | list[str] = "all", limit: int = 500, public_only: bool = False, - src_ip: Optional[str] = None, - dest_ip: Optional[str] = None, + src_ip: str | list[str] | None = None, + dest_ip: str | list[str] | None = None, no_filters: bool = False, mode: Optional[int] = None, version: Optional[int] = None, + time_from: Optional[str] = None, + time_to: Optional[str] = None, ) -> str: """Search Zeek NTP logs — time synchronisation and amplification detection. Args: mode: Filter by NTP mode (3=client, 4=server, 6=control, 7=private). version: Filter by NTP version (exact match, integer). + time_from: Absolute start timestamp (ISO 8601). Overrides time_range when both are set. + time_to: Absolute end timestamp (ISO 8601). """ try: params = _base_params( - time_range, sensor, limit, public_only, src_ip, None, no_filters, dest_ip + time_range, + sensor, + limit, + public_only, + src_ip, + None, + no_filters, + dest_ip, + time_from, + time_to, ) if mode is not None: params["mode"] = mode @@ -633,24 +868,37 @@ def search_ntp( @mcp.tool() def search_modbus( time_range: str = "now-24h", - sensor: str = "all", + sensor: str | list[str] = "all", limit: int = 500, public_only: bool = False, - src_ip: Optional[str] = None, - dest_ip: Optional[str] = None, + src_ip: str | list[str] | None = None, + dest_ip: str | list[str] | None = None, no_filters: bool = False, function: Optional[str] = None, exceptions_only: bool = False, + time_from: Optional[str] = None, + time_to: Optional[str] = None, ) -> str: """Search Zeek Modbus/TCP logs — OT/SCADA protocol for PLCs and RTUs. Args: function: Filter by Modbus function (e.g. "Read Coils", "Write Single Register"). exceptions_only: Show only records with exception codes. + time_from: Absolute start timestamp (ISO 8601). Overrides time_range when both are set. + time_to: Absolute end timestamp (ISO 8601). """ try: params = _base_params( - time_range, sensor, limit, public_only, src_ip, None, no_filters, dest_ip + time_range, + sensor, + limit, + public_only, + src_ip, + None, + no_filters, + dest_ip, + time_from, + time_to, ) if function: params["function"] = function @@ -665,22 +913,35 @@ def search_modbus( @mcp.tool() def search_dnp3( time_range: str = "now-24h", - sensor: str = "all", + sensor: str | list[str] = "all", limit: int = 500, public_only: bool = False, - src_ip: Optional[str] = None, - dest_ip: Optional[str] = None, + src_ip: str | list[str] | None = None, + dest_ip: str | list[str] | None = None, no_filters: bool = False, function: Optional[str] = None, + time_from: Optional[str] = None, + time_to: Optional[str] = None, ) -> str: """Search Zeek DNP3 logs — SCADA protocol for utilities (electric, water, gas). Args: function: Filter by DNP3 function request (substring match). + time_from: Absolute start timestamp (ISO 8601). Overrides time_range when both are set. + time_to: Absolute end timestamp (ISO 8601). """ try: params = _base_params( - time_range, sensor, limit, public_only, src_ip, None, no_filters, dest_ip + time_range, + sensor, + limit, + public_only, + src_ip, + None, + no_filters, + dest_ip, + time_from, + time_to, ) if function: params["function"] = function diff --git a/src/querier/builder.py b/src/querier/builder.py index b24d287..94c920b 100644 --- a/src/querier/builder.py +++ b/src/querier/builder.py @@ -127,13 +127,16 @@ def build_base_query( sensors: list | None, datasets: list, public_only: bool = False, - src_ip_filter: str | None = None, - dest_ip_filter: str | None = None, + src_ip_filter: str | list[str] | None = None, + dest_ip_filter: str | list[str] | None = None, any_ip_filter: str | None = None, direction: str | None = None, time_from: str | None = None, time_to: str | None = None, sort: bool = True, + src_port_filter: int | list[int] | None = None, + dest_port_filter: int | list[int] | None = None, + proto_filter: str | list[str] | None = None, ) -> tuple: """Build the OpenSearch query body and request params. @@ -154,10 +157,34 @@ def build_base_query( must_clauses.append({"terms": {"host.name": sensors}}) if src_ip_filter: - must_clauses.append({"term": {"source.ip": src_ip_filter}}) + if isinstance(src_ip_filter, list): + must_clauses.append({"terms": {"source.ip": src_ip_filter}}) + else: + must_clauses.append({"term": {"source.ip": src_ip_filter}}) if dest_ip_filter: - must_clauses.append({"term": {"destination.ip": dest_ip_filter}}) + if isinstance(dest_ip_filter, list): + must_clauses.append({"terms": {"destination.ip": dest_ip_filter}}) + else: + must_clauses.append({"term": {"destination.ip": dest_ip_filter}}) + + if src_port_filter is not None: + if isinstance(src_port_filter, list): + must_clauses.append({"terms": {"source.port": src_port_filter}}) + else: + must_clauses.append({"term": {"source.port": src_port_filter}}) + + if dest_port_filter is not None: + if isinstance(dest_port_filter, list): + must_clauses.append({"terms": {"destination.port": dest_port_filter}}) + else: + must_clauses.append({"term": {"destination.port": dest_port_filter}}) + + if proto_filter: + if isinstance(proto_filter, list): + must_clauses.append({"terms": {"network.transport": proto_filter}}) + else: + must_clauses.append({"term": {"network.transport": proto_filter}}) if any_ip_filter: must_clauses.append( diff --git a/src/querier/runner.py b/src/querier/runner.py index d081ea7..63b5b8f 100644 --- a/src/querier/runner.py +++ b/src/querier/runner.py @@ -100,8 +100,11 @@ def run_query(module, search_params: dict) -> list: sensors: list | None = None sensor_val = search_params.get("sensor", "all") - if sensor_val and str(sensor_val).lower() != "all": - sensors = [s.strip() for s in str(sensor_val).split(",")] + if sensor_val: + if isinstance(sensor_val, list): + sensors = [s.strip() for s in sensor_val] + elif str(sensor_val).lower() != "all": + sensors = [s.strip() for s in str(sensor_val).split(",")] extra_must, post_filters = module.build_extra_must(search_params) @@ -113,6 +116,13 @@ def run_query(module, search_params: dict) -> list: dest_ip_for_query = search_params.get("dest_ip") if has_dest else None any_ip_for_query = search_params.get("any_ip") if (has_src or has_dest) else None + has_src_port = "source.port" in module.SOURCE_FIELDS + has_dest_port = "destination.port" in module.SOURCE_FIELDS + has_proto = "network.transport" in module.SOURCE_FIELDS + src_port_for_query = search_params.get("src_port") if has_src_port else None + dest_port_for_query = search_params.get("dest_port") if has_dest_port else None + proto_for_query = search_params.get("proto") if has_proto else None + # Over-fetch when post-filters are active so truncation still yields enough rows. requested_limit = search_params.get("limit", 500) query_limit = ( @@ -134,6 +144,9 @@ def run_query(module, search_params: dict) -> list: direction=search_params.get("direction"), time_from=search_params.get("time_from"), time_to=search_params.get("time_to"), + src_port_filter=src_port_for_query, + dest_port_filter=dest_port_for_query, + proto_filter=proto_for_query, ) if search_params.get("profile"): @@ -202,8 +215,11 @@ async def run_query_async(module, search_params: dict) -> list: sensors: list | None = None sensor_val = search_params.get("sensor", "all") - if sensor_val and str(sensor_val).lower() != "all": - sensors = [s.strip() for s in str(sensor_val).split(",")] + if sensor_val: + if isinstance(sensor_val, list): + sensors = [s.strip() for s in sensor_val] + elif str(sensor_val).lower() != "all": + sensors = [s.strip() for s in str(sensor_val).split(",")] extra_must, post_filters = module.build_extra_must(search_params) @@ -213,6 +229,13 @@ async def run_query_async(module, search_params: dict) -> list: dest_ip_for_query = search_params.get("dest_ip") if has_dest else None any_ip_for_query = search_params.get("any_ip") if (has_src or has_dest) else None + has_src_port = "source.port" in module.SOURCE_FIELDS + has_dest_port = "destination.port" in module.SOURCE_FIELDS + has_proto = "network.transport" in module.SOURCE_FIELDS + src_port_for_query = search_params.get("src_port") if has_src_port else None + dest_port_for_query = search_params.get("dest_port") if has_dest_port else None + proto_for_query = search_params.get("proto") if has_proto else None + requested_limit = search_params.get("limit", 500) query_limit = ( min(requested_limit * _OVERFETCH_MULTIPLIER, 5000) if post_filters else requested_limit @@ -233,6 +256,9 @@ async def run_query_async(module, search_params: dict) -> list: direction=search_params.get("direction"), time_from=search_params.get("time_from"), time_to=search_params.get("time_to"), + src_port_filter=src_port_for_query, + dest_port_filter=dest_port_for_query, + proto_filter=proto_for_query, ) raw = await query_opensearch_async(body, params) diff --git a/tests/test_zeek_base.py b/tests/test_zeek_base.py index b6e50e1..318ba8d 100644 --- a/tests/test_zeek_base.py +++ b/tests/test_zeek_base.py @@ -343,3 +343,158 @@ def test_dedup_collects_sensors() -> None: def test_dedup_empty_list() -> None: assert deduplicate_zeek([], _key) == [] + + +# --------------------------------------------------------------------------- +# §4.3 Port / proto filters in build_base_query +# --------------------------------------------------------------------------- + + +def test_build_base_query_dest_port_filter() -> None: + body, _ = build_base_query( + must_not=[], + extra_must=[], + source_fields=["destination.port"], + limit=10, + time_range="now-1h", + sensors=None, + datasets=["conn"], + dest_port_filter=443, + ) + must = body["query"]["bool"]["filter"] + clause = next((c for c in must if "term" in c and "destination.port" in c["term"]), None) + assert clause is not None + assert clause["term"]["destination.port"] == 443 + + +def test_build_base_query_src_port_filter() -> None: + body, _ = build_base_query( + must_not=[], + extra_must=[], + source_fields=["source.port"], + limit=10, + time_range="now-1h", + sensors=None, + datasets=["conn"], + src_port_filter=12345, + ) + must = body["query"]["bool"]["filter"] + clause = next((c for c in must if "term" in c and "source.port" in c["term"]), None) + assert clause is not None + assert clause["term"]["source.port"] == 12345 + + +def test_build_base_query_proto_filter() -> None: + body, _ = build_base_query( + must_not=[], + extra_must=[], + source_fields=[], + limit=10, + time_range="now-1h", + sensors=None, + datasets=["conn"], + proto_filter="udp", + ) + must = body["query"]["bool"]["filter"] + clause = next((c for c in must if "term" in c and "network.transport" in c["term"]), None) + assert clause is not None + assert clause["term"]["network.transport"] == "udp" + + +def test_build_base_query_dest_port_list() -> None: + body, _ = build_base_query( + must_not=[], + extra_must=[], + source_fields=["destination.port"], + limit=10, + time_range="now-1h", + sensors=None, + datasets=["conn"], + dest_port_filter=[80, 443, 8080], + ) + must = body["query"]["bool"]["filter"] + clause = next((c for c in must if "terms" in c and "destination.port" in c["terms"]), None) + assert clause is not None + assert clause["terms"]["destination.port"] == [80, 443, 8080] + + +# --------------------------------------------------------------------------- +# §4.4 Multi-value src_ip / dest_ip in build_base_query +# --------------------------------------------------------------------------- + + +def test_build_base_query_src_ip_list() -> None: + body, _ = build_base_query( + must_not=[], + extra_must=[], + source_fields=["source.ip"], + limit=10, + time_range="now-1h", + sensors=None, + datasets=["conn"], + src_ip_filter=["1.1.1.1", "2.2.2.2"], + ) + must = body["query"]["bool"]["filter"] + clause = next((c for c in must if "terms" in c and "source.ip" in c["terms"]), None) + assert clause is not None + assert clause["terms"]["source.ip"] == ["1.1.1.1", "2.2.2.2"] + + +def test_build_base_query_dest_ip_list() -> None: + body, _ = build_base_query( + must_not=[], + extra_must=[], + source_fields=["destination.ip"], + limit=10, + time_range="now-1h", + sensors=None, + datasets=["conn"], + dest_ip_filter=["8.8.8.8", "1.1.1.1"], + ) + must = body["query"]["bool"]["filter"] + clause = next((c for c in must if "terms" in c and "destination.ip" in c["terms"]), None) + assert clause is not None + assert clause["terms"]["destination.ip"] == ["8.8.8.8", "1.1.1.1"] + + +def test_build_base_query_proto_list() -> None: + body, _ = build_base_query( + must_not=[], + extra_must=[], + source_fields=[], + limit=10, + time_range="now-1h", + sensors=None, + datasets=["conn"], + proto_filter=["tcp", "udp"], + ) + must = body["query"]["bool"]["filter"] + clause = next((c for c in must if "terms" in c and "network.transport" in c["terms"]), None) + assert clause is not None + assert clause["terms"]["network.transport"] == ["tcp", "udp"] + + +# --------------------------------------------------------------------------- +# §4.6 Absolute timestamp override in build_base_query +# --------------------------------------------------------------------------- + + +def test_build_base_query_absolute_timestamps() -> None: + time_from = "2026-04-19T00:00:00Z" + time_to = "2026-04-20T00:00:00Z" + body, _ = build_base_query( + must_not=[], + extra_must=[], + source_fields=[], + limit=10, + time_range="now-24h", + sensors=None, + datasets=["conn"], + time_from=time_from, + time_to=time_to, + ) + must = body["query"]["bool"]["filter"] + ts_clause = next((c for c in must if "range" in c and "@timestamp" in c["range"]), None) + assert ts_clause is not None + assert ts_clause["range"]["@timestamp"]["gte"] == time_from + assert ts_clause["range"]["@timestamp"]["lte"] == time_to From 7dc3a543325ffee412875b776a632e47315d3431 Mon Sep 17 00:00:00 2001 From: liamadale Date: Tue, 12 May 2026 10:26:36 -0700 Subject: [PATCH 094/109] feat(histogram): add date-histogram aggregation for event-volume analysis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces query_histogram() in src/querier/histogram.py as a shared helper that runs a date_histogram ES aggregation over any Zeek log type. It reuses build_base_query() and load_with_remap() so all existing filters, sensor lists, src/dest IP filtering, and absolute timestamp support carry over automatically. - src/querier/histogram.py: core helper, returns [{key, key_as_string, doc_count}] buckets; empty list on connection failure - src/querier/histogram_cli.py: pisces-histogram CLI with Unicode block bar chart (_render) and sparse time-axis labels (_time_axis); supports --interval, --time-range, --time-from/to, --src-ip, --dest-ip, --sensor, --no-filters - mcp/opensearch/server.py: histogram() MCP tool (§4.9) wrapping the core helper; returns JSON with bucket_count, total_events, buckets - pyproject.toml: register pisces-histogram entrypoint - tests/test_histogram.py: 11 tests covering _render edge cases, _time_axis, and query construction (datasets, None response, list src_ip → terms clause, absolute timestamps) --- mcp/opensearch/server.py | 60 ++++++++++++++ pyproject.toml | 1 + src/querier/histogram.py | 84 +++++++++++++++++++ src/querier/histogram_cli.py | 153 +++++++++++++++++++++++++++++++++++ tests/test_histogram.py | 140 ++++++++++++++++++++++++++++++++ 5 files changed, 438 insertions(+) create mode 100644 src/querier/histogram.py create mode 100644 src/querier/histogram_cli.py create mode 100644 tests/test_histogram.py diff --git a/mcp/opensearch/server.py b/mcp/opensearch/server.py index 4814d1d..5adf14e 100644 --- a/mcp/opensearch/server.py +++ b/mcp/opensearch/server.py @@ -36,6 +36,7 @@ from mcp.server.fastmcp import FastMCP from src.enricher.threat_intel import enrich_ip +from src.querier.histogram import query_histogram from src.querier.fp_manager import ( append_clauses_to_file, ensure_subcategory, @@ -1670,6 +1671,65 @@ def investigate( return _err(str(exc)) +# --------------------------------------------------------------------------- +# §4.9 Date-histogram primitive +# --------------------------------------------------------------------------- + + +@mcp.tool() +def histogram( + log_type: str, + interval: str = "1h", + time_range: str = "now-24h", + src_ip: str | list[str] | None = None, + dest_ip: str | list[str] | None = None, + sensor: str | list[str] = "all", + no_filters: bool = False, + time_from: Optional[str] = None, + time_to: Optional[str] = None, +) -> str: + """Return a date histogram of event volume over time for a Zeek log type. + + Buckets are sorted chronologically. Use this to spot traffic spikes, quiet + periods, or to establish a baseline before diving into per-record searches. + + Args: + log_type: Zeek log type (e.g. "conn", "notice", "dns") or "all" for all datasets. + interval: Bucket width in ES fixed_interval notation, e.g. "15m", "1h", "6h", "1d". + time_range: ES date-math range (default: now-24h). Ignored when time_from/time_to are set. + src_ip: Filter to a source IP or list of source IPs. + dest_ip: Filter to a destination IP or list of destination IPs. + sensor: Sensor hostname or list of hostnames; "all" returns all sensors. + no_filters: Skip false-positive YAML filters (useful for debugging). + time_from: Absolute start timestamp (ISO 8601), e.g. "2026-04-19T00:00:00Z". + time_to: Absolute end timestamp (ISO 8601). + """ + try: + buckets = query_histogram( + log_type=log_type, + interval=interval, + time_range=time_range, + src_ip=src_ip, + dest_ip=dest_ip, + sensor=sensor, + no_filters=no_filters, + time_from=time_from, + time_to=time_to, + ) + total = sum(b["doc_count"] for b in buckets) + return _ok( + { + "log_type": log_type, + "interval": interval, + "bucket_count": len(buckets), + "total_events": total, + "buckets": buckets, + } + ) + except Exception as exc: + return _err(str(exc)) + + # --------------------------------------------------------------------------- # Entrypoint # --------------------------------------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index ef872c3..9bf9e1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ pisces-mantis-report = "src.mantis.activity_report:main" pisces-threat-model = "src.mantis.mantis_threat_model:main" pisces-fleet-scan = "src.profiler.fleet_scanner:main" pisces-profile = "src.profiler.device_profiler:main" +pisces-histogram = "src.querier.histogram_cli:main" # Web servers (standalone) pisces-all = "run_all:main" pisces-opensearch = "apps.opensearch_web.run:main" diff --git a/src/querier/histogram.py b/src/querier/histogram.py new file mode 100644 index 0000000..ba85504 --- /dev/null +++ b/src/querier/histogram.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Date-histogram aggregation helper shared by the MCP tool and CLI renderer.""" + +from typing import Optional + +from src.querier.builder import build_base_query +from src.querier.client import query_opensearch +from src.querier.runner import FILTERS_DIR, load_with_remap +from src.querier.zeek_modules import MODULES + + +def query_histogram( + log_type: str, + interval: str = "1h", + time_range: str = "now-24h", + src_ip: str | list[str] | None = None, + dest_ip: str | list[str] | None = None, + sensor: str | list[str] = "all", + no_filters: bool = False, + time_from: Optional[str] = None, + time_to: Optional[str] = None, +) -> list[dict]: + """Run a date_histogram aggregation and return the time buckets. + + Returns a list of {key: epoch_ms, key_as_string: ISO-8601, doc_count: int}. + An empty list is returned when OpenSearch is unreachable or returns no data. + """ + if no_filters: + must_not: list = [] + else: + must_not, _, _ = load_with_remap(FILTERS_DIR) + + sensors: list | None = None + if sensor: + if isinstance(sensor, list): + sensors = [s.strip() for s in sensor] + elif str(sensor).lower() != "all": + sensors = [s.strip() for s in str(sensor).split(",")] + + datasets: list[str] + if log_type == "all" or log_type not in MODULES: + datasets = ["all"] + else: + datasets = MODULES[log_type].DATASETS + + body, params = build_base_query( + must_not=must_not, + extra_must=[], + source_fields=[], + limit=0, + time_range=time_range, + sensors=sensors, + datasets=datasets, + src_ip_filter=src_ip, + dest_ip_filter=dest_ip, + time_from=time_from, + time_to=time_to, + sort=False, + ) + body["size"] = 0 + body["track_total_hits"] = False + body["aggs"] = { + "over_time": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": interval, + "min_doc_count": 0, + } + } + } + + raw = query_opensearch(body, params) + if raw is None: + return [] + + buckets = raw.get("aggregations", {}).get("over_time", {}).get("buckets", []) + return [ + { + "key": b["key"], + "key_as_string": b["key_as_string"], + "doc_count": b["doc_count"], + } + for b in buckets + ] diff --git a/src/querier/histogram_cli.py b/src/querier/histogram_cli.py new file mode 100644 index 0000000..504cd52 --- /dev/null +++ b/src/querier/histogram_cli.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""pisces-histogram — terminal bar chart of Zeek event volume over time.""" + +import argparse +import os +import shutil +import sys + +from dotenv import load_dotenv + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + +from src.querier.client import OpenSearchAuthError, OpenSearchConnectionError +from src.querier.histogram import query_histogram +from src.querier.zeek_modules import MODULES +from src.utils.dns import setup_dns + +_BLOCKS = " ▁▂▃▄▅▆▇█" +_VALID_LOG_TYPES = sorted(MODULES.keys()) + ["all"] + + +def _render(buckets: list[dict], width: int) -> str: + """Return a single-row Unicode bar chart string of `width` characters.""" + if not buckets: + return "(no data)" + + counts = [b["doc_count"] for b in buckets] + max_count = max(counts) if any(counts) else 1 + n = len(buckets) + + if n <= width: + bars = [_BLOCKS[round(c / max_count * 8)] for c in counts] + else: + # Compress: average counts into `width` columns + bars = [] + for i in range(width): + lo = int(i * n / width) + hi = int((i + 1) * n / width) + chunk = counts[lo:hi] if lo < hi else counts[lo : lo + 1] + avg = sum(chunk) / len(chunk) + bars.append(_BLOCKS[round(avg / max_count * 8)]) + + return "".join(bars) + + +def _time_axis(buckets: list[dict], bar_width: int) -> str: + """Return a sparse time-label row aligned to the bar chart.""" + if not buckets: + return "" + + def _short(ts: str) -> str: + # "2026-04-19T14:00:00.000Z" → "04-19 14:00" or just "HH:MM" if same day + try: + date_part, time_part = ts.split("T") + hhmm = time_part[:5] + return f"{date_part[5:]} {hhmm}" + except Exception: + return ts[:16] + + n = len(buckets) + label_positions = [0, n // 4, n // 2, 3 * n // 4, n - 1] + row = [" "] * bar_width + + for pos in label_positions: + # Map bucket index → bar column + col = round(pos * bar_width / max(n - 1, 1)) + label = _short(buckets[pos]["key_as_string"]) + start = max(0, min(col, bar_width - len(label))) + for i, ch in enumerate(label): + if start + i < bar_width: + row[start + i] = ch + + return "".join(row) + + +def main() -> None: + """Entry point for the pisces-histogram CLI tool.""" + parser = argparse.ArgumentParser( + description="Show a terminal bar chart of Zeek event volume over time.", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "log_type", + choices=_VALID_LOG_TYPES, + help="Zeek log type to histogram, or 'all' for every dataset.", + ) + parser.add_argument( + "--interval", default="1h", help="Bucket width, e.g. 15m, 1h, 1d (default: 1h)" + ) + parser.add_argument( + "--time-range", default="now-24h", help="ES date-math range (default: now-24h)" + ) + parser.add_argument( + "--time-from", help="Absolute start timestamp (ISO 8601), overrides --time-range" + ) + parser.add_argument("--time-to", help="Absolute end timestamp (ISO 8601)") + parser.add_argument("--src-ip", help="Filter to a source IP (or comma-separated list)") + parser.add_argument("--dest-ip", help="Filter to a destination IP (or comma-separated list)") + parser.add_argument("--sensor", default="all", help="Sensor hostname, or 'all' (default)") + parser.add_argument( + "--no-filters", action="store_true", help="Disable false-positive YAML filters" + ) + args = parser.parse_args() + + load_dotenv() + setup_dns() + + src_ip: str | list[str] | None = args.src_ip + if args.src_ip and "," in args.src_ip: + src_ip = [s.strip() for s in args.src_ip.split(",")] + dest_ip: str | list[str] | None = args.dest_ip + if args.dest_ip and "," in args.dest_ip: + dest_ip = [s.strip() for s in args.dest_ip.split(",")] + + try: + buckets = query_histogram( + log_type=args.log_type, + interval=args.interval, + time_range=args.time_range, + src_ip=src_ip, + dest_ip=dest_ip, + sensor=args.sensor, + no_filters=args.no_filters, + time_from=args.time_from, + time_to=args.time_to, + ) + except (OpenSearchConnectionError, OpenSearchAuthError) as exc: + print(f"Error: {exc}", file=sys.stderr) + sys.exit(1) + + if not buckets: + print("No data returned for the given parameters.") + return + + total = sum(b["doc_count"] for b in buckets) + max_count = max(b["doc_count"] for b in buckets) + term_width = shutil.get_terminal_size((120, 24)).columns + bar_width = min(term_width - 2, len(buckets)) + + time_desc = ( + f"{args.time_from}–{args.time_to}" if args.time_from and args.time_to else args.time_range + ) + header = ( + f"{args.log_type} — {time_desc}, interval={args.interval}," + f" total={total:,}, max={max_count:,}" + ) + print(header) + print(_render(buckets, bar_width)) + print(_time_axis(buckets, bar_width)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_histogram.py b/tests/test_histogram.py new file mode 100644 index 0000000..318f2ab --- /dev/null +++ b/tests/test_histogram.py @@ -0,0 +1,140 @@ +"""Tests for the histogram core helper and CLI renderer (pure/offline functions only).""" + +from __future__ import annotations + +import os +import sys +from unittest.mock import patch + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from src.querier.histogram import query_histogram +from src.querier.histogram_cli import _render, _time_axis + +# --------------------------------------------------------------------------- +# _render — pure function, no I/O +# --------------------------------------------------------------------------- + +_SAMPLE_BUCKETS = [ + {"key": i * 3_600_000, "key_as_string": f"2026-04-19T{i:02d}:00:00.000Z", "doc_count": c} + for i, c in enumerate([0, 10, 50, 100, 80, 20, 5, 0]) +] + + +def test_render_returns_correct_length() -> None: + result = _render(_SAMPLE_BUCKETS, width=len(_SAMPLE_BUCKETS)) + assert len(result) == len(_SAMPLE_BUCKETS) + + +def test_render_empty_buckets() -> None: + assert _render([], width=80) == "(no data)" + + +def test_render_all_zero_counts() -> None: + buckets = [{"key": 0, "key_as_string": "2026-04-19T00:00:00.000Z", "doc_count": 0}] + result = _render(buckets, width=10) + # All-zero: max is treated as 1, round(0/1*8)=0 → first block char (space) + assert result == " " + + +def test_render_single_spike() -> None: + buckets = [ + {"key": 0, "key_as_string": "2026-04-19T00:00:00.000Z", "doc_count": 0}, + {"key": 3_600_000, "key_as_string": "2026-04-19T01:00:00.000Z", "doc_count": 100}, + {"key": 7_200_000, "key_as_string": "2026-04-19T02:00:00.000Z", "doc_count": 0}, + ] + result = _render(buckets, width=3) + # Middle bucket is max → full block + assert result[1] == "█" + + +def test_render_compresses_when_wider_than_width() -> None: + # 20 buckets into width=10 → result length == 10 + many = [{"key": i * 3_600_000, "key_as_string": f"T{i}", "doc_count": i * 5} for i in range(20)] + result = _render(many, width=10) + assert len(result) == 10 + + +# --------------------------------------------------------------------------- +# _time_axis +# --------------------------------------------------------------------------- + + +def test_time_axis_length_matches_width() -> None: + result = _time_axis(_SAMPLE_BUCKETS, bar_width=80) + assert len(result) == 80 + + +def test_time_axis_empty_buckets() -> None: + assert _time_axis([], bar_width=80) == "" + + +# --------------------------------------------------------------------------- +# query_histogram — mock OpenSearch to test query construction +# --------------------------------------------------------------------------- + + +def _fake_raw(buckets: list[dict]) -> dict: + return {"aggregations": {"over_time": {"buckets": buckets}}} + + +def test_query_histogram_passes_log_type_datasets() -> None: + fake_buckets = [{"key": 0, "key_as_string": "2026-04-19T00:00:00.000Z", "doc_count": 42}] + with ( + patch("src.querier.histogram.load_with_remap", return_value=([], 0, [])), + patch( + "src.querier.histogram.query_opensearch", return_value=_fake_raw(fake_buckets) + ) as mock_qos, + ): + result = query_histogram("conn", interval="1h", time_range="now-24h") + + assert result == [{"key": 0, "key_as_string": "2026-04-19T00:00:00.000Z", "doc_count": 42}] + body_arg = mock_qos.call_args[0][0] + assert body_arg["size"] == 0 + assert "over_time" in body_arg["aggs"] + assert body_arg["aggs"]["over_time"]["date_histogram"]["fixed_interval"] == "1h" + + +def test_query_histogram_none_when_opensearch_fails() -> None: + with ( + patch("src.querier.histogram.load_with_remap", return_value=([], 0, [])), + patch("src.querier.histogram.query_opensearch", return_value=None), + ): + result = query_histogram("dns") + + assert result == [] + + +def test_query_histogram_src_ip_list_produces_terms_clause() -> None: + with ( + patch("src.querier.histogram.load_with_remap", return_value=([], 0, [])), + patch("src.querier.histogram.query_opensearch", return_value=_fake_raw([])) as mock_qos, + ): + query_histogram("conn", src_ip=["1.1.1.1", "2.2.2.2"]) + + body_arg = mock_qos.call_args[0][0] + filter_clauses = body_arg["query"]["bool"]["filter"] + terms_clause = next( + (c for c in filter_clauses if "terms" in c and "source.ip" in c["terms"]), None + ) + assert terms_clause is not None + assert terms_clause["terms"]["source.ip"] == ["1.1.1.1", "2.2.2.2"] + + +def test_query_histogram_absolute_timestamps() -> None: + time_from = "2026-04-19T00:00:00Z" + time_to = "2026-04-20T00:00:00Z" + with ( + patch("src.querier.histogram.load_with_remap", return_value=([], 0, [])), + patch("src.querier.histogram.query_opensearch", return_value=_fake_raw([])) as mock_qos, + ): + query_histogram("notice", time_from=time_from, time_to=time_to) + + body_arg = mock_qos.call_args[0][0] + filter_clauses = body_arg["query"]["bool"]["filter"] + ts_clause = next( + (c for c in filter_clauses if "range" in c and "@timestamp" in c["range"]), None + ) + assert ts_clause is not None + assert ts_clause["range"]["@timestamp"]["gte"] == time_from + assert ts_clause["range"]["@timestamp"]["lte"] == time_to From 612d208d848712e3fd36ae229acf503b4302ac81 Mon Sep 17 00:00:00 2001 From: liamadale Date: Tue, 12 May 2026 10:39:28 -0700 Subject: [PATCH 095/109] refactor(mcp/opensearch): consolidate pivot/profile/investigate and aggregate tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the three separate pivot_ip, profile_device, and investigate MCP tools with a single pivot(ip, mode, ...) tool. mode="records" covers the former pivot_ip behaviour, mode="profile" the former profile_device, and mode="incident" the former investigate (requires dest_ip). This reduces surface area for AI assistants and enforces the dest_ip guard at the tool boundary rather than relying on callers to invoke the right tool. Replace aggregate_by_source_ip with a generic aggregate(field, ...) tool that accepts any ES field name and optional log_type/notice_type filters, making it a general-purpose frequency-ranking primitive instead of a notice-specific one. Implements §2.1 and §2.2 from the MCP improvements plan (Tier 4). Test suite updated to match new signatures; two new guard tests added for the incident mode validation paths, and three new tests cover aggregate. --- mcp/opensearch/server.py | 312 +++++++++++++++++++-------------------- tests/test_correlator.py | 109 ++++++++++++-- 2 files changed, 245 insertions(+), 176 deletions(-) diff --git a/mcp/opensearch/server.py b/mcp/opensearch/server.py index 5adf14e..c3b869d 100644 --- a/mcp/opensearch/server.py +++ b/mcp/opensearch/server.py @@ -953,75 +953,132 @@ def search_dnp3( # --------------------------------------------------------------------------- -# Pivot tools +# Pivot tool (§2.1) # --------------------------------------------------------------------------- @mcp.tool() -def pivot_ip( +def pivot( ip: str, - time_range: str = "now-24h", + mode: str = "records", + dest_ip: Optional[str] = None, sensor: str = "all", + time_range: str = "now-24h", limit: int = 500, public_only: bool = False, no_filters: bool = False, ) -> str: - """Run all 10 Zeek protocol queries in parallel for a single IP address. + """IP-centric investigation tool with three operating modes. + + Choose ``mode`` based on the question you are trying to answer: + + * **records** — Cross-protocol raw records for *ip* (as src or dest). + Returns per-protocol counts plus full deduplicated records. Also runs + ``lookup_org`` to surface cloud/CDN/scanner ownership. Use when you + want to see what an IP was actually doing. - Returns per-protocol record counts plus full records for every protocol where - the IP appeared (as either source or destination). Also runs lookup_org to - identify cloud/CDN/scanner ownership. + * **profile** — Aggregated device card for *ip*. For private IPs: role, + OS, software, inbound services, hostnames, fingerprints. For public + IPs: sensor presence, reverse DNS, exposed services, TLS/cert info, + inbound attack signals. Use when you want to understand what a device + is, not just what it did. - This is the primary pivot tool for IP-centric investigations. + * **incident** — Full incident bundle for a src/dest pair. Requires + ``dest_ip``. Runs device profiling, auth history, attack chain, Mantis + ticket search, and threat-intel enrichment in parallel. Use for + alert triage or incident response. + + Args: + ip: IP address to pivot on (actor / attacker for ``mode="incident"``). + mode: One of "records", "profile", or "incident". + dest_ip: Destination IP — required when ``mode="incident"``. + sensor: Sensor hostname. Required for private-IP profiling; "all" for + ``mode="records"``. + time_range: ES date-math range, e.g. "now-24h" or "now-7d". + limit: Max records per protocol (``mode="records"`` only). + public_only: Exclude RFC-1918 addresses (``mode="records"`` only). + no_filters: Bypass FP filter files (``mode="records"`` only). """ try: - base = _base_params(time_range, sensor, limit, public_only, ip, None, no_filters) - - org = lookup_org(ip) - - def _run(log_type: str) -> tuple[str, list]: - try: - params = dict(base) - records = run_query(MODULES[log_type], params) - # Also include records where IP is destination - dest_hits = [] - if ip: - dest_params = _base_params( - time_range, sensor, limit, public_only, None, None, no_filters - ) - dest_records = run_query(MODULES[log_type], dest_params) - dest_hits = [r for r in dest_records if r.get("dest_ip") == ip] - # Merge, deduplicate by identity - seen_keys: set = set() - merged = [] - for r in records + dest_hits: - k = MODULES[log_type].dedup_key(r) - if k not in seen_keys: - seen_keys.add(k) - merged.append(r) - return log_type, merged - except Exception: - return log_type, [] - - results: dict = {} - with ThreadPoolExecutor(max_workers=10) as pool: - futures = {pool.submit(_run, lt): lt for lt in MODULES} - for future in as_completed(futures): - log_type, records = future.result() - results[log_type] = { - "count": len(records), - "records": _serialise_records(records), - } + if mode == "records": + base = _base_params(time_range, sensor, limit, public_only, ip, None, no_filters) + + org = lookup_org(ip) + + def _run(log_type: str) -> tuple[str, list]: + try: + params = dict(base) + records = run_query(MODULES[log_type], params) + dest_hits: list = [] + if ip: + dest_params = _base_params( + time_range, sensor, limit, public_only, None, None, no_filters + ) + dest_records = run_query(MODULES[log_type], dest_params) + dest_hits = [r for r in dest_records if r.get("dest_ip") == ip] + seen_keys: set = set() + merged = [] + for r in records + dest_hits: + k = MODULES[log_type].dedup_key(r) + if k not in seen_keys: + seen_keys.add(k) + merged.append(r) + return log_type, merged + except Exception: + return log_type, [] + + results: dict = {} + with ThreadPoolExecutor(max_workers=10) as pool: + futures = {pool.submit(_run, lt): lt for lt in MODULES} + for future in as_completed(futures): + lt, recs = future.result() + results[lt] = { + "count": len(recs), + "records": _serialise_records(recs), + } - summary = {lt: r["count"] for lt, r in results.items()} - return _ok( - { - "ip": ip, - "org": org, - "summary": summary, - "protocols": results, - } - ) + summary = {lt: r["count"] for lt, r in results.items()} + return _ok({"ip": ip, "org": org, "summary": summary, "protocols": results}) + + elif mode == "profile": + from dataclasses import asdict + + if is_private(ip): + from src.profiler.device_profiler import profile_device as _profile_device + + profile = _profile_device(ip, time_range=time_range, sensor=sensor) + else: + from src.profiler.public_ip_profiler import profile_public_ip + + profile = profile_public_ip(ip, time_range=time_range) + return _ok(asdict(profile)) + + elif mode == "incident": + if not dest_ip: + return _err("dest_ip is required when mode='incident'") + from dataclasses import asdict + + from src.correlator.incident_context import investigate as _investigate + + ctx = _investigate(ip, dest_ip, sensor, time_range) + data = asdict(ctx) + for key in ("src_profile", "dest_profile"): + p = data.get(key) + if p is not None: + data[key] = { + "ip": p["ip"], + "hostname": p.get("hostname"), + "role": p["role"], + "confidence": p["confidence"], + "os_family": p.get("os_family"), + "software": p.get("software", []), + "users": p.get("users", []), + "inbound_services": p.get("inbound_services", []), + } + return _ok(data) + + else: + return _err(f"Unknown mode '{mode}'. Must be 'records', 'profile', or 'incident'.") except Exception as exc: return _err(str(exc)) @@ -1153,36 +1210,49 @@ def raw_opensearch_search( @mcp.tool() -def aggregate_by_source_ip( - notice_type: str, +def aggregate( + field: str, + log_type: Optional[str] = None, + notice_type: Optional[str] = None, time_range: str = "now-24h", sensor: str = "all", limit: int = 25, ) -> str: - """Rank source IPs by how many times they triggered a specific Zeek notice type. + """Rank any ES field by frequency across Zeek logs. - Complements get_notice_summary (which ranks by notice type) — this ranks by - source IP *within* a single notice type. + Generic aggregation primitive. Common uses: + + * Top source IPs for a notice: + ``field="source.ip", log_type="notice", notice_type="Scan::Port_Scan"`` + * Top destination IPs overall: ``field="destination.ip"`` + * Top destination ports: ``field="destination.port", log_type="conn"`` + * Top user agents: ``field="zeek.http.user_agent", log_type="http"`` Args: - notice_type: Exact notice type to filter by, e.g. "Scan::Port_Scan". - limit: Maximum number of source IPs to return. + field: ES field name to aggregate on, e.g. "source.ip", "destination.port". + log_type: Scope to a single Zeek module's datasets, e.g. "conn", "notice", + "http". Omit to aggregate across all datasets. + notice_type: Filter by exact Zeek notice type (only meaningful with + ``log_type="notice"``), e.g. "Scan::Port_Scan". + time_range: ES date-math range, e.g. "now-24h". + sensor: Sensor hostname or "all". + limit: Maximum number of buckets to return. """ try: - must: list = [ - {"range": {"@timestamp": {"gte": time_range, "lte": "now"}}}, - {"terms": {"event.dataset": MODULES["notice"].DATASETS}}, - {"term": {"zeek.notice.note": notice_type}}, - ] + must: list = [{"range": {"@timestamp": {"gte": time_range, "lte": "now"}}}] + if log_type and log_type in MODULES: + must.append({"terms": {"event.dataset": MODULES[log_type].DATASETS}}) + if notice_type: + must.append({"term": {"zeek.notice.note": notice_type}}) if sensor != "all": must.append({"terms": {"host.name": [s.strip() for s in sensor.split(",")]}}) body = { "size": 0, "query": {"bool": {"must": must}}, "aggs": { - "top_sources": { + "buckets": { "terms": { - "field": "source.ip", + "field": field, "size": limit, "order": {"_count": "desc"}, } @@ -1193,9 +1263,17 @@ def aggregate_by_source_ip( raw = query_opensearch(body, params) if raw is None: return _err("OpenSearch query failed — check credentials and OPENSEARCH_URL") - buckets = raw.get("aggregations", {}).get("top_sources", {}).get("buckets", []) - sources = [{"ip": b["key"], "count": b["doc_count"]} for b in buckets] - return _ok({"notice_type": notice_type, "time_range": time_range, "sources": sources}) + buckets = raw.get("aggregations", {}).get("buckets", {}).get("buckets", []) + results = [{"value": b["key"], "count": b["doc_count"]} for b in buckets] + return _ok( + { + "field": field, + "log_type": log_type, + "notice_type": notice_type, + "time_range": time_range, + "results": results, + } + ) except Exception as exc: return _err(str(exc)) @@ -1481,48 +1559,6 @@ def create_fp_filter( return _err(str(exc)) -# --------------------------------------------------------------------------- -# Device profiler -# --------------------------------------------------------------------------- - - -@mcp.tool() -def profile_device( - ip: str, - sensor: str = "all", - time_range: str = "now-7d", -) -> str: - """Profile an IP by aggregating cross-protocol Zeek signals into a device card. - - For private IPs: runs 9 parallel aggregation queries (conn, DNS, SSL, HTTP, - SMB, RDP, SSH) to identify the device's role, OS, installed software, inbound - services, hostnames, fingerprints, and behavioral patterns. - - For public IPs: runs 8 parallel queries to build a network-perspective profile - showing sensor presence, reverse DNS, services exposed, TLS/cert info, and - inbound attack signals. Sensor is optional for public IPs. - - Args: - ip: IP address to profile (private or public). - sensor: Sensor hostname — required for private IPs, optional for public. - time_range: ES date-math range (default: now-7d). - """ - try: - from dataclasses import asdict - - if is_private(ip): - from src.profiler.device_profiler import profile_device as _profile_device - - profile = _profile_device(ip, time_range=time_range, sensor=sensor) - else: - from src.profiler.public_ip_profiler import profile_public_ip - - profile = profile_public_ip(ip, time_range=time_range) - return _ok(asdict(profile)) - except Exception as exc: - return _err(str(exc)) - - # --------------------------------------------------------------------------- # Share URLs # --------------------------------------------------------------------------- @@ -1621,56 +1657,6 @@ def build_share_urls( return _err(str(exc)) -# --------------------------------------------------------------------------- -# Incident correlator -# --------------------------------------------------------------------------- - - -@mcp.tool() -def investigate( - src_ip: str, - dest_ip: str, - sensor: str = "all", - time_range: str = "now-24h", -) -> str: - """Build full incident context for a source/destination IP pair. - - Runs device profiling, auth history, attack chain, Mantis ticket search, - and threat intel enrichment in parallel. Returns a unified context bundle - for incident investigation. - - Args: - src_ip: Source IP address (the actor / attacker). - dest_ip: Destination IP address (the target / victim). - sensor: Sensor hostname — required for private IP profiling. - time_range: ES date-math range (default: now-24h). - """ - try: - from dataclasses import asdict - - from src.correlator.incident_context import investigate as _investigate - - ctx = _investigate(src_ip, dest_ip, sensor, time_range) - data = asdict(ctx) - # Trim raw profile dicts to a compact summary for LLM context - for key in ("src_profile", "dest_profile"): - p = data.get(key) - if p is not None: - data[key] = { - "ip": p["ip"], - "hostname": p.get("hostname"), - "role": p["role"], - "confidence": p["confidence"], - "os_family": p.get("os_family"), - "software": p.get("software", []), - "users": p.get("users", []), - "inbound_services": p.get("inbound_services", []), - } - return _ok(data) - except Exception as exc: - return _err(str(exc)) - - # --------------------------------------------------------------------------- # §4.9 Date-histogram primitive # --------------------------------------------------------------------------- diff --git a/tests/test_correlator.py b/tests/test_correlator.py index e8c36b0..7e9c7d2 100644 --- a/tests/test_correlator.py +++ b/tests/test_correlator.py @@ -502,11 +502,13 @@ def _make_incident_context(**overrides) -> IncidentContext: @_skip_no_mcp -def test_mcp_investigate_returns_ok_json() -> None: - """MCP investigate tool returns JSON with status='ok' and all top-level keys.""" +def test_mcp_pivot_incident_returns_ok_json() -> None: + """pivot(mode='incident') returns JSON with status='ok' and all top-level keys.""" ctx = _make_incident_context() with patch("src.correlator.incident_context.investigate", return_value=ctx): - result_str = mcp_server.investigate(PRIVATE_SRC, PRIVATE_DEST, SENSOR, TIME_RANGE) + result_str = mcp_server.pivot( + PRIVATE_SRC, mode="incident", dest_ip=PRIVATE_DEST, sensor=SENSOR, time_range=TIME_RANGE + ) result = json.loads(result_str) assert result["status"] == "ok" @@ -526,8 +528,8 @@ def test_mcp_investigate_returns_ok_json() -> None: @_skip_no_mcp -def test_mcp_investigate_profile_trimmed() -> None: - """Profile dicts are trimmed to compact summaries — full DeviceProfile keys stripped.""" +def test_mcp_pivot_incident_profile_trimmed() -> None: + """pivot(mode='incident') trims profile dicts to compact summaries.""" from src.profiler.device_profiler import DeviceProfile profile = DeviceProfile( @@ -541,12 +543,13 @@ def test_mcp_investigate_profile_trimmed() -> None: ) ctx = _make_incident_context(src_profile=profile) with patch("src.correlator.incident_context.investigate", return_value=ctx): - result_str = mcp_server.investigate(PRIVATE_SRC, PRIVATE_DEST, SENSOR, TIME_RANGE) + result_str = mcp_server.pivot( + PRIVATE_SRC, mode="incident", dest_ip=PRIVATE_DEST, sensor=SENSOR, time_range=TIME_RANGE + ) result = json.loads(result_str) assert result["status"] == "ok" trimmed = result["data"]["src_profile"] - # Only summary keys present — full DeviceProfile fields like dest_port_distribution stripped expected_keys = { "ip", "hostname", @@ -565,13 +568,15 @@ def test_mcp_investigate_profile_trimmed() -> None: @_skip_no_mcp -def test_mcp_investigate_error_json() -> None: - """MCP investigate tool returns JSON with status='error' when backend raises.""" +def test_mcp_pivot_incident_error_json() -> None: + """pivot(mode='incident') returns JSON with status='error' when backend raises.""" with patch( "src.correlator.incident_context.investigate", side_effect=RuntimeError("OpenSearch unreachable"), ): - result_str = mcp_server.investigate(PRIVATE_SRC, PRIVATE_DEST, SENSOR, TIME_RANGE) + result_str = mcp_server.pivot( + PRIVATE_SRC, mode="incident", dest_ip=PRIVATE_DEST, sensor=SENSOR, time_range=TIME_RANGE + ) result = json.loads(result_str) assert result["status"] == "error" @@ -579,13 +584,91 @@ def test_mcp_investigate_error_json() -> None: @_skip_no_mcp -def test_mcp_investigate_null_profiles_preserved() -> None: - """None profiles (public IPs) are preserved as null in JSON — not trimmed.""" +def test_mcp_pivot_incident_null_profiles_preserved() -> None: + """pivot(mode='incident') preserves None profiles as null in JSON.""" ctx = _make_incident_context(src_profile=None, dest_profile=None) with patch("src.correlator.incident_context.investigate", return_value=ctx): - result_str = mcp_server.investigate(PUBLIC_SRC, PUBLIC_DEST, SENSOR, TIME_RANGE) + result_str = mcp_server.pivot( + PUBLIC_SRC, mode="incident", dest_ip=PUBLIC_DEST, sensor=SENSOR, time_range=TIME_RANGE + ) result = json.loads(result_str) assert result["status"] == "ok" assert result["data"]["src_profile"] is None assert result["data"]["dest_profile"] is None + + +@_skip_no_mcp +def test_mcp_pivot_incident_requires_dest_ip() -> None: + """pivot(mode='incident') returns error when dest_ip is omitted.""" + result_str = mcp_server.pivot(PRIVATE_SRC, mode="incident") + result = json.loads(result_str) + assert result["status"] == "error" + assert "dest_ip" in result["message"] + + +@_skip_no_mcp +def test_mcp_pivot_unknown_mode_returns_error() -> None: + """pivot() returns error for unrecognised mode values.""" + result_str = mcp_server.pivot(PRIVATE_SRC, mode="bogus") + result = json.loads(result_str) + assert result["status"] == "error" + assert "bogus" in result["message"] + + +# --------------------------------------------------------------------------- +# MCP tool — aggregate (§2.2) +# --------------------------------------------------------------------------- + +_AGGREGATE_RESPONSE = { + "aggregations": { + "buckets": { + "buckets": [ + {"key": "1.2.3.4", "doc_count": 42}, + {"key": "5.6.7.8", "doc_count": 17}, + ] + } + } +} + + +@_skip_no_mcp +def test_mcp_aggregate_returns_ok_json() -> None: + """aggregate() returns status='ok' with field/results keys.""" + with patch("src.querier.zeek_modules.base.query_opensearch", return_value=_AGGREGATE_RESPONSE): + result_str = mcp_server.aggregate( + "source.ip", log_type="notice", notice_type="Scan::Port_Scan" + ) + + result = json.loads(result_str) + assert result["status"] == "ok" + data = result["data"] + assert data["field"] == "source.ip" + assert data["log_type"] == "notice" + assert data["notice_type"] == "Scan::Port_Scan" + assert len(data["results"]) == 2 + assert data["results"][0] == {"value": "1.2.3.4", "count": 42} + assert data["results"][1] == {"value": "5.6.7.8", "count": 17} + + +@_skip_no_mcp +def test_mcp_aggregate_no_log_type() -> None: + """aggregate() works without log_type (cross-dataset).""" + empty_resp = {"aggregations": {"buckets": {"buckets": []}}} + with patch("src.querier.zeek_modules.base.query_opensearch", return_value=empty_resp): + result_str = mcp_server.aggregate("destination.ip") + + result = json.loads(result_str) + assert result["status"] == "ok" + assert result["data"]["log_type"] is None + assert result["data"]["results"] == [] + + +@_skip_no_mcp +def test_mcp_aggregate_opensearch_failure() -> None: + """aggregate() returns error when query_opensearch returns None.""" + with patch("src.querier.zeek_modules.base.query_opensearch", return_value=None): + result_str = mcp_server.aggregate("source.ip") + + result = json.loads(result_str) + assert result["status"] == "error" From d97ec5e09a4ba440e5fe66467190b714e2f0ffae Mon Sep 17 00:00:00 2001 From: liamadale Date: Tue, 12 May 2026 10:53:48 -0700 Subject: [PATCH 096/109] fix(mcp): rename mcp/ to mcp_servers/ to resolve package namespace collision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Python resolved `from mcp.server.fastmcp import FastMCP` against the local `mcp/` directory instead of the installed `mcp` package, causing a silent import failure and all MCP-dependent tests to be skipped. - Rename `mcp/` → `mcp_servers/` (7 files) to eliminate the shadowing - Update ruff per-file-ignores glob in pyproject.toml - Update ruff check, ruff format, and bandit scan paths in ci.yml - Fix `_MCP_SERVER_PATH` in tests/test_correlator.py to reflect new dir - Fix aggregate test patches to use `patch.object(mcp_server, ...)` on the server module's namespace instead of the base module, since `query_opensearch` is imported by name into the server module --- .github/workflows/ci.yml | 6 +++--- {mcp => mcp_servers}/enrichment/requirements.txt | 0 {mcp => mcp_servers}/enrichment/server.py | 0 {mcp => mcp_servers}/mantis/requirements.txt | 0 {mcp => mcp_servers}/mantis/server.py | 0 {mcp => mcp_servers}/opensearch/requirements.txt | 0 {mcp => mcp_servers}/opensearch/server.py | 0 {mcp => mcp_servers}/requirements.txt | 0 pyproject.toml | 2 +- tests/test_correlator.py | 8 ++++---- 10 files changed, 8 insertions(+), 8 deletions(-) rename {mcp => mcp_servers}/enrichment/requirements.txt (100%) rename {mcp => mcp_servers}/enrichment/server.py (100%) rename {mcp => mcp_servers}/mantis/requirements.txt (100%) rename {mcp => mcp_servers}/mantis/server.py (100%) rename {mcp => mcp_servers}/opensearch/requirements.txt (100%) rename {mcp => mcp_servers}/opensearch/server.py (100%) rename {mcp => mcp_servers}/requirements.txt (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 853c134..b8e03de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,11 +30,11 @@ jobs: continue-on-error: ${{ env.ADVISORY == 'true' }} - name: Lint (ruff check) - run: uv run ruff check src/ apps/ mcp/ tests/ *.py + run: uv run ruff check src/ apps/ mcp_servers/ tests/ *.py continue-on-error: ${{ env.ADVISORY == 'true' }} - name: Format check (ruff format) — advisory - run: uv run ruff format --check src/ apps/ mcp/ tests/ *.py + run: uv run ruff format --check src/ apps/ mcp_servers/ tests/ *.py continue-on-error: true - name: Dependency audit (pip-audit) — advisory @@ -42,7 +42,7 @@ jobs: continue-on-error: true - name: SAST (bandit) — advisory - run: uv run bandit -r src/ apps/ mcp/ -c pyproject.toml + run: uv run bandit -r src/ apps/ mcp_servers/ -c pyproject.toml continue-on-error: true - name: Lint HTML (djlint check) diff --git a/mcp/enrichment/requirements.txt b/mcp_servers/enrichment/requirements.txt similarity index 100% rename from mcp/enrichment/requirements.txt rename to mcp_servers/enrichment/requirements.txt diff --git a/mcp/enrichment/server.py b/mcp_servers/enrichment/server.py similarity index 100% rename from mcp/enrichment/server.py rename to mcp_servers/enrichment/server.py diff --git a/mcp/mantis/requirements.txt b/mcp_servers/mantis/requirements.txt similarity index 100% rename from mcp/mantis/requirements.txt rename to mcp_servers/mantis/requirements.txt diff --git a/mcp/mantis/server.py b/mcp_servers/mantis/server.py similarity index 100% rename from mcp/mantis/server.py rename to mcp_servers/mantis/server.py diff --git a/mcp/opensearch/requirements.txt b/mcp_servers/opensearch/requirements.txt similarity index 100% rename from mcp/opensearch/requirements.txt rename to mcp_servers/opensearch/requirements.txt diff --git a/mcp/opensearch/server.py b/mcp_servers/opensearch/server.py similarity index 100% rename from mcp/opensearch/server.py rename to mcp_servers/opensearch/server.py diff --git a/mcp/requirements.txt b/mcp_servers/requirements.txt similarity index 100% rename from mcp/requirements.txt rename to mcp_servers/requirements.txt diff --git a/pyproject.toml b/pyproject.toml index 9bf9e1a..cac9c05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,7 +89,7 @@ select = ["E", "F", "W", "I"] "*_web_run.py" = ["E402"] "tests/*" = ["E402"] "apps/*/run.py" = ["E402"] -"mcp/*/server.py" = ["E402", "I001"] +"mcp_servers/*/server.py" = ["E402", "I001"] "src/utils/banner.py" = ["E501"] [tool.bandit] diff --git a/tests/test_correlator.py b/tests/test_correlator.py index 7e9c7d2..aefbf4a 100644 --- a/tests/test_correlator.py +++ b/tests/test_correlator.py @@ -462,7 +462,7 @@ def test_investigate_context_gather_skips_enrichment_for_private() -> None: # --------------------------------------------------------------------------- # Load the MCP server module via its file path (it is not an installed package). -_MCP_SERVER_PATH = Path(__file__).parent.parent / "mcp" / "opensearch" / "server.py" +_MCP_SERVER_PATH = Path(__file__).parent.parent / "mcp_servers" / "opensearch" / "server.py" try: import importlib.util as _ilu @@ -635,7 +635,7 @@ def test_mcp_pivot_unknown_mode_returns_error() -> None: @_skip_no_mcp def test_mcp_aggregate_returns_ok_json() -> None: """aggregate() returns status='ok' with field/results keys.""" - with patch("src.querier.zeek_modules.base.query_opensearch", return_value=_AGGREGATE_RESPONSE): + with patch.object(mcp_server, "query_opensearch", return_value=_AGGREGATE_RESPONSE): result_str = mcp_server.aggregate( "source.ip", log_type="notice", notice_type="Scan::Port_Scan" ) @@ -655,7 +655,7 @@ def test_mcp_aggregate_returns_ok_json() -> None: def test_mcp_aggregate_no_log_type() -> None: """aggregate() works without log_type (cross-dataset).""" empty_resp = {"aggregations": {"buckets": {"buckets": []}}} - with patch("src.querier.zeek_modules.base.query_opensearch", return_value=empty_resp): + with patch.object(mcp_server, "query_opensearch", return_value=empty_resp): result_str = mcp_server.aggregate("destination.ip") result = json.loads(result_str) @@ -667,7 +667,7 @@ def test_mcp_aggregate_no_log_type() -> None: @_skip_no_mcp def test_mcp_aggregate_opensearch_failure() -> None: """aggregate() returns error when query_opensearch returns None.""" - with patch("src.querier.zeek_modules.base.query_opensearch", return_value=None): + with patch.object(mcp_server, "query_opensearch", return_value=None): result_str = mcp_server.aggregate("source.ip") result = json.loads(result_str) From 91f4e4b338a0cef187df76d2fb9e4204d6d4d0c8 Mon Sep 17 00:00:00 2001 From: liamadale Date: Tue, 12 May 2026 11:12:46 -0700 Subject: [PATCH 097/109] feat(fp-manager): extract delete_ip_from_filter into fp_manager module Adds a reusable `delete_ip_from_filter(path, ip)` function that removes must_not clauses matching an IP from a YAML filter file. Handles both single-value `term` and multi-value `terms` clause shapes; for `terms` with multiple IPs, only the matching IP is removed and the clause is kept with the remainder. Raises FileNotFoundError if the filter file is absent and ValueError if no clauses matched the given IP, so callers get actionable errors. --- src/querier/fp_manager.py | 52 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/querier/fp_manager.py b/src/querier/fp_manager.py index 83aebe2..a192d90 100755 --- a/src/querier/fp_manager.py +++ b/src/querier/fp_manager.py @@ -87,6 +87,58 @@ def write_filter_file(path: str, data: dict) -> None: yaml.dump(data, fh, default_flow_style=False, allow_unicode=True, sort_keys=False) +def delete_ip_from_filter(path: str, ip: str) -> int: + """Remove must_not clauses that match *ip* as src_ip or dest_ip. + + Handles single-value ``term`` and multi-value ``terms`` clauses. + For ``terms`` clauses with multiple IPs only the matching IP is removed; + the clause is kept with the remaining IPs. If the clause had only that + one IP it is dropped entirely. + + Returns the number of clause entries affected (removed or shrunken). + Raises ``FileNotFoundError`` if *path* does not exist. + Raises ``ValueError`` if no clauses matched the IP. + """ + if not os.path.exists(path): + raise FileNotFoundError(path) + + data = load_filter_file(path) + original_clauses: list = data.get("must_not", []) + kept: list = [] + removed_count = 0 + + for clause in original_clauses: + term = clause.get("term", {}) + if term.get("src_ip") == ip or term.get("dest_ip") == ip: + removed_count += 1 + continue + + terms = clause.get("terms", {}) + matched_field: str | None = None + for field in ("src_ip", "dest_ip"): + if ip in terms.get(field, []): + matched_field = field + break + + if matched_field: + remaining = [v for v in terms[matched_field] if v != ip] + if remaining: + new_clause = dict(clause) + new_clause["terms"] = {**terms, matched_field: remaining} + kept.append(new_clause) + removed_count += 1 + continue + + kept.append(clause) + + if removed_count == 0: + raise ValueError(f"No clauses found matching IP {ip}") + + data["must_not"] = kept + write_filter_file(path, data) + return removed_count + + def append_clauses_to_file(path: str, new_clauses: list[dict], author: str = "analyst") -> None: """Append must_not clauses to an existing filter file, or create it.""" if os.path.exists(path): From b3c21e74a290cf1fdae2f9fb48863c0458af7eb9 Mon Sep 17 00:00:00 2001 From: liamadale Date: Tue, 12 May 2026 11:12:59 -0700 Subject: [PATCH 098/109] feat(mcp/opensearch): add list_filter_categories, list_fp_filters, delete_fp_filter tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exposes three new MCP tools for read/delete access to FP filter files: - list_filter_categories: returns the full category → subcategory map from categories.yaml so callers know valid inputs before creating filters - list_fp_filters: returns a summary of all filter files, or all files in a category, or the full clause list for a specific category/subcategory - delete_fp_filter: removes every must_not clause matching an IP address from a given filter file, delegating to the new delete_ip_from_filter backend Also adds load_categories and load_filter_file to the fp_manager import set and covers all three tools with 9 regression tests in tests/test_fp_filter_tools.py. --- mcp_servers/opensearch/server.py | 118 +++++++++++++++++++++++++ tests/test_fp_filter_tools.py | 147 +++++++++++++++++++++++++++++++ 2 files changed, 265 insertions(+) create mode 100644 tests/test_fp_filter_tools.py diff --git a/mcp_servers/opensearch/server.py b/mcp_servers/opensearch/server.py index c3b869d..fa8171f 100644 --- a/mcp_servers/opensearch/server.py +++ b/mcp_servers/opensearch/server.py @@ -39,8 +39,11 @@ from src.querier.histogram import query_histogram from src.querier.fp_manager import ( append_clauses_to_file, + delete_ip_from_filter, ensure_subcategory, filter_file_path, + load_categories, + load_filter_file, ) from src.querier.zeek_modules import MODULES from src.querier.zeek_modules.base import INDEX, is_private, query_opensearch, run_query @@ -1559,6 +1562,121 @@ def create_fp_filter( return _err(str(exc)) +@mcp.tool() +def list_filter_categories() -> str: + """List all valid filter category / subcategory pairs from categories.yaml. + + Returns a mapping of category → list[subcategory]. Use this before calling + create_fp_filter so you can supply a valid category and subcategory. + """ + try: + data = load_categories() + cats = data.get("categories", {}) + return _ok(cats) + except Exception as exc: + return _err(str(exc)) + + +@mcp.tool() +def list_fp_filters( + category: Optional[str] = None, + subcategory: Optional[str] = None, +) -> str: + """Read existing false-positive filter clauses. + + When called with no arguments returns a summary of all filter files. + When called with only category returns all files in that category. + When called with both category and subcategory returns the full clause list + for that file so you can inspect what is already suppressed. + + Args: + category: Filter category directory, e.g. "ips" or "notices". + subcategory: Filter subcategory (filename without .yaml), e.g. "false_positives". + """ + try: + import os + + if category and subcategory: + path = filter_file_path(category, subcategory) + if not os.path.exists(path): + return _err(f"Filter file not found: filters/{category}/{subcategory}.yaml") + data = load_filter_file(path) + return _ok( + { + "category": category, + "subcategory": subcategory, + "enabled": data.get("enabled", True), + "description": data.get("description", ""), + "clause_count": len(data.get("must_not", [])), + "clauses": data.get("must_not", []), + } + ) + + results = [] + cats = load_categories().get("categories", {}) + + for cat, cat_data in sorted(cats.items()): + if category and cat != category: + continue + for sub in sorted(cat_data.get("subcategories", [])): + path = filter_file_path(cat, sub) + if os.path.exists(path): + fdata = load_filter_file(path) + results.append( + { + "category": cat, + "subcategory": sub, + "enabled": fdata.get("enabled", True), + "clause_count": len(fdata.get("must_not", [])), + } + ) + else: + results.append( + {"category": cat, "subcategory": sub, "enabled": None, "clause_count": 0} + ) + + return _ok(results) + except Exception as exc: + return _err(str(exc)) + + +@mcp.tool() +def delete_fp_filter( + category: str, + subcategory: str, + ip: str, +) -> str: + """Remove false-positive filter clauses that match an IP address. + + Removes every must_not clause in filters/{category}/{subcategory}.yaml + where the given IP appears as a src_ip or dest_ip value (handles both + single-value ``term`` and multi-value ``terms`` clauses). For ``terms`` + clauses with multiple IPs, only the matching IP is removed; the clause is + kept with the remaining IPs. + + Args: + category: Filter category directory, e.g. "ips". + subcategory: Filter subcategory (filename without .yaml), e.g. "false_positives". + ip: IP address to remove from the filter file. + """ + try: + path = filter_file_path(category, subcategory) + removed = delete_ip_from_filter(path, ip) + data = load_filter_file(path) + return _ok( + { + "deleted": removed, + "remaining_clauses": len(data.get("must_not", [])), + "file": path, + "ip": ip, + } + ) + except (FileNotFoundError, ValueError) as exc: + return _err(str(exc)) + except Exception as exc: + return _err(str(exc)) + + # --------------------------------------------------------------------------- # Share URLs # --------------------------------------------------------------------------- diff --git a/tests/test_fp_filter_tools.py b/tests/test_fp_filter_tools.py new file mode 100644 index 0000000..7de61d1 --- /dev/null +++ b/tests/test_fp_filter_tools.py @@ -0,0 +1,147 @@ +"""Tests for the Tier-5 FP filter read/delete helpers in fp_manager.py.""" + +from __future__ import annotations + +import os +import sys + +import pytest +import yaml + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from src.querier.fp_manager import ( + delete_ip_from_filter, + load_filter_file, +) + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +def _write_filter(path: str, clauses: list[dict]) -> None: + os.makedirs(os.path.dirname(path), exist_ok=True) + data = { + "description": "test filter", + "author": "pytest", + "date_added": "2026-01-01", + "category": "ips", + "subcategory": "test", + "enabled": True, + "must_not": clauses, + } + with open(path, "w") as fh: + yaml.dump(data, fh, default_flow_style=False) + + +# --------------------------------------------------------------------------- +# delete_ip_from_filter — term clauses +# --------------------------------------------------------------------------- + + +def test_delete_term_src_ip(tmp_path: str) -> None: + path = os.path.join(str(tmp_path), "ips", "test.yaml") + _write_filter( + path, + [ + {"term": {"src_ip": "1.2.3.4"}}, + {"term": {"src_ip": "5.6.7.8"}}, + ], + ) + removed = delete_ip_from_filter(path, "1.2.3.4") + assert removed == 1 + remaining = load_filter_file(path)["must_not"] + assert len(remaining) == 1 + assert remaining[0]["term"]["src_ip"] == "5.6.7.8" + + +def test_delete_term_dest_ip(tmp_path: str) -> None: + path = os.path.join(str(tmp_path), "ips", "test.yaml") + _write_filter(path, [{"term": {"dest_ip": "10.0.0.1"}}]) + removed = delete_ip_from_filter(path, "10.0.0.1") + assert removed == 1 + assert load_filter_file(path)["must_not"] == [] + + +def test_delete_leaves_other_clauses_untouched(tmp_path: str) -> None: + path = os.path.join(str(tmp_path), "ips", "test.yaml") + _write_filter( + path, + [ + {"term": {"src_ip": "1.2.3.4"}, "comment": "remove this"}, + {"term": {"src_ip": "9.9.9.9"}, "comment": "keep this"}, + ], + ) + delete_ip_from_filter(path, "1.2.3.4") + remaining = load_filter_file(path)["must_not"] + assert len(remaining) == 1 + assert remaining[0]["term"]["src_ip"] == "9.9.9.9" + + +# --------------------------------------------------------------------------- +# delete_ip_from_filter — terms (multi-value) clauses +# --------------------------------------------------------------------------- + + +def test_delete_single_ip_from_terms_list(tmp_path: str) -> None: + path = os.path.join(str(tmp_path), "ips", "test.yaml") + _write_filter(path, [{"terms": {"src_ip": ["1.1.1.1", "2.2.2.2", "3.3.3.3"]}}]) + removed = delete_ip_from_filter(path, "2.2.2.2") + assert removed == 1 + remaining = load_filter_file(path)["must_not"] + assert len(remaining) == 1 + assert "2.2.2.2" not in remaining[0]["terms"]["src_ip"] + assert set(remaining[0]["terms"]["src_ip"]) == {"1.1.1.1", "3.3.3.3"} + + +def test_delete_last_ip_from_terms_drops_clause(tmp_path: str) -> None: + path = os.path.join(str(tmp_path), "ips", "test.yaml") + _write_filter(path, [{"terms": {"src_ip": ["1.1.1.1"]}}]) + removed = delete_ip_from_filter(path, "1.1.1.1") + assert removed == 1 + assert load_filter_file(path)["must_not"] == [] + + +def test_delete_ip_from_terms_dest_ip(tmp_path: str) -> None: + path = os.path.join(str(tmp_path), "ips", "test.yaml") + _write_filter(path, [{"terms": {"dest_ip": ["10.0.0.1", "10.0.0.2"]}}]) + delete_ip_from_filter(path, "10.0.0.1") + remaining = load_filter_file(path)["must_not"] + assert remaining[0]["terms"]["dest_ip"] == ["10.0.0.2"] + + +# --------------------------------------------------------------------------- +# delete_ip_from_filter — error cases +# --------------------------------------------------------------------------- + + +def test_delete_raises_file_not_found(tmp_path: str) -> None: + path = os.path.join(str(tmp_path), "ips", "missing.yaml") + with pytest.raises(FileNotFoundError): + delete_ip_from_filter(path, "1.2.3.4") + + +def test_delete_raises_value_error_when_no_match(tmp_path: str) -> None: + path = os.path.join(str(tmp_path), "ips", "test.yaml") + _write_filter(path, [{"term": {"src_ip": "9.9.9.9"}}]) + with pytest.raises(ValueError, match="No clauses found"): + delete_ip_from_filter(path, "1.2.3.4") + + +def test_delete_removes_multiple_matching_clauses(tmp_path: str) -> None: + """An IP that appears in two separate clauses — both are removed.""" + path = os.path.join(str(tmp_path), "ips", "test.yaml") + _write_filter( + path, + [ + {"term": {"src_ip": "1.2.3.4"}, "comment": "first"}, + {"term": {"src_ip": "1.2.3.4"}, "comment": "duplicate"}, + {"term": {"src_ip": "5.5.5.5"}}, + ], + ) + removed = delete_ip_from_filter(path, "1.2.3.4") + assert removed == 2 + remaining = load_filter_file(path)["must_not"] + assert len(remaining) == 1 + assert remaining[0]["term"]["src_ip"] == "5.5.5.5" From f2eff4e42938dd22f7afd3fbe3b9d383ba9c4a6e Mon Sep 17 00:00:00 2001 From: liamadale Date: Tue, 12 May 2026 11:32:21 -0700 Subject: [PATCH 099/109] feat(zeek/notice,weird): support ES wildcard queries for notice_note and weird_name filters Both modules previously used a hard `term` clause, which required exact matches. Consumers now pass glob patterns (e.g. "Scan::*", "bad_*") and the module selects `wildcard` vs `term` automatically based on whether the value contains `*` or `?`. --- src/querier/zeek_modules/notice.py | 7 +++++-- src/querier/zeek_modules/weird.py | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/querier/zeek_modules/notice.py b/src/querier/zeek_modules/notice.py index 7bbd8ea..c03445c 100644 --- a/src/querier/zeek_modules/notice.py +++ b/src/querier/zeek_modules/notice.py @@ -42,8 +42,11 @@ class NoticeModule(ZeekModule): def build_extra_must(self, search_params: dict) -> tuple: clauses = [] - if search_params.get("notice_note"): - clauses.append({"term": {"zeek.notice.note": search_params["notice_note"]}}) + if val := search_params.get("notice_note"): + if "*" in val or "?" in val: + clauses.append({"wildcard": {"zeek.notice.note": val}}) + else: + clauses.append({"term": {"zeek.notice.note": val}}) return clauses, [] def parse_hit(self, src: dict) -> dict: diff --git a/src/querier/zeek_modules/weird.py b/src/querier/zeek_modules/weird.py index b53d35a..cf60346 100644 --- a/src/querier/zeek_modules/weird.py +++ b/src/querier/zeek_modules/weird.py @@ -41,8 +41,11 @@ class WeirdModule(ZeekModule): def build_extra_must(self, search_params: dict) -> tuple: clauses = [] - if search_params.get("weird_name"): - clauses.append({"term": {"rule.name": search_params["weird_name"]}}) + if val := search_params.get("weird_name"): + if "*" in val or "?" in val: + clauses.append({"wildcard": {"rule.name": val}}) + else: + clauses.append({"term": {"rule.name": val}}) return clauses, [] def parse_hit(self, src: dict) -> dict: From 8c8ac0411d7c6ebc509d6ec396f49224ecf55623 Mon Sep 17 00:00:00 2001 From: liamadale Date: Tue, 12 May 2026 11:32:26 -0700 Subject: [PATCH 100/109] test(zeek/notice,weird): add 9 tests for wildcard vs exact-match query dispatch Covers both NoticeModule and WeirdModule: exact strings produce a `term` clause, patterns containing `*` or `?` produce a `wildcard` clause, and an empty params dict returns no clauses. --- tests/test_zeek_base.py | 53 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/test_zeek_base.py b/tests/test_zeek_base.py index 318ba8d..61afb26 100644 --- a/tests/test_zeek_base.py +++ b/tests/test_zeek_base.py @@ -13,6 +13,8 @@ deduplicate_zeek, is_private, ) +from src.querier.zeek_modules.notice import NoticeModule +from src.querier.zeek_modules.weird import WeirdModule # --------------------------------------------------------------------------- # is_private @@ -498,3 +500,54 @@ def test_build_base_query_absolute_timestamps() -> None: assert ts_clause is not None assert ts_clause["range"]["@timestamp"]["gte"] == time_from assert ts_clause["range"]["@timestamp"]["lte"] == time_to + + +# --------------------------------------------------------------------------- +# NoticeModule.build_extra_must — wildcard / exact match +# --------------------------------------------------------------------------- + +_notice = NoticeModule() +_weird = WeirdModule() + + +def test_notice_exact_match_uses_term() -> None: + clauses, _ = _notice.build_extra_must({"notice_note": "Scan::Port_Scan"}) + assert len(clauses) == 1 + assert "term" in clauses[0] + assert clauses[0]["term"]["zeek.notice.note"] == "Scan::Port_Scan" + + +def test_notice_trailing_wildcard_uses_wildcard_query() -> None: + clauses, _ = _notice.build_extra_must({"notice_note": "Scan::*"}) + assert len(clauses) == 1 + assert "wildcard" in clauses[0] + assert clauses[0]["wildcard"]["zeek.notice.note"] == "Scan::*" + + +def test_notice_question_mark_uses_wildcard_query() -> None: + clauses, _ = _notice.build_extra_must({"notice_note": "SSH::Brute_Force_?"}) + assert "wildcard" in clauses[0] + + +def test_notice_no_filter_returns_empty() -> None: + clauses, _ = _notice.build_extra_must({}) + assert clauses == [] + + +def test_weird_exact_match_uses_term() -> None: + clauses, _ = _weird.build_extra_must({"weird_name": "bad_HTTP_reply"}) + assert len(clauses) == 1 + assert "term" in clauses[0] + assert clauses[0]["term"]["rule.name"] == "bad_HTTP_reply" + + +def test_weird_trailing_wildcard_uses_wildcard_query() -> None: + clauses, _ = _weird.build_extra_must({"weird_name": "bad_*"}) + assert len(clauses) == 1 + assert "wildcard" in clauses[0] + assert clauses[0]["wildcard"]["rule.name"] == "bad_*" + + +def test_weird_no_filter_returns_empty() -> None: + clauses, _ = _weird.build_extra_must({}) + assert clauses == [] From 8c453572259deb6652d597363324a87ffe5cfecd Mon Sep 17 00:00:00 2001 From: liamadale Date: Tue, 12 May 2026 11:32:34 -0700 Subject: [PATCH 101/109] feat(mcp/opensearch): add bulk_enrich_ips and count tools; surface truncated flag on all search results - _search_result() helper now returns a `truncated: bool` field on every search response so callers can tell whether the result set was capped by the limit parameter, without inspecting the count - bulk_enrich_ips() enriches a list of IPs concurrently (up to 5 workers), skips RFC-1918 addresses, and preserves input ordering - count() hits the _count endpoint so callers can check event volume or confirm an IP is active without paying the cost of fetching records; supports the same time/sensor/IP filters as search tools - Updated search_notice and search_weird docstrings to advertise wildcard support added to the query layer --- mcp_servers/opensearch/server.py | 168 +++++++++++++++++++++++++++---- 1 file changed, 148 insertions(+), 20 deletions(-) diff --git a/mcp_servers/opensearch/server.py b/mcp_servers/opensearch/server.py index fa8171f..e81d87b 100644 --- a/mcp_servers/opensearch/server.py +++ b/mcp_servers/opensearch/server.py @@ -45,8 +45,15 @@ load_categories, load_filter_file, ) +from src.querier.runner import FILTERS_DIR, load_with_remap from src.querier.zeek_modules import MODULES -from src.querier.zeek_modules.base import INDEX, is_private, query_opensearch, run_query +from src.querier.zeek_modules.base import ( + INDEX, + build_base_query, + is_private, + query_opensearch, + run_query, +) from src.utils.ip_org import lookup_org mcp = FastMCP("pisces") @@ -67,6 +74,15 @@ def _serialise_records(records: list) -> list: return out +def _search_result(records: list, limit: int) -> dict: + """Build a standard search response dict with a truncation hint.""" + return { + "count": len(records), + "truncated": len(records) >= limit, + "records": _serialise_records(records), + } + + def _ok(data) -> str: return json.dumps({"status": "ok", "data": data}, default=str) @@ -161,7 +177,7 @@ def search_conn( if proto: params["proto"] = proto records = run_query(MODULES["conn"], params) - return _ok({"count": len(records), "records": _serialise_records(records)}) + return _ok(_search_result(records, limit)) except Exception as exc: return _err(str(exc)) @@ -211,7 +227,7 @@ def search_dns( if dns_qtype: params["qtype"] = dns_qtype records = run_query(MODULES["dns"], params) - return _ok({"count": len(records), "records": _serialise_records(records)}) + return _ok(_search_result(records, limit)) except Exception as exc: return _err(str(exc)) @@ -269,7 +285,7 @@ def search_http( if dest_port is not None: params["dest_port"] = dest_port records = run_query(MODULES["http"], params) - return _ok({"count": len(records), "records": _serialise_records(records)}) + return _ok(_search_result(records, limit)) except Exception as exc: return _err(str(exc)) @@ -319,7 +335,7 @@ def search_ssl( if dest_port is not None: params["dest_port"] = dest_port records = run_query(MODULES["ssl"], params) - return _ok({"count": len(records), "records": _serialise_records(records)}) + return _ok(_search_result(records, limit)) except Exception as exc: return _err(str(exc)) @@ -369,7 +385,7 @@ def search_smtp( if smtp_subject: params["smtp_subject"] = smtp_subject records = run_query(MODULES["smtp"], params) - return _ok({"count": len(records), "records": _serialise_records(records)}) + return _ok(_search_result(records, limit)) except Exception as exc: return _err(str(exc)) @@ -419,7 +435,7 @@ def search_rdp( if dest_port is not None: params["dest_port"] = dest_port records = run_query(MODULES["rdp"], params) - return _ok({"count": len(records), "records": _serialise_records(records)}) + return _ok(_search_result(records, limit)) except Exception as exc: return _err(str(exc)) @@ -469,7 +485,7 @@ def search_smb( if dest_port is not None: params["dest_port"] = dest_port records = run_query(MODULES["smb"], params) - return _ok({"count": len(records), "records": _serialise_records(records)}) + return _ok(_search_result(records, limit)) except Exception as exc: return _err(str(exc)) @@ -519,7 +535,7 @@ def search_ssh( if dest_port is not None: params["dest_port"] = dest_port records = run_query(MODULES["ssh"], params) - return _ok({"count": len(records), "records": _serialise_records(records)}) + return _ok(_search_result(records, limit)) except Exception as exc: return _err(str(exc)) @@ -544,7 +560,8 @@ def search_notice( SSH brute-force detected, etc.). Args: - notice_note: Notice type to filter by, e.g. "Scan::Port_Scan". + notice_note: Notice type to filter by, e.g. "Scan::Port_Scan". Use a trailing + wildcard to match a namespace, e.g. "Scan::*" or "SSH::*". time_from: Absolute start timestamp (ISO 8601). Overrides time_range when both are set. time_to: Absolute end timestamp (ISO 8601). """ @@ -564,7 +581,7 @@ def search_notice( if notice_note: params["notice_note"] = notice_note records = run_query(MODULES["notice"], params) - return _ok({"count": len(records), "records": _serialise_records(records)}) + return _ok(_search_result(records, limit)) except Exception as exc: return _err(str(exc)) @@ -589,7 +606,8 @@ def search_weird( couldn't classify normally. Args: - weird_name: Weird event name to filter by, e.g. "bad_HTTP_reply". + weird_name: Weird event name to filter by, e.g. "bad_HTTP_reply". Supports + wildcards, e.g. "bad_*". time_from: Absolute start timestamp (ISO 8601). Overrides time_range when both are set. time_to: Absolute end timestamp (ISO 8601). """ @@ -609,7 +627,7 @@ def search_weird( if weird_name: params["weird_name"] = weird_name records = run_query(MODULES["weird"], params) - return _ok({"count": len(records), "records": _serialise_records(records)}) + return _ok(_search_result(records, limit)) except Exception as exc: return _err(str(exc)) @@ -675,7 +693,7 @@ def search_suricata_alert( if tag: params["tag"] = tag records = run_query(MODULES["suricata_alert"], params) - return _ok({"count": len(records), "records": _serialise_records(records)}) + return _ok(_search_result(records, limit)) except Exception as exc: return _err(str(exc)) @@ -729,7 +747,7 @@ def search_radius( if failed_only: params["failed_only"] = failed_only records = run_query(MODULES["radius"], params) - return _ok({"count": len(records), "records": _serialise_records(records)}) + return _ok(_search_result(records, limit)) except Exception as exc: return _err(str(exc)) @@ -778,7 +796,7 @@ def search_sip( if user_agent: params["user_agent"] = user_agent records = run_query(MODULES["sip"], params) - return _ok({"count": len(records), "records": _serialise_records(records)}) + return _ok(_search_result(records, limit)) except Exception as exc: return _err(str(exc)) @@ -819,7 +837,7 @@ def search_tunnel( if tunnel_type: params["tunnel_type"] = tunnel_type records = run_query(MODULES["tunnel"], params) - return _ok({"count": len(records), "records": _serialise_records(records)}) + return _ok(_search_result(records, limit)) except Exception as exc: return _err(str(exc)) @@ -864,7 +882,7 @@ def search_ntp( if version is not None: params["version"] = version records = run_query(MODULES["ntp"], params) - return _ok({"count": len(records), "records": _serialise_records(records)}) + return _ok(_search_result(records, limit)) except Exception as exc: return _err(str(exc)) @@ -909,7 +927,7 @@ def search_modbus( if exceptions_only: params["exceptions_only"] = True records = run_query(MODULES["modbus"], params) - return _ok({"count": len(records), "records": _serialise_records(records)}) + return _ok(_search_result(records, limit)) except Exception as exc: return _err(str(exc)) @@ -950,7 +968,7 @@ def search_dnp3( if function: params["function"] = function records = run_query(MODULES["dnp3"], params) - return _ok({"count": len(records), "records": _serialise_records(records)}) + return _ok(_search_result(records, limit)) except Exception as exc: return _err(str(exc)) @@ -1385,6 +1403,116 @@ def _enrich_one(item: tuple) -> dict: return _err(str(exc)) +@mcp.tool() +def bulk_enrich_ips(ips: list[str]) -> str: + """Enrich a list of IP addresses through the full threat-intelligence pipeline. + + Runs GreyNoise → AbuseIPDB → Shodan → VirusTotal concurrently for each public IP. + Private/RFC-1918 IPs are skipped — no enrichment data is available for them. + Results are returned in the same order as the input list. + + Args: + ips: List of IP addresses to enrich. + """ + try: + ip_order = {ip: i for i, ip in enumerate(ips)} + public = [ip for ip in ips if not is_private(ip)] + private_count = len(ips) - len(public) + + def _enrich_one(ip: str) -> dict: + return {"ip": ip, "enrichment": enrich_ip(ip, offer_fp=False)} + + results: list = [] + if public: + with ThreadPoolExecutor(max_workers=min(len(public), 5)) as pool: + futures = {pool.submit(_enrich_one, ip): ip for ip in public} + for future in as_completed(futures): + results.append(future.result()) + results.sort(key=lambda x: ip_order.get(x["ip"], len(ips))) + + return _ok( + { + "total_requested": len(ips), + "enriched_count": len(results), + "skipped_private": private_count, + "results": results, + } + ) + except Exception as exc: + return _err(str(exc)) + + +@mcp.tool() +def count( + log_type: str, + time_range: str = "now-24h", + sensor: str | list[str] = "all", + src_ip: str | list[str] | None = None, + dest_ip: str | list[str] | None = None, + no_filters: bool = False, + time_from: Optional[str] = None, + time_to: Optional[str] = None, +) -> str: + """Count matching events without fetching records — cheaper than search for pure counts. + + Hits the ``_count`` endpoint so no documents are shipped. Use this when you + only need a number, e.g. to check whether an IP is active before pivoting, or + to compare volume across time windows. + + Args: + log_type: Zeek log type, e.g. "conn", "notice", "dns". Use "all" for all datasets. + time_range: ES date-math range, ignored when time_from/time_to are set. + sensor: Sensor hostname or "all". + src_ip: Source IP or list of source IPs to filter by. + dest_ip: Destination IP or list of destination IPs to filter by. + no_filters: Skip FP filters. + time_from: Absolute start timestamp (ISO 8601). + time_to: Absolute end timestamp (ISO 8601). + """ + try: + if no_filters: + must_not: list = [] + else: + must_not, _fc, _errs = load_with_remap(FILTERS_DIR) + + sensors: list | None = None + if isinstance(sensor, list): + sensors = [s.strip() for s in sensor] + elif str(sensor).lower() != "all": + sensors = [s.strip() for s in str(sensor).split(",")] + + datasets = MODULES[log_type].DATASETS if log_type in MODULES and log_type != "all" else [] + + body, _params = build_base_query( + must_not=must_not, + extra_must=[], + source_fields=[], + limit=0, + time_range=time_range, + sensors=sensors, + datasets=datasets, + src_ip_filter=src_ip, + dest_ip_filter=dest_ip, + time_from=time_from, + time_to=time_to, + sort=False, + ) + count_body = {"query": body["query"]} + params = {"path": f"{INDEX}/_count", "method": "POST"} + raw = query_opensearch(count_body, params) + if raw is None: + return _err("OpenSearch query failed — check credentials and OPENSEARCH_URL") + return _ok( + { + "log_type": log_type, + "time_range": time_range, + "count": raw.get("count", 0), + } + ) + except Exception as exc: + return _err(str(exc)) + + @mcp.tool() def compare_to_baseline( notice_type: str, From fc103b676637f9d092cac18caa5dca428cb96478 Mon Sep 17 00:00:00 2001 From: liamadale Date: Thu, 4 Jun 2026 16:24:16 -0700 Subject: [PATCH 102/109] fix(zeek/notice,weird): target .keyword subfield for exact/wildcard match MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The zeek.notice.note and rule.name fields are mapped as text on rolled-over write indices, so a term query for "Scan::Port_Scan" never matches — the analyzer tokenizes it to ["scan","port_scan"]. Switching to the .keyword subfield makes term and wildcard queries match correctly across indices. The get_notice_summary aggregation was unaffected because it reads _source via a Painless script, bypassing the mapping. --- src/querier/zeek_modules/notice.py | 4 ++-- src/querier/zeek_modules/weird.py | 4 ++-- tests/test_zeek_base.py | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/querier/zeek_modules/notice.py b/src/querier/zeek_modules/notice.py index c03445c..4fa302c 100644 --- a/src/querier/zeek_modules/notice.py +++ b/src/querier/zeek_modules/notice.py @@ -44,9 +44,9 @@ def build_extra_must(self, search_params: dict) -> tuple: clauses = [] if val := search_params.get("notice_note"): if "*" in val or "?" in val: - clauses.append({"wildcard": {"zeek.notice.note": val}}) + clauses.append({"wildcard": {"zeek.notice.note.keyword": val}}) else: - clauses.append({"term": {"zeek.notice.note": val}}) + clauses.append({"term": {"zeek.notice.note.keyword": val}}) return clauses, [] def parse_hit(self, src: dict) -> dict: diff --git a/src/querier/zeek_modules/weird.py b/src/querier/zeek_modules/weird.py index cf60346..1e1d81a 100644 --- a/src/querier/zeek_modules/weird.py +++ b/src/querier/zeek_modules/weird.py @@ -43,9 +43,9 @@ def build_extra_must(self, search_params: dict) -> tuple: clauses = [] if val := search_params.get("weird_name"): if "*" in val or "?" in val: - clauses.append({"wildcard": {"rule.name": val}}) + clauses.append({"wildcard": {"rule.name.keyword": val}}) else: - clauses.append({"term": {"rule.name": val}}) + clauses.append({"term": {"rule.name.keyword": val}}) return clauses, [] def parse_hit(self, src: dict) -> dict: diff --git a/tests/test_zeek_base.py b/tests/test_zeek_base.py index 61afb26..ecc875b 100644 --- a/tests/test_zeek_base.py +++ b/tests/test_zeek_base.py @@ -514,14 +514,14 @@ def test_notice_exact_match_uses_term() -> None: clauses, _ = _notice.build_extra_must({"notice_note": "Scan::Port_Scan"}) assert len(clauses) == 1 assert "term" in clauses[0] - assert clauses[0]["term"]["zeek.notice.note"] == "Scan::Port_Scan" + assert clauses[0]["term"]["zeek.notice.note.keyword"] == "Scan::Port_Scan" def test_notice_trailing_wildcard_uses_wildcard_query() -> None: clauses, _ = _notice.build_extra_must({"notice_note": "Scan::*"}) assert len(clauses) == 1 assert "wildcard" in clauses[0] - assert clauses[0]["wildcard"]["zeek.notice.note"] == "Scan::*" + assert clauses[0]["wildcard"]["zeek.notice.note.keyword"] == "Scan::*" def test_notice_question_mark_uses_wildcard_query() -> None: @@ -538,14 +538,14 @@ def test_weird_exact_match_uses_term() -> None: clauses, _ = _weird.build_extra_must({"weird_name": "bad_HTTP_reply"}) assert len(clauses) == 1 assert "term" in clauses[0] - assert clauses[0]["term"]["rule.name"] == "bad_HTTP_reply" + assert clauses[0]["term"]["rule.name.keyword"] == "bad_HTTP_reply" def test_weird_trailing_wildcard_uses_wildcard_query() -> None: clauses, _ = _weird.build_extra_must({"weird_name": "bad_*"}) assert len(clauses) == 1 assert "wildcard" in clauses[0] - assert clauses[0]["wildcard"]["rule.name"] == "bad_*" + assert clauses[0]["wildcard"]["rule.name.keyword"] == "bad_*" def test_weird_no_filter_returns_empty() -> None: From 4ff630bb303d4a3199dce4cd532bb237121a4923 Mon Sep 17 00:00:00 2001 From: liamadale Date: Sun, 14 Jun 2026 09:30:12 -0700 Subject: [PATCH 103/109] fix(opensearch): use _source script agg to survive mapping drift Old indices map certain Zeek fields (e.g. zeek.notice.note, rule.name, event.dataset) as `keyword`. The rolled-over write index maps the same fields as `text` + `.keyword` subfield. A native `terms` aggregation sent over the wildcard index pattern hits both shards and fails on the write-index shard. Introduce `source_terms_script(field)` in `src/querier/builder.py`. It returns a Painless script object that walks `_source` segment by segment, making the bucketing immune to mapping type. Swap every affected `"field": ...` call site in `apps/opensearch_web/app.py` and `mcp_servers/opensearch/server.py` to use the script form. Re-export through `src/querier/zeek_modules/base.py` so existing importers continue to work without touching their import paths. Three tests in `tests/test_zeek_base.py` cover single fields, dotted paths, and list-valued fields. --- apps/opensearch_web/app.py | 12 +++++++++--- mcp_servers/opensearch/server.py | 5 +++-- src/querier/builder.py | 24 ++++++++++++++++++++++++ src/querier/zeek_modules/base.py | 2 ++ tests/test_zeek_base.py | 30 ++++++++++++++++++++++++++++++ 5 files changed, 68 insertions(+), 5 deletions(-) diff --git a/apps/opensearch_web/app.py b/apps/opensearch_web/app.py index 1ed3d03..5f6ce47 100644 --- a/apps/opensearch_web/app.py +++ b/apps/opensearch_web/app.py @@ -492,6 +492,7 @@ def api_summary(log_type: str): build_base_query, load_with_remap, query_opensearch, + source_terms_script, ) if log_type not in MODULES: @@ -528,13 +529,18 @@ def api_summary(log_type: str): body["size"] = 0 body.pop("_source", None) + # Read the summary field from _source so this works across indices + # whose mappings disagree (older indices have the field as keyword; + # the rolled-over write index has it as text + .keyword and a native + # terms agg over the wildcard pattern fails on that shard). + terms_script = source_terms_script(mod.SUMMARY_FIELD) + if mod.SUMMARY_TYPE == "grouped": # Aggregate on the full rule.name field; Python groups into prefixes. - # Replaces a painless script that ran on every document server-side. body["aggs"] = { "rules": { "terms": { - "field": mod.SUMMARY_FIELD, + "script": terms_script, "size": 500, "order": {"_count": "desc"}, }, @@ -552,7 +558,7 @@ def api_summary(log_type: str): body["aggs"] = { "items": { "terms": { - "field": mod.SUMMARY_FIELD, + "script": terms_script, "size": 500, "order": {"_count": "asc"}, } diff --git a/mcp_servers/opensearch/server.py b/mcp_servers/opensearch/server.py index e81d87b..97e42a6 100644 --- a/mcp_servers/opensearch/server.py +++ b/mcp_servers/opensearch/server.py @@ -53,6 +53,7 @@ is_private, query_opensearch, run_query, + source_terms_script, ) from src.utils.ip_org import lookup_org @@ -1174,7 +1175,7 @@ def get_notice_summary( "aggs": { "notice_types": { "terms": { - "field": "zeek.notice.note", + "script": source_terms_script("zeek.notice.note"), "size": limit, "order": {"_count": "desc"}, } @@ -1273,7 +1274,7 @@ def aggregate( "aggs": { "buckets": { "terms": { - "field": field, + "script": source_terms_script(field), "size": limit, "order": {"_count": "desc"}, } diff --git a/src/querier/builder.py b/src/querier/builder.py index 94c920b..943821d 100644 --- a/src/querier/builder.py +++ b/src/querier/builder.py @@ -65,6 +65,30 @@ } +def source_terms_script(field: str) -> dict: + """Return a Painless `script` object that reads `field` from `_source`. + + Drop into a ``terms`` aggregation in place of ``"field": ...`` to make the + agg work across indices whose mapping for the same dotted field disagrees + (e.g. ``keyword`` on old indices vs ``text`` + ``.keyword`` subfield on a + rolled-over write index — native terms aggs fail on the latter). + + For list-valued fields, only the first element is bucketed; in practice the + fields we use this for (notice.note, rule.name, event.dataset) are scalar. + Slower than doc-value aggs — use only when mapping drift forces it. + """ + parts = field.split(".") + src = ["def v = params._source;"] + for p in parts: + src.append(f"if (v == null) return null; v = v['{p}'];") + src.append( + "if (v == null) return null; " + "if (v instanceof List) v = v.isEmpty() ? null : v[0]; " + "return v == null ? null : v.toString();" + ) + return {"source": " ".join(src)} + + def is_private(ip: str) -> bool: """Return True if *ip* falls within any RFC-1918 / non-routable range.""" try: diff --git a/src/querier/zeek_modules/base.py b/src/querier/zeek_modules/base.py index 5390762..09c179f 100644 --- a/src/querier/zeek_modules/base.py +++ b/src/querier/zeek_modules/base.py @@ -19,6 +19,7 @@ _remap_clause, build_base_query, is_private, + source_terms_script, ) from src.querier.cli_loop import ( _search_again_prompt, @@ -73,6 +74,7 @@ "_remap_clause", "build_base_query", "is_private", + "source_terms_script", # runner "FILTERS_DIR", "_OVERFETCH_MULTIPLIER", diff --git a/tests/test_zeek_base.py b/tests/test_zeek_base.py index ecc875b..a6ce1a9 100644 --- a/tests/test_zeek_base.py +++ b/tests/test_zeek_base.py @@ -12,6 +12,7 @@ build_base_query, deduplicate_zeek, is_private, + source_terms_script, ) from src.querier.zeek_modules.notice import NoticeModule from src.querier.zeek_modules.weird import WeirdModule @@ -551,3 +552,32 @@ def test_weird_trailing_wildcard_uses_wildcard_query() -> None: def test_weird_no_filter_returns_empty() -> None: clauses, _ = _weird.build_extra_must({}) assert clauses == [] + + +# --------------------------------------------------------------------------- +# source_terms_script +# --------------------------------------------------------------------------- + + +def test_source_terms_script_single_field() -> None: + script = source_terms_script("note") + assert set(script.keys()) == {"source"} + src = script["source"] + assert src.startswith("def v = params._source;") + assert "v = v['note'];" in src + assert "v.toString()" in src + + +def test_source_terms_script_dotted_field_walks_each_segment() -> None: + src = source_terms_script("zeek.notice.note")["source"] + assert "v = v['zeek'];" in src + assert "v = v['notice'];" in src + assert "v = v['note'];" in src + # One null-guard per segment plus the final return guard. + assert src.count("if (v == null) return null;") == 4 + + +def test_source_terms_script_handles_list_valued_fields() -> None: + src = source_terms_script("rule.name")["source"] + assert "v instanceof List" in src + assert "v.isEmpty() ? null : v[0]" in src From d630adeb3e4985ba5cb0ac19e69ef25bda63064e Mon Sep 17 00:00:00 2001 From: liamadale Date: Sun, 14 Jun 2026 09:30:18 -0700 Subject: [PATCH 104/109] perf(client): raise HTTP timeout from 30s to 60s Script-based terms aggregations (_source Painless scripts) run noticeably slower than doc-value aggs. The previous 30s ceiling caused timeouts on busy clusters. 60s gives enough headroom for both sync and async clients without being unreasonably permissive. --- src/querier/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/querier/client.py b/src/querier/client.py index b3c9618..985b679 100644 --- a/src/querier/client.py +++ b/src/querier/client.py @@ -63,7 +63,7 @@ def _opensearch_client() -> tuple[str, httpx.Client]: verify=False, headers=_DEFAULT_HEADERS, limits=httpx.Limits(max_connections=20, max_keepalive_connections=16), - timeout=30.0, + timeout=60.0, ) _client_cache = (opensearch_url, username, password, client) atexit.register(client.close) @@ -135,7 +135,7 @@ async def _get_async_client() -> tuple[str, httpx.AsyncClient]: verify=False, headers=_DEFAULT_HEADERS, limits=httpx.Limits(max_connections=40, max_keepalive_connections=32), - timeout=30.0, + timeout=60.0, ) _async_client_cache = (opensearch_url, username, password, async_client) return opensearch_url, async_client From 6a47411dc489952e07306d5966df462c7277056b Mon Sep 17 00:00:00 2001 From: liamadale Date: Sun, 14 Jun 2026 09:35:55 -0700 Subject: [PATCH 105/109] chore(release): v1.1.0 Bump version to 1.1.0, add CHANGELOG.md (Keep-a-Changelog format, synthesised from 103 commits since v1.0.0), and wire up discovery paths so the log is easy to find: - README.md: one-line pointer to CHANGELOG.md under Contributing and security section - Hub footer: version string now links to CHANGELOG.md on GitHub Notable breaking-ish changes called out in the changelog that future bisects should be aware of: - mcp/ directory renamed to mcp_servers/ - mantis_web app renamed to threat_model --- CHANGELOG.md | 128 ++++++++++++++++++++++++++++++++++ README.md | 2 + apps/hub/templates/index.html | 2 +- pyproject.toml | 2 +- 4 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..6241107 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,128 @@ +# Changelog + +All notable changes to this project are documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [1.1.0] — 2026-06-14 + +### Added + +- **Incident correlator** — new Phase 1 orchestrator (`src/correlator/`) that assembles + incident context from OpenSearch, Mantis tickets, and enrichment in one call. + Exposed via the MCP `investigate` tool and `/investigate` web pages in + opensearch_web. +- **Public IP profiler** — `profile_device` now supports public IPs, with sensor + presence and reverse DNS surfaced in web device cards. +- **Mantis Explorer** — new Flask app for browsing student activity, registered in + `run_all.py` and the Hub landing page. +- **Hub redesign** — list layout, live data-freshness indicators, settings page with + theme dropdown, version + git update status in the footer. +- **Theme system** — 8 community themes (Gruvbox, Tokyo Night, Catppuccin variants), + CSS-variable-driven ECharts colors so charts render correctly on every theme. +- **Shared static blueprint** — `apps/shared/` serves tokens, base CSS, and logos + across all four web apps; per-app duplicate assets removed. +- **Dashboard** — alert trend chart, triage workqueues, per-sensor log time-series, + Tickets tab, sensor filter modal, date-range controls wired into the toolbar. +- **MCP OpenSearch tools** — `investigate`, `histogram`, `aggregate`, + `bulk_enrich_ips`, `count`, `list_filter_categories`, `list_fp_filters`, + `delete_fp_filter`, `get_notice_summary`, `build_share_urls`, + `compare_to_baseline`, `enrich_top_talkers`. +- **MCP querier** — port/proto filters, multi-value IP/sensor parameters, absolute + timestamps, `truncated` flag surfaced on all search results. +- **Wildcard filters** — `notice_note` and `weird_name` accept ES wildcard syntax, + dispatched to `wildcard` queries with exact-match fallback. +- **Filter loader** — validates category/subcategory pairs against + `filters/categories.yaml`. +- **FP manager** — `delete_ip_from_filter` extracted into reusable module. +- **Investigate UX** — escalation indicators on ticket cards, Investigate entry-points + on IP pivot and notice/Suricata records, public device cards, profile buttons. +- **Web UX** — sidebar nav with per-tab persisted filters, sticky-column rendering, + src/dest/both IP role toggle on ip_pivot, error banners for OpenSearch/Mantis, + destination IP filter in search bar. +- **Enricher** — `prewarm_enrichment_cache` background warmer, parallel execution + and result caching on the web enrich path. + +### Changed + +- **Querier refactor** — `src/querier/zeek_modules/base.py` split into focused + modules; silent `None` returns replaced with typed exceptions. +- **Web concurrency** — overview route switched from `ThreadPoolExecutor` fan-out + to `asyncio.gather`; single-flight dedup, shared thread pool, and ETag support + added; `bool.must` switched to `bool.filter` context for cacheability. +- **MCP package rename** — `mcp/` → `mcp_servers/` to resolve a namespace collision + with the upstream `mcp` package. **Breaking** for anyone importing the old path. +- **App rename** — `mantis_web` → `threat_model`. **Breaking** for any external + bookmarks or imports referencing the old name. +- **Pivot/profile/investigate** and `aggregate` MCP tools consolidated. +- **Mantis Explorer** — escalation detection rewritten; `is_escalated` surfaced on + tickets; warning modal added about escalated-count accuracy. +- **Enricher clients** — persistent HTTP sessions, retry adapter, shared console, + `atexit` cleanup across all enricher modules. +- **Dashboard / OpenSearch panels** — low-signal Malcolm panels pruned; protocol + bar replaced with time-series area charts; unified to horizontal bars. +- **Hub branding** — heading renamed to "PISCES Toolkit" with toolbox icon; brand + link navigates to hub; redundant home button removed. +- **OpenSearch web** — search bar redesigned as two-row pill layout; sensor + selector reworked as single clickable button; Investigate button moved to the + global search bar; auth history section replaced with search-all-logs. + +### Fixed + +- **OpenSearch mapping drift** — `terms` aggregations now use a `_source` Painless + script via `source_terms_script()`, surviving indices whose mapping for the + same field disagrees (keyword vs text + `.keyword` subfield on rolled-over + write index). +- **Zeek notice/weird** — exact and wildcard filters now target the `.keyword` + subfield. +- **Dashboard XSS** — date query parameters sanitised; CodeQL taint chain broken + by returning parsed ISO-format date. +- **MCP dest_ip** — pushed into the ES query instead of being post-filtered. +- **Web exceptions** — bare `except` handlers that silently swallowed tracebacks + now log the exception. +- **Cross-protocol query handler** — logs protocol name and error on failure. +- **Querier** — `FilesModule` IP filter flag corrected; `SuricataAlert` summary + type fix; `build_extra_must` tuple correctly unpacked before + `build_base_query`. +- **Correlator** — parallel profile fetches, timeline key override, ticket + deduplication. +- **OpenSearch web** — doubled `script_name` prefix removed from investigate + HTMX paths; em-dash placeholder IPs skipped in overview table. +- **Threat model / Mantis Explorer** — one-time-per-session notice modal. +- **Surface hierarchy** — Catppuccin and Gruvbox themes corrected. + +### Performance + +- **HTTP timeout** — sync and async OpenSearch clients bumped from 30s → 60s to + accommodate slower script aggregations. +- **OpenSearch client cache** — session reused across queries; query construction + optimised. +- **Filter loader** — mtime-based cache avoids re-parsing YAML on every query. +- **Mantis** — index pagination parallelised; HTTP sessions reused; linear scan + and per-request sorts replaced with dict lookups in `data.py`. +- **Filter loading / remapping / post-filtering** — redundant work removed. + +### Removed + +- Root-level standalone app launcher shims. +- `cryptography` dependency dropped; `geoip2` moved to the `offline-enrichment` + extra. +- `pytest` moved out of main dependencies into dev dependencies. +- Theme toggle buttons removed from per-app navbars (now centralised in Hub + settings). + +### CI + +- `djlint` HTML linting added to pre-commit and the CI pipeline. + +## [1.0.0] — 2026-XX-XX + +Initial tagged release. Dashboard redesign, theming, threat model rename, and +Mantis Explorer (PR #40). + +[Unreleased]: https://github.com/liamadale/pisces-scripts/compare/v1.1.0...HEAD +[1.1.0]: https://github.com/liamadale/pisces-scripts/compare/v1.0.0...v1.1.0 +[1.0.0]: https://github.com/liamadale/pisces-scripts/releases/tag/v1.0.0 diff --git a/README.md b/README.md index 644cc9a..c73958f 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,8 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for development guidelines and how to ope To report a vulnerability, follow the process in [SECURITY.md](SECURITY.md). +Release notes for each version are recorded in [CHANGELOG.md](CHANGELOG.md). + --- ## Development Transparency — Use of AI Tooling diff --git a/apps/hub/templates/index.html b/apps/hub/templates/index.html index baf2924..4ddf89c 100644 --- a/apps/hub/templates/index.html +++ b/apps/hub/templates/index.html @@ -180,7 +180,7 @@

PISCES Toolkit