From fc4834c47d8960fb2ec445dd09946b4b299c632d Mon Sep 17 00:00:00 2001 From: DjGreen Date: Thu, 16 Apr 2026 00:04:26 +0200 Subject: [PATCH 1/6] =?UTF-8?q?1.=20poprawiona=20systematyka=20plik=C3=B3w?= =?UTF-8?q?=20i=20katalog=C3=B3w=202.=20dodany=20plik=20SECURITY.md=203.?= =?UTF-8?q?=20dodany=20plik=20.python-version=204.=20dodany=20plik=20.gith?= =?UTF-8?q?ub/dependabot.yml=205.=20poprawione=20pliki=20README.md=20i=20.?= =?UTF-8?q?gitignore=206.=20dodana=20poprawiona=20implementacja=20adapter?= =?UTF-8?q?=C3=B3w,=20parser=C3=B3w=20i=20generator=C3=B3w=20dla=20APRS,?= =?UTF-8?q?=20CoT=20i=20Mesh=207.=20dodane=20testy=20dla=20adapter=C3=B3w?= =?UTF-8?q?=20CoT-Mesh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/dependabot.yml | 21 ++++ .gitignore | 3 +- .python-version | 1 + README.md | 108 +++++++++++++++++++- SECURITY.md | 74 ++++++++++++++ aprs/__init__.py | 20 ++++ aprs/adapter.py | 12 +++ aprs/agwpe.py | 10 +- aprs/agwpe_server.py | 24 +++-- aprs/encoder.py | 7 ++ aprs/generator.py | 2 + aprs/is_server.py | 18 ++-- aprs/parser.py | 39 +++++-- cot/__init__.py | 15 +++ cot/adapter.py | 7 ++ cot/cot.py | 85 +--------------- cot/cot_encoder.py | 82 +++++++++++++++ cot/cot_parser.py | 133 ++++++++++++++++++++++++ cot/encoder.py | 6 ++ cot/parser.py | 8 ++ main.py | 11 +- mesh/__init__.py | 14 ++- mesh/adapter.py | 125 ++++++++--------------- mesh/decoder.py | 7 ++ mesh/encoder.py | 8 ++ mesh/filter.py | 6 ++ mesh/mesh_decoder.py | 160 +++++++++++++++++++++++++++++ mesh/mesh_encoder.py | 60 +++++++++++ mesh/mesh_filter.py | 70 +++++++++++++ sartrack/__init__.py | 20 ++++ sartrack/adapter.py | 9 ++ sartrack/aprs_server_client.py | 13 ++- sartrack/parser.py | 41 ++++++-- sartrack/tcp_server.py | 10 +- tests/test_cot_mesh_adapters.py | 174 ++++++++++++++++++++++++++++++++ 35 files changed, 1182 insertions(+), 221 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .python-version create mode 100644 SECURITY.md create mode 100644 aprs/__init__.py create mode 100644 aprs/adapter.py create mode 100644 aprs/encoder.py create mode 100644 cot/__init__.py create mode 100644 cot/adapter.py create mode 100644 cot/cot_encoder.py create mode 100644 cot/cot_parser.py create mode 100644 cot/encoder.py create mode 100644 cot/parser.py create mode 100644 mesh/decoder.py create mode 100644 mesh/encoder.py create mode 100644 mesh/filter.py create mode 100644 mesh/mesh_decoder.py create mode 100644 mesh/mesh_encoder.py create mode 100644 mesh/mesh_filter.py create mode 100644 sartrack/__init__.py create mode 100644 sartrack/adapter.py create mode 100644 tests/test_cot_mesh_adapters.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..b8b83ec --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,21 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 5 + groups: + python-dependencies: + patterns: + - "*" + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 5 + groups: + github-actions: + patterns: + - "*" diff --git a/.gitignore b/.gitignore index d7a8b34..eba1354 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,5 @@ dist/ build/ .DS_Store -manual aprs decode.md \ No newline at end of file +manual aprs decode.md +atak cot and mesh adapter proposition.md diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..e4fba21 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/README.md b/README.md index 4ce2247..52539c5 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,29 @@ Main project areas: - `utils/` - logging setup +Preferred public module layout: + +- `aprs/` + - `adapter.py` for transport-facing APRS adapters + - `encoder.py` for APRS packet builders + - `parser.py` for `raw -> canonical -> internal event` +- `sartrack/` + - `adapter.py` for SARTrack transport adapters + - `parser.py` for `raw -> canonical -> internal event` +- `mesh/` + - `adapter.py` for Meshtastic transport integration + - `decoder.py` for `raw -> internal event` + - `encoder.py` for `internal event -> mesh payload` + - `filter.py` for outbound throttling/dedup shaping + - `parser.py` for lower-level packet normalization into canonical form +- `cot/` + - `adapter.py` for CoT-facing adapter entrypoints + - `encoder.py` for `internal event -> CoT XML/canonical` + - `parser.py` for `CoT XML -> canonical -> internal event` + +Historical implementation modules still exist under names like `generator.py`, `cot_encoder.py`, `mesh_encoder.py` or `mesh_decoder.py`. +They are intentionally kept as implementation backends for now, while the newer `adapter/parser/encoder/decoder/filter` modules are the preferred human-facing entrypoints when extending the repo. + Main runtime entrypoint: - [main.py](main.py) @@ -79,6 +102,51 @@ Main operator configuration: - [config.py](config.py) +## Event Pipeline + +The preferred mental model for the gateway is: + +`RX protocol -> parser/decoder -> canonical data -> internal event -> router -> encoder -> TX protocol` + +In plain language: + +1. An ingress adapter receives raw network or radio-facing data. +2. A protocol parser or decoder translates that payload into a canonical/internal shape. +3. The router deduplicates and dispatches the resulting event. +4. Egress adapters encode that internal event into their own transport format. + +Text diagram: + +```text +APRS / SARTrack / Mesh / CoT RX + | + v + parser / decoder layer + | + v + canonical structure + | + v + PositionEvent / MessageEvent + | + v + router + | + +------+------+------+ + | | | + v v v + APRS TX Mesh TX CoT TX + encoder encoder encoder +``` + +This is the main rule to keep in mind while extending the codebase: + +- decoding belongs in `parser.py` or `decoder.py` +- event mapping belongs close to the parser/decoder layer +- routing belongs in `core/router.py` +- transport-specific output shaping belongs in `encoder.py` +- socket/session handling belongs in `adapter.py` + ## Internal Event Model The gateway no longer treats every APRS position as the same kind of thing. @@ -435,6 +503,18 @@ Important runtime dependencies include: Python `3.11+` is recommended. +For GitHub automatic dependency submission with Python, the repository is prepared with: + +- root-level `requirements.txt` +- root-level `.python-version` +- `.github/dependabot.yml` for regular dependency update checks + +Important GitHub behavior: + +- Python autosubmission only runs when `requirements.txt` exists in the repository root +- GitHub's Python autosubmission also requires `.python-version` +- automatic dependency submission itself is enabled in repository settings, not by a repo file alone + ## Logging The gateway writes operational logs and also mirrors them into the GUI state store. @@ -490,6 +570,24 @@ Run tests with: python -m unittest tests.test_aprs_parser ``` +## GitHub Dependency Submission + +This repository is prepared for GitHub's Python automatic dependency submission so GitHub can detect transitive dependencies and raise Dependabot alerts for supported vulnerabilities. + +Prepared in-repo: + +- [requirements.txt](requirements.txt) +- [.python-version](.python-version) +- [.github/dependabot.yml](.github/dependabot.yml) + +Still required in GitHub repository settings: + +1. Enable `Dependency graph` +2. Enable `GitHub Actions` +3. Enable `Automatic dependency submission` + +GitHub will then watch pushes, run the ecosystem-specific dependency graph build, and submit detected dependencies automatically. + ## Known Limitations - Meshtastic TCP interoperability is still environment-dependent @@ -507,11 +605,15 @@ Good starting points when extending the gateway: - [main.py](main.py) - [core/model.py](core/model.py) - [core/router.py](core/router.py) +- [aprs/adapter.py](aprs/adapter.py) - [aprs/parser.py](aprs/parser.py) -- [aprs/agwpe.py](aprs/agwpe.py) -- [aprs/is_server.py](aprs/is_server.py) +- [sartrack/adapter.py](sartrack/adapter.py) +- [sartrack/parser.py](sartrack/parser.py) - [mesh/adapter.py](mesh/adapter.py) -- [cot/cot.py](cot/cot.py) +- [mesh/decoder.py](mesh/decoder.py) +- [mesh/encoder.py](mesh/encoder.py) +- [cot/adapter.py](cot/adapter.py) +- [cot/parser.py](cot/parser.py) - [gui/server.py](gui/server.py) ## License diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..a2f2785 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,74 @@ +# Security Policy + +## Supported Versions + +This project is under active development and does not currently maintain multiple supported release lines. + +Security fixes, when possible, are applied to the current development version in the default branch. + +| Version | Supported | +| ------- | --------- | +| `main` | Yes | +| older branches / snapshots | No | + +## Reporting a Vulnerability + +Please do not report security vulnerabilities through public GitHub issues. + +If you believe you have found a security issue, please report it privately to the maintainer and include as much of the following as possible: + +- affected component or file +- dependency name and version, if relevant +- environment details + - OS + - Python version + - deployment style such as local laptop, field node, VM, or test environment +- steps to reproduce +- expected behavior +- actual behavior +- logs, stack traces, screenshots, or packet samples if safe to share +- whether the issue requires local access, LAN access, radio access, or remote network access + +Examples of issues worth reporting: + +- remote code execution +- command injection +- path traversal +- credential or token leakage +- unsafe deserialization +- unauthenticated control of adapters or GUI endpoints +- denial of service caused by malformed network or radio input +- dependency vulnerabilities with realistic impact on this project + +## Disclosure Expectations + +Please allow reasonable time for investigation and remediation before any public disclosure. + +Because this is an actively evolving field/hobby project, response time may vary. The goal is to acknowledge valid reports, reproduce the issue, assess impact, and apply a fix or mitigation when feasible. + +## Scope Notes + +This repository includes: + +- network-facing adapters +- local GUI and control surfaces +- protocol parsers for APRS, SARTrack, Meshtastic, and CoT-related paths + +When reporting an issue, it is especially helpful to describe whether the issue affects: + +- input parsing +- routing logic +- adapter connection management +- configuration handling +- local operator panel behavior +- third-party dependencies + +## Dependency Security + +This repository is prepared for GitHub dependency monitoring and Dependabot-based checks. + +Dependency-related reports are still welcome, especially if: + +- a vulnerable package is in active use by the project +- the impact is reachable in real gateway workflows +- the issue affects a transitive dependency that may not be obvious from `requirements.txt` diff --git a/aprs/__init__.py b/aprs/__init__.py new file mode 100644 index 0000000..0449ad0 --- /dev/null +++ b/aprs/__init__.py @@ -0,0 +1,20 @@ +from aprs.adapter import AGWPEAdapter, AGWPEConfig, AGWPEServerAdapter, AGWPEServerConfig, APRSISServerAdapter, APRSISServerConfig +from aprs.encoder import build_message_packet, build_object_packet, build_position_packet +from aprs.parser import APRSParseError, canonical_to_aprs_event, parse_aprs_canonical, parse_aprs_event, parse_aprs_frame + +__all__ = [ + "AGWPEAdapter", + "AGWPEConfig", + "AGWPEServerAdapter", + "AGWPEServerConfig", + "APRSISServerAdapter", + "APRSISServerConfig", + "APRSParseError", + "build_message_packet", + "build_object_packet", + "build_position_packet", + "canonical_to_aprs_event", + "parse_aprs_canonical", + "parse_aprs_event", + "parse_aprs_frame", +] diff --git a/aprs/adapter.py b/aprs/adapter.py new file mode 100644 index 0000000..6311285 --- /dev/null +++ b/aprs/adapter.py @@ -0,0 +1,12 @@ +from aprs.agwpe import AGWPEAdapter, AGWPEConfig +from aprs.agwpe_server import AGWPEServerAdapter, AGWPEServerConfig +from aprs.is_server import APRSISServerAdapter, APRSISServerConfig + +__all__ = [ + "AGWPEAdapter", + "AGWPEConfig", + "AGWPEServerAdapter", + "AGWPEServerConfig", + "APRSISServerAdapter", + "APRSISServerConfig", +] diff --git a/aprs/agwpe.py b/aprs/agwpe.py index 99f1ead..c887a41 100644 --- a/aprs/agwpe.py +++ b/aprs/agwpe.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from aprs.generator import build_message_packet, build_object_packet, build_position_packet -from aprs.parser import APRSParseError, parse_aprs_frame +from aprs.parser import APRSParseError, parse_aprs_event from core.model import MessageEvent, PositionEvent, Source from core.router import EventRouter from core.state import StateStore @@ -354,9 +354,11 @@ def _decode_aprs_event(self, header: dict[str, object], payload: bytes) -> Posit self.log.debug("RX | decoded-frame | adapter=agwpe frame=%s", frame) try: - event = parse_aprs_frame(frame) - if isinstance(event.raw, dict): - event.raw["ingress_adapter"] = "agwpe" + event = parse_aprs_event( + frame, + source=Source.APRS, + ingress_adapter="agwpe", + ) return event except APRSParseError as exc: self.log.warning("RX | unsupported | adapter=agwpe error=%s", exc) diff --git a/aprs/agwpe_server.py b/aprs/agwpe_server.py index 37c43eb..ec01721 100644 --- a/aprs/agwpe_server.py +++ b/aprs/agwpe_server.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from aprs.generator import build_message_packet, build_object_packet, build_position_packet -from aprs.parser import APRSParseError, parse_aprs_frame +from aprs.parser import APRSParseError, parse_aprs_event from core.model import MessageEvent, PositionEvent, Source from core.router import EventRouter from core.state import StateStore @@ -344,7 +344,11 @@ def _decode_unproto_event( frame = f"{from_call}>{to_call}:{text}" self.log.info("RX | decoded-frame | adapter=agwpe.server frame=%s", frame) try: - return parse_aprs_frame(frame) + return parse_aprs_event( + frame, + source=Source.APRS, + ingress_adapter="agwpe_server", + ) except APRSParseError as exc: self.log.warning("RX | unsupported | adapter=agwpe.server error=%s", exc) return None @@ -389,15 +393,19 @@ def _decode_v_event( self._log_special_message_type(from_call, body) try: - event = parse_aprs_frame(frame) + event = parse_aprs_event( + frame, + source=Source.APRS, + ingress_adapter="agwpe_server", + raw_context={ + "agwpe_v_prefix": prefix, + "agwpe_v_body": body, + "agwpe_v_path": frame_path, + }, + ) except APRSParseError as exc: self.log.warning("RX | unsupported | adapter=agwpe.server kind=V error=%s", exc) return None - - if isinstance(event.raw, dict): - event.raw["agwpe_v_prefix"] = prefix - event.raw["agwpe_v_body"] = body - event.raw["agwpe_v_path"] = frame_path return event def _log_special_message_type(self, source_id: str, body: str) -> None: diff --git a/aprs/encoder.py b/aprs/encoder.py new file mode 100644 index 0000000..215c8ec --- /dev/null +++ b/aprs/encoder.py @@ -0,0 +1,7 @@ +from aprs.generator import build_message_packet, build_object_packet, build_position_packet + +__all__ = [ + "build_message_packet", + "build_object_packet", + "build_position_packet", +] diff --git a/aprs/generator.py b/aprs/generator.py index 0124c69..c6d10d5 100644 --- a/aprs/generator.py +++ b/aprs/generator.py @@ -1,5 +1,7 @@ from __future__ import annotations +"""Legacy APRS packet builder implementation behind aprs.encoder.""" + import time from core.model import MessageEvent, PositionEvent diff --git a/aprs/is_server.py b/aprs/is_server.py index 46347ae..43e0190 100644 --- a/aprs/is_server.py +++ b/aprs/is_server.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from aprs.generator import build_message_packet, build_object_packet, build_position_packet -from aprs.parser import APRSParseError, parse_aprs_frame +from aprs.parser import APRSParseError, parse_aprs_event from core.model import MessageEvent, PositionEvent, Source from core.router import EventRouter from core.state import StateStore @@ -186,17 +186,19 @@ async def _handle_client(self, reader: asyncio.StreamReader, writer: asyncio.Str continue try: - event = parse_aprs_frame(raw) + event = parse_aprs_event( + raw, + source=Source.SARTRACK, + ingress_adapter="aprs_is_server", + raw_context={ + "aprs_is_peer": peer, + "aprs_is_line": raw, + }, + ) except APRSParseError as exc: self.log.warning("RX | unsupported | adapter=aprs.is peer=%s error=%s", peer, exc) continue - event.source = Source.SARTRACK - if isinstance(event.raw, dict): - event.raw["ingress_adapter"] = "aprs_is_server" - event.raw["aprs_is_peer"] = peer - event.raw["aprs_is_line"] = raw - self.log.info("RX | accepted | adapter=aprs.is peer=%s frame=%s", peer, raw) if self.router: await self.router.publish(event) diff --git a/aprs/parser.py b/aprs/parser.py index cd7cbe6..07183fe 100644 --- a/aprs/parser.py +++ b/aprs/parser.py @@ -44,9 +44,20 @@ class APRSParseError(ValueError): pass -def parse_aprs_frame(frame: str) -> PositionEvent | MessageEvent: +def parse_aprs_event( + frame: str, + *, + source: Source = Source.APRS, + ingress_adapter: str | None = None, + raw_context: dict[str, object] | None = None, +) -> PositionEvent | MessageEvent: canonical = parse_aprs_canonical(frame) - event = canonical_to_aprs_event(canonical) + event = canonical_to_aprs_event( + canonical, + source=source, + ingress_adapter=ingress_adapter, + raw_context=raw_context, + ) log.debug( "APRS canonical -> %s source=%s subtype=%s msg_id=%r", event.kind, @@ -57,6 +68,10 @@ def parse_aprs_frame(frame: str) -> PositionEvent | MessageEvent: return event +def parse_aprs_frame(frame: str) -> PositionEvent | MessageEvent: + return parse_aprs_event(frame) + + def parse_aprs_canonical(frame: str) -> dict[str, object]: frame = frame.strip() match = APRS_FRAME_RE.match(frame) @@ -121,7 +136,13 @@ def parse_aprs_canonical(frame: str) -> dict[str, object]: raise APRSParseError(f"unsupported APRS body: {body}") -def canonical_to_aprs_event(canonical: dict[str, object]) -> PositionEvent | MessageEvent: +def canonical_to_aprs_event( + canonical: dict[str, object], + *, + source: Source = Source.APRS, + ingress_adapter: str | None = None, + raw_context: dict[str, object] | None = None, +) -> PositionEvent | MessageEvent: identity = canonical["identity"] position = canonical["position"] text = canonical["text"] @@ -130,14 +151,20 @@ def canonical_to_aprs_event(canonical: dict[str, object]) -> PositionEvent | Mes if not source_id: raise APRSParseError("canonical APRS event missing source callsign") + raw_payload: dict[str, object] = {"canonical": canonical} + if raw_context: + raw_payload.update(raw_context) + if ingress_adapter: + raw_payload["ingress_adapter"] = ingress_adapter + if text["text"]: return MessageEvent( id=str(source_id), target=str(canonical["sartrack"]["receiver"]) if canonical["sartrack"]["receiver"] else None, message=str(text["text"]), message_id=str(meta["message_id"]) if meta["message_id"] else None, - source=Source.APRS, - raw={"canonical": canonical}, + source=source, + raw=raw_payload, ) if position["lat"] is None or position["lon"] is None: @@ -160,7 +187,7 @@ def canonical_to_aprs_event(canonical: dict[str, object]) -> PositionEvent | Mes owner_id=str(canonical["sartrack"]["custom"].get("owner_callsign")) if isinstance(canonical["sartrack"]["custom"], dict) and canonical["sartrack"]["custom"].get("owner_callsign") else None, - raw={"canonical": canonical}, + raw=raw_payload, ) diff --git a/cot/__init__.py b/cot/__init__.py new file mode 100644 index 0000000..994fd63 --- /dev/null +++ b/cot/__init__.py @@ -0,0 +1,15 @@ +from cot.adapter import CoTAdapter, CoTConfig, build_cot_event +from cot.encoder import encode_event_to_cot_xml, position_event_to_canonical +from cot.parser import CoTParseError, canonical_to_cot_event, parse_cot_canonical, parse_cot_xml + +__all__ = [ + "CoTAdapter", + "CoTConfig", + "CoTParseError", + "build_cot_event", + "canonical_to_cot_event", + "encode_event_to_cot_xml", + "parse_cot_canonical", + "parse_cot_xml", + "position_event_to_canonical", +] diff --git a/cot/adapter.py b/cot/adapter.py new file mode 100644 index 0000000..2cdf4fe --- /dev/null +++ b/cot/adapter.py @@ -0,0 +1,7 @@ +from cot.cot import CoTAdapter, CoTConfig, build_cot_event + +__all__ = [ + "CoTAdapter", + "CoTConfig", + "build_cot_event", +] diff --git a/cot/cot.py b/cot/cot.py index b7766cc..fc04667 100644 --- a/cot/cot.py +++ b/cot/cot.py @@ -1,13 +1,11 @@ from __future__ import annotations import logging -import time -import xml.etree.ElementTree as ET from dataclasses import dataclass -from core.canonical import empty_canonical_event from core.model import MessageEvent, PositionEvent, Source from core.state import StateStore +from cot.cot_encoder import encode_event_to_cot_xml, position_event_to_canonical @dataclass(slots=True) @@ -74,88 +72,11 @@ async def reconnect_now(self) -> None: def build_cot_event(event: PositionEvent) -> str: canonical = position_event_to_canonical(event) - position = canonical["position"] - tak = canonical["tak"] - identity = canonical["identity"] - text = canonical["text"] - - now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp)) - stale = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp + event.ttl)) - - root = ET.Element( - "event", - { - "version": str(tak["cot_version"] or "2.0"), - "uid": str(identity["uid"] or event.id), - "type": str(tak["cot_type"] or "a-f-G-U-C"), - "time": str(position["timestamp_utc"] or now), - "start": str(tak["start_utc"] or now), - "stale": str(tak["stale_utc"] or stale), - "how": str(tak["how"] or "m-g"), - }, - ) - if position["lat"] is not None and position["lon"] is not None: - ET.SubElement( - root, - "point", - { - "lat": str(position["lat"]), - "lon": str(position["lon"]), - "hae": str(position["alt_m"] or 0), - "ce": str(position["ce_m"] or 9999999), - "le": str(position["le_m"] or 9999999), - }, - ) - detail = ET.SubElement(root, "detail") - remarks = ET.SubElement(detail, "remarks") - remarks.text = str(text["remarks"] or text["comment"] or "") logging.getLogger("cot").debug( "CoT canonical -> xml source=%s lat=%s lon=%s type=%s", - identity["uid"], - position["lat"], - position["lon"], - tak["cot_type"] or "a-f-G-U-C", - ) - return ET.tostring(root, encoding="unicode") - - -def position_event_to_canonical(event: PositionEvent) -> dict[str, object]: - canonical = empty_canonical_event() - canonical["meta"]["source_format"] = "cot_xml" - canonical["meta"]["source_subtype"] = "position" - canonical["meta"]["parser_version"] = 1 - canonical["identity"]["uid"] = event.object_name or event.id - canonical["identity"]["callsign"] = event.owner_id or event.id - canonical["identity"]["short_name"] = event.tactical_name or event.object_name or event.id - canonical["identity"]["object_type"] = "unknown" if event.is_object() else "person" - canonical["position"]["lat"] = event.lat - canonical["position"]["lon"] = event.lon - canonical["position"]["alt_m"] = event.alt - canonical["position"]["ce_m"] = 9999999 - canonical["position"]["le_m"] = 9999999 - canonical["position"]["timestamp_utc"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp)) - canonical["text"]["comment"] = event.message - if event.is_object(): - parts = [f"APRS object {event.object_name or event.id}"] - if event.owner_id: - parts.append(f"owner {event.owner_id}") - if event.message: - parts.append(event.message) - canonical["text"]["remarks"] = " | ".join(parts) - else: - canonical["text"]["remarks"] = event.message - canonical["tak"]["cot_version"] = "2.0" - canonical["tak"]["cot_type"] = "b-m-p-s-p-loc" if event.is_object() else "a-f-G-U-C" - canonical["tak"]["how"] = "m-g" - canonical["tak"]["start_utc"] = canonical["position"]["timestamp_utc"] - canonical["tak"]["stale_utc"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp + event.ttl)) - logging.getLogger("cot").debug( - "CoT canonical event created format=%s subtype=%s entity=%s uid=%s lat=%s lon=%s", - canonical["meta"]["source_format"], - canonical["meta"]["source_subtype"], - event.entity_kind, canonical["identity"]["uid"], canonical["position"]["lat"], canonical["position"]["lon"], + canonical["tak"]["cot_type"] or "a-f-G-U-C", ) - return canonical + return encode_event_to_cot_xml(event) diff --git a/cot/cot_encoder.py b/cot/cot_encoder.py new file mode 100644 index 0000000..abcda57 --- /dev/null +++ b/cot/cot_encoder.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +"""Legacy CoT encoding implementation behind cot.encoder.""" + +import time +import xml.etree.ElementTree as ET + +from core.canonical import empty_canonical_event +from core.model import PositionEvent + + +def encode_event_to_cot_xml(event: PositionEvent) -> str: + canonical = position_event_to_canonical(event) + position = canonical["position"] + tak = canonical["tak"] + identity = canonical["identity"] + text = canonical["text"] + + now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp)) + stale = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp + event.ttl)) + + root = ET.Element( + "event", + { + "version": str(tak["cot_version"] or "2.0"), + "uid": str(identity["uid"] or event.id), + "type": str(tak["cot_type"] or "a-f-G-U-C"), + "time": str(position["timestamp_utc"] or now), + "start": str(tak["start_utc"] or now), + "stale": str(tak["stale_utc"] or stale), + "how": str(tak["how"] or "m-g"), + }, + ) + if position["lat"] is not None and position["lon"] is not None: + ET.SubElement( + root, + "point", + { + "lat": str(position["lat"]), + "lon": str(position["lon"]), + "hae": str(position["alt_m"] or 0), + "ce": str(position["ce_m"] or 9999999), + "le": str(position["le_m"] or 9999999), + }, + ) + detail = ET.SubElement(root, "detail") + remarks = ET.SubElement(detail, "remarks") + remarks.text = str(text["remarks"] or text["comment"] or "") + return ET.tostring(root, encoding="unicode") + + +def position_event_to_canonical(event: PositionEvent) -> dict[str, object]: + canonical = empty_canonical_event() + canonical["meta"]["source_format"] = "cot_xml" + canonical["meta"]["source_subtype"] = "position" + canonical["meta"]["parser_version"] = 1 + canonical["identity"]["uid"] = event.object_name or event.id + canonical["identity"]["callsign"] = event.owner_id or event.id + canonical["identity"]["short_name"] = event.tactical_name or event.object_name or event.id + canonical["identity"]["object_type"] = "unknown" if event.is_object() else "person" + canonical["position"]["lat"] = event.lat + canonical["position"]["lon"] = event.lon + canonical["position"]["alt_m"] = event.alt + canonical["position"]["ce_m"] = 9999999 + canonical["position"]["le_m"] = 9999999 + canonical["position"]["timestamp_utc"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp)) + canonical["text"]["comment"] = event.message + if event.is_object(): + parts = [f"APRS object {event.object_name or event.id}"] + if event.owner_id: + parts.append(f"owner {event.owner_id}") + if event.message: + parts.append(event.message) + canonical["text"]["remarks"] = " | ".join(parts) + else: + canonical["text"]["remarks"] = event.message + canonical["tak"]["cot_version"] = "2.0" + canonical["tak"]["cot_type"] = "b-m-p-s-p-loc" if event.is_object() else "a-f-G-U-C" + canonical["tak"]["how"] = "m-g" + canonical["tak"]["start_utc"] = canonical["position"]["timestamp_utc"] + canonical["tak"]["stale_utc"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp + event.ttl)) + return canonical diff --git a/cot/cot_parser.py b/cot/cot_parser.py new file mode 100644 index 0000000..72f8921 --- /dev/null +++ b/cot/cot_parser.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +import re +import xml.etree.ElementTree as ET + +from core.canonical import empty_canonical_event +from core.model import PositionEvent, Source + + +class CoTParseError(ValueError): + pass + + +OBJECT_REMARKS_RE = re.compile( + r"^APRS object (?P[^|]+?)(?: \| owner (?P[^|]+?))?(?: \| (?P.*))?$" +) + + +def parse_cot_xml(xml_text: str) -> PositionEvent: + canonical = parse_cot_canonical(xml_text) + return canonical_to_cot_event(canonical, raw={"cot_xml": xml_text}) + + +def parse_cot_canonical(xml_text: str) -> dict[str, object]: + try: + root = ET.fromstring(xml_text) + except ET.ParseError as exc: + raise CoTParseError(str(exc)) from exc + + if root.tag != "event": + raise CoTParseError(f"unsupported root tag: {root.tag}") + + point = root.find("point") + if point is None: + raise CoTParseError("missing point element") + + uid = root.attrib.get("uid") + if not uid: + raise CoTParseError("missing uid") + + try: + lat = float(point.attrib["lat"]) + lon = float(point.attrib["lon"]) + except (KeyError, ValueError) as exc: + raise CoTParseError("invalid point coordinates") from exc + + detail = root.find("detail") + remarks = None + if detail is not None: + remarks_elem = detail.find("remarks") + if remarks_elem is not None and remarks_elem.text: + remarks = remarks_elem.text.strip() or None + + cot_type = root.attrib.get("type", "") + entity_kind = "object" if cot_type == "b-m-p-s-p-loc" else "station" + canonical = empty_canonical_event() + canonical["meta"]["source_format"] = "cot_xml" + canonical["meta"]["source_subtype"] = "object" if entity_kind == "object" else "position" + canonical["meta"]["parser_version"] = 1 + canonical["meta"]["raw_message"] = xml_text + canonical["identity"]["uid"] = uid + canonical["identity"]["callsign"] = uid + canonical["identity"]["short_name"] = uid + canonical["identity"]["object_type"] = "unknown" if entity_kind == "object" else "person" + canonical["position"]["lat"] = lat + canonical["position"]["lon"] = lon + canonical["position"]["alt_m"] = _optional_float(point.attrib.get("hae")) + canonical["position"]["ce_m"] = _optional_float(point.attrib.get("ce")) + canonical["position"]["le_m"] = _optional_float(point.attrib.get("le")) + canonical["position"]["timestamp_utc"] = root.attrib.get("time") + canonical["text"]["remarks"] = remarks + canonical["text"]["comment"] = remarks + canonical["tak"]["cot_version"] = root.attrib.get("version") + canonical["tak"]["cot_type"] = cot_type or None + canonical["tak"]["how"] = root.attrib.get("how") + canonical["tak"]["start_utc"] = root.attrib.get("start") + canonical["tak"]["stale_utc"] = root.attrib.get("stale") + return canonical + + +def canonical_to_cot_event( + canonical: dict[str, object], + *, + raw: dict[str, object] | None = None, +) -> PositionEvent: + identity = canonical["identity"] + position = canonical["position"] + text = canonical["text"] + tak = canonical["tak"] + + uid = identity["uid"] + if not uid: + raise CoTParseError("canonical CoT event missing uid") + if position["lat"] is None or position["lon"] is None: + raise CoTParseError("canonical CoT event missing coordinates") + + cot_type = str(tak["cot_type"] or "") + entity_kind = "object" if cot_type == "b-m-p-s-p-loc" else "station" + position_type = "object" if entity_kind == "object" else "position" + remarks = str(text["remarks"]) if text["remarks"] else None + object_name = None + owner_id = None + message = remarks + + if entity_kind == "object" and remarks: + match = OBJECT_REMARKS_RE.match(remarks) + if match: + object_name = (match.group("object_name") or "").strip() or None + owner_id = (match.group("owner_id") or "").strip() or None + message = (match.group("message") or "").strip() or None + + return PositionEvent( + id=str(object_name or uid), + entity_kind=entity_kind, + type=position_type, + lat=float(position["lat"]), + lon=float(position["lon"]), + alt=_optional_float(position.get("alt_m")), + message=message, + object_name=object_name, + owner_id=owner_id, + source=Source.COT, + raw=raw or {"canonical": canonical}, + ) + + +def _optional_float(value: object) -> float | None: + if value in (None, ""): + return None + try: + return float(value) + except (TypeError, ValueError): + return None diff --git a/cot/encoder.py b/cot/encoder.py new file mode 100644 index 0000000..c2d3adf --- /dev/null +++ b/cot/encoder.py @@ -0,0 +1,6 @@ +from cot.cot_encoder import encode_event_to_cot_xml, position_event_to_canonical + +__all__ = [ + "encode_event_to_cot_xml", + "position_event_to_canonical", +] diff --git a/cot/parser.py b/cot/parser.py new file mode 100644 index 0000000..c7439db --- /dev/null +++ b/cot/parser.py @@ -0,0 +1,8 @@ +from cot.cot_parser import CoTParseError, canonical_to_cot_event, parse_cot_canonical, parse_cot_xml + +__all__ = [ + "CoTParseError", + "canonical_to_cot_event", + "parse_cot_canonical", + "parse_cot_xml", +] diff --git a/main.py b/main.py index 4c7accf..ce43245 100644 --- a/main.py +++ b/main.py @@ -3,9 +3,7 @@ import asyncio import logging -from aprs.agwpe import AGWPEAdapter -from aprs.agwpe_server import AGWPEServerAdapter -from aprs.is_server import APRSISServerAdapter +from aprs import AGWPEAdapter, AGWPEServerAdapter, APRSISServerAdapter from config import ( DemoConfig, load_config, @@ -21,11 +19,10 @@ from core.model import MessageEvent, PositionEvent, Source from core.router import EventRouter from core.state import StateStore -from cot.cot import CoTAdapter +from cot import CoTAdapter from gui import ConnectionControl, GuiServer -from mesh.adapter import MeshtasticAdapter -from sartrack.aprs_server_client import SARTrackAPRSServerClient -from sartrack.tcp_server import SARTrackTCPServer +from mesh import MeshtasticAdapter +from sartrack import SARTrackAPRSServerClient, SARTrackTCPServer from utils.logger import setup_logging diff --git a/mesh/__init__.py b/mesh/__init__.py index 19c0d87..05cc31d 100644 --- a/mesh/__init__.py +++ b/mesh/__init__.py @@ -1,10 +1,22 @@ from mesh.adapter import MeshtasticAdapter, MeshtasticConfig +from mesh.decoder import decode_mesh_packet, decode_mesh_position_packet, decode_mesh_text_packet +from mesh.encoder import build_waypoint_fields, build_waypoint_id, encode_position_payload, encode_text_payload +from mesh.filter import MeshFilter, MeshFilterDecision from mesh.parser import MeshtasticParseError, canonical_to_event, parse_meshtastic_packet __all__ = [ "MeshtasticAdapter", "MeshtasticConfig", + "MeshFilter", + "MeshFilterDecision", "MeshtasticParseError", - "parse_meshtastic_packet", + "build_waypoint_fields", + "build_waypoint_id", "canonical_to_event", + "decode_mesh_packet", + "decode_mesh_position_packet", + "decode_mesh_text_packet", + "encode_position_payload", + "encode_text_payload", + "parse_meshtastic_packet", ] diff --git a/mesh/adapter.py b/mesh/adapter.py index 221f252..cfef67f 100644 --- a/mesh/adapter.py +++ b/mesh/adapter.py @@ -15,7 +15,10 @@ from core.model import MessageEvent, PositionEvent, Source from core.router import EventRouter from core.state import StateStore -from mesh.parser import canonical_to_event, parse_meshtastic_packet +from mesh.mesh_decoder import decode_mesh_packet +from mesh.mesh_encoder import build_waypoint_fields, build_waypoint_id, encode_position_payload, encode_text_payload +from mesh.mesh_filter import MeshFilter +from mesh.parser import parse_meshtastic_packet APRS_CALLSIGN_RE = re.compile(r"^[A-Z0-9]{1,6}(?:-[0-9]{1,2})?$") @@ -156,6 +159,7 @@ def __init__(self, config: MeshtasticConfig | None = None) -> None: self._reconnect_task: asyncio.Task[None] | None = None self._reconnect_attempt = 0 self.state_store: StateStore | None = None + self.mesh_filter = MeshFilter() def attach_router(self, router: EventRouter) -> None: self.router = router @@ -231,10 +235,20 @@ async def send_position(self, event: PositionEvent) -> None: self.log.debug("Meshtastic position send skipped: event id=%s has no coordinates", event.id) return + decision = self.mesh_filter.allow_position(event) + if not decision.allowed: + self.log.debug( + "Meshtastic position send skipped: entity=%s id=%s reason=%s", + event.entity_kind, + event.id, + decision.reason, + ) + return + try: if self.config.position_transport_mode == "waypoint": - name, description = self._build_waypoint_fields(event) - waypoint_id = self._build_waypoint_id(event.object_name or event.id) + name, description = build_waypoint_fields(event) + waypoint_id = build_waypoint_id(event.object_name or event.id) self.log.debug( "Meshtastic APRS waypoint payload: entity=%s id=%s waypoint_id=%s name=%r description=%r lat=%s lon=%s", event.entity_kind, @@ -254,7 +268,7 @@ async def send_position(self, event: PositionEvent) -> None: ) return - payload = self._build_position_payload(event) + payload = encode_position_payload(event) self.log.debug("Meshtastic APRS position payload: %s", payload) self._remember_outbound_payload(payload) await asyncio.to_thread(self._send_text_blocking, payload, self.config.destination_id) @@ -266,7 +280,11 @@ async def send_message(self, event: MessageEvent) -> None: if not self.config.enabled: self.log.debug("Meshtastic message send skipped: adapter disabled") return - payload = self._build_text_message(event) + decision = self.mesh_filter.allow_message(event) + if not decision.allowed: + self.log.debug("Meshtastic message send skipped: id=%s reason=%s", event.id, decision.reason) + return + payload = encode_text_payload(event) self.log.info("Meshtastic message outbound id=%s target=%s", event.id, event.target) self.log.debug("Meshtastic APRS message payload: %s", payload) @@ -483,56 +501,47 @@ def _resolve_target_from_packet(self, packet: dict[str, Any]) -> str | None: def _on_text_receive(self, packet: dict[str, Any], interface: Any = None, topic: Any = None) -> None: self.log.debug("Meshtastic raw text packet: %s", packet) - canonical = parse_meshtastic_packet(packet) - text = canonical.get("text", {}).get("text") + decoded = packet.get("decoded") or {} + text = decoded.get("text") if not text: return - text = text.strip() + text = str(text).strip() if self._is_recent_outbound_payload(text): self.log.debug("Meshtastic RX text ignored as bridge echo: %r", text) return - if text.startswith("APRS_POS|") or text.startswith("[APRS "): - self.log.debug("Meshtastic RX text ignored as bridge-formatted payload: %r", text) - return - source_id = self._resolve_callsign_from_packet(packet) if not source_id: self.log.debug("Meshtastic RX text ignored: no APRS callsign mapping for packet %s", packet) return - event = canonical_to_event( - canonical, + decoded_event = decode_mesh_packet( + packet, source_id=source_id, - target=self._resolve_target_from_packet(packet), - raw=packet, + target_id=self._resolve_target_from_packet(packet), ) - if not isinstance(event, MessageEvent): - self.log.warning("Meshtastic text packet could not be normalized into MessageEvent: %s", packet) + if decoded_event is not None: + self.log.info( + "Meshtastic RX decoded bridge payload kind=%s id=%s target=%s", + decoded_event.kind, + decoded_event.id, + getattr(decoded_event, "target", None), + ) + self._publish_from_callback(decoded_event) return - - self.log.info("Meshtastic RX text source=%s target=%s msg=%r", event.id, event.target, event.message) - self._publish_from_callback(event) + self.log.warning("Meshtastic text packet could not be decoded into internal event: %s", packet) def _on_position_receive(self, packet: dict[str, Any], interface: Any = None, topic: Any = None) -> None: self.log.debug("Meshtastic raw position packet: %s", packet) - canonical = parse_meshtastic_packet(packet) - position = canonical.get("position", {}) - lat = position.get("lat") - lon = position.get("lon") - if lat is None or lon is None: - return - source_id = self._resolve_callsign_from_packet(packet) if not source_id: self.log.debug("Meshtastic RX position ignored: no APRS callsign mapping for packet %s", packet) return - event = canonical_to_event( - canonical, + event = decode_mesh_packet( + packet, source_id=source_id, - raw=packet, ) if not isinstance(event, PositionEvent): self.log.warning("Meshtastic position packet could not be normalized into PositionEvent: %s", packet) @@ -652,57 +661,3 @@ def _normalize_callsign(value: str | None) -> str | None: return candidate return None - @staticmethod - def _build_position_payload(event: PositionEvent) -> str: - parts = [ - "APRS_POS", - f"entity={event.entity_kind}", - f"id={event.id}", - f"name={event.tactical_name or ''}", - f"lat={event.lat}", - f"lon={event.lon}", - f"ts={event.timestamp}", - ] - if event.object_name: - parts.append(f"object={event.object_name}") - if event.owner_id: - parts.append(f"owner={event.owner_id}") - if event.symbol_table: - parts.append(f"symbol_table={event.symbol_table}") - if event.symbol_code: - parts.append(f"symbol_code={event.symbol_code}") - if event.message: - parts.append(f"msg={event.message}") - return "|".join(parts) - - @staticmethod - def _build_text_message(event: MessageEvent) -> str: - if event.target: - if event.message_id: - return f"[APRS {event.id}->{event.target} #{event.message_id}] {event.message}" - return f"[APRS {event.id}->{event.target}] {event.message}" - - if event.message_id: - return f"[APRS {event.id} #{event.message_id}] {event.message}" - return f"[APRS {event.id}] {event.message}" - - @staticmethod - def _build_waypoint_fields(event: PositionEvent) -> tuple[str, str]: - label_source = event.object_name or event.tactical_name or event.id - label = label_source.strip()[:30] - if event.is_object(): - description_parts = [f"APRS object {event.object_name or event.id}"] - if event.owner_id: - description_parts.append(f"owner {event.owner_id}") - else: - description_parts = [f"APRS station {event.id}"] - if event.message: - description_parts.append(event.message.strip()) - return label, " | ".join(description_parts)[:120] - - @staticmethod - def _build_waypoint_id(source_id: str) -> int: - value = 0 - for char in source_id.upper(): - value = ((value * 33) + ord(char)) % 1_000_000_000 - return value or 1 diff --git a/mesh/decoder.py b/mesh/decoder.py new file mode 100644 index 0000000..0c76d6f --- /dev/null +++ b/mesh/decoder.py @@ -0,0 +1,7 @@ +from mesh.mesh_decoder import decode_mesh_packet, decode_mesh_position_packet, decode_mesh_text_packet + +__all__ = [ + "decode_mesh_packet", + "decode_mesh_position_packet", + "decode_mesh_text_packet", +] diff --git a/mesh/encoder.py b/mesh/encoder.py new file mode 100644 index 0000000..aa2578d --- /dev/null +++ b/mesh/encoder.py @@ -0,0 +1,8 @@ +from mesh.mesh_encoder import build_waypoint_fields, build_waypoint_id, encode_position_payload, encode_text_payload + +__all__ = [ + "build_waypoint_fields", + "build_waypoint_id", + "encode_position_payload", + "encode_text_payload", +] diff --git a/mesh/filter.py b/mesh/filter.py new file mode 100644 index 0000000..1ab73c5 --- /dev/null +++ b/mesh/filter.py @@ -0,0 +1,6 @@ +from mesh.mesh_filter import MeshFilter, MeshFilterDecision + +__all__ = [ + "MeshFilter", + "MeshFilterDecision", +] diff --git a/mesh/mesh_decoder.py b/mesh/mesh_decoder.py new file mode 100644 index 0000000..b64ca02 --- /dev/null +++ b/mesh/mesh_decoder.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +"""Legacy Meshtastic decoder implementation behind mesh.decoder.""" + +import re +from typing import Any + +from core.model import MessageEvent, PositionEvent, Source +from mesh.parser import canonical_to_event, parse_meshtastic_packet + + +APRS_TEXT_RE = re.compile( + r"^\[APRS (?P.+?)(?:->(?P.+?))?(?: #(?P[^\]]+))?\] (?P.*)$" +) + + +def decode_mesh_packet( + packet: dict[str, Any], + *, + source_id: str, + target_id: str | None = None, +) -> PositionEvent | MessageEvent | None: + canonical = parse_meshtastic_packet(packet) + decoded = packet.get("decoded") or {} + portnum = canonical.get("comms", {}).get("portnum") + + if "position" in decoded or portnum == "POSITION_APP": + native_position = decode_mesh_position_packet(packet, source_id=source_id, canonical=canonical) + if native_position is not None: + return native_position + + text_event = decode_mesh_text_packet( + packet, + source_id=source_id, + target_id=target_id, + ) + if text_event is not None: + return text_event + + return canonical_to_event( + canonical, + source_id=source_id, + target_id=target_id, + raw=packet, + ) + + +def decode_mesh_text_packet( + packet: dict[str, Any], + *, + source_id: str, + target_id: str | None = None, +) -> PositionEvent | MessageEvent | None: + decoded = packet.get("decoded") or {} + text = decoded.get("text") + if not text: + return None + return decode_mesh_text_payload(str(text).strip(), source_id=source_id, target_id=target_id, raw=packet) + + +def decode_mesh_text_payload( + text: str, + *, + source_id: str, + target_id: str | None = None, + raw: dict[str, Any] | None = None, +) -> PositionEvent | MessageEvent | None: + if not text: + return None + + if text.startswith("APRS_POS|"): + values = _parse_pipe_payload(text) + event_id = values.get("id") or source_id + lat = _parse_float(values.get("lat")) + lon = _parse_float(values.get("lon")) + if lat is None or lon is None: + return None + return PositionEvent( + id=event_id, + entity_kind=values.get("entity") or ("object" if values.get("object") else "station"), + type="object" if values.get("entity") == "object" or values.get("object") else "position", + lat=lat, + lon=lon, + tactical_name=values.get("name") or None, + message=values.get("msg") or None, + object_name=values.get("object") or None, + owner_id=values.get("owner") or None, + symbol_table=values.get("symbol_table") or None, + symbol_code=values.get("symbol_code") or None, + source=Source.MESHTASTIC, + raw=raw or {"mesh_text": text}, + ) + + aprs_match = APRS_TEXT_RE.match(text) + if aprs_match: + event_source = (aprs_match.group("source") or source_id).strip() + event_target = (aprs_match.group("target") or target_id or "").strip() or None + event_message = (aprs_match.group("message") or "").strip() + return MessageEvent( + id=event_source, + target=event_target, + message=event_message, + message_id=(aprs_match.group("message_id") or "").strip() or None, + source=Source.MESHTASTIC, + raw=raw or {"mesh_text": text}, + ) + + return MessageEvent( + id=source_id, + target=target_id, + message=text, + source=Source.MESHTASTIC, + raw=raw or {"mesh_text": text}, + ) + + +def decode_mesh_position_packet( + packet: dict[str, Any], + *, + source_id: str, + canonical: dict[str, Any] | None = None, +) -> PositionEvent | None: + position = (canonical or {}).get("position", {}) + if not position: + canonical = canonical or parse_meshtastic_packet(packet) + position = canonical.get("position", {}) + + lat = position.get("lat") + lon = position.get("lon") + if lat is None or lon is None: + return None + return PositionEvent( + id=source_id, + entity_kind="station", + type="position", + lat=float(lat), + lon=float(lon), + alt=position.get("alt_m"), + source=Source.MESHTASTIC, + raw=packet, + ) + + +def _parse_pipe_payload(text: str) -> dict[str, str]: + values: dict[str, str] = {} + for part in text.split("|")[1:]: + if "=" not in part: + continue + key, value = part.split("=", 1) + values[key.strip()] = value.strip() + return values + + +def _parse_float(value: str | None) -> float | None: + if value is None or value == "": + return None + try: + return float(value) + except ValueError: + return None diff --git a/mesh/mesh_encoder.py b/mesh/mesh_encoder.py new file mode 100644 index 0000000..b1bd684 --- /dev/null +++ b/mesh/mesh_encoder.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +"""Legacy Meshtastic payload encoder implementation behind mesh.encoder.""" + +from core.model import MessageEvent, PositionEvent + + +def encode_position_payload(event: PositionEvent) -> str: + parts = [ + "APRS_POS", + f"entity={event.entity_kind}", + f"id={event.id}", + f"name={event.tactical_name or ''}", + f"lat={event.lat}", + f"lon={event.lon}", + f"ts={event.timestamp}", + ] + if event.object_name: + parts.append(f"object={event.object_name}") + if event.owner_id: + parts.append(f"owner={event.owner_id}") + if event.symbol_table: + parts.append(f"symbol_table={event.symbol_table}") + if event.symbol_code: + parts.append(f"symbol_code={event.symbol_code}") + if event.message: + parts.append(f"msg={event.message}") + return "|".join(parts) + + +def encode_text_payload(event: MessageEvent) -> str: + if event.target: + if event.message_id: + return f"[APRS {event.id}->{event.target} #{event.message_id}] {event.message}" + return f"[APRS {event.id}->{event.target}] {event.message}" + + if event.message_id: + return f"[APRS {event.id} #{event.message_id}] {event.message}" + return f"[APRS {event.id}] {event.message}" + + +def build_waypoint_fields(event: PositionEvent) -> tuple[str, str]: + label_source = event.object_name or event.tactical_name or event.id + label = label_source.strip()[:30] + if event.is_object(): + description_parts = [f"APRS object {event.object_name or event.id}"] + if event.owner_id: + description_parts.append(f"owner {event.owner_id}") + else: + description_parts = [f"APRS station {event.id}"] + if event.message: + description_parts.append(event.message.strip()) + return label, " | ".join(description_parts)[:120] + + +def build_waypoint_id(source_id: str) -> int: + value = 0 + for char in source_id.upper(): + value = ((value * 33) + ord(char)) % 1_000_000_000 + return value or 1 diff --git a/mesh/mesh_filter.py b/mesh/mesh_filter.py new file mode 100644 index 0000000..8df9ff8 --- /dev/null +++ b/mesh/mesh_filter.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import math +from dataclasses import dataclass, field + +from core.model import MessageEvent, PositionEvent + + +@dataclass(slots=True) +class MeshFilterDecision: + allowed: bool + reason: str + + +@dataclass(slots=True) +class MeshFilter: + min_position_interval_seconds: int = 180 + min_position_distance_meters: float = 75.0 + last_positions: dict[str, tuple[float, float, int]] = field(default_factory=dict) + last_objects: dict[str, tuple[str | None, float | None, float | None]] = field(default_factory=dict) + last_status: dict[str, str] = field(default_factory=dict) + + def allow_position(self, event: PositionEvent) -> MeshFilterDecision: + uid = event.object_name or event.id + if event.lat is None or event.lon is None: + return MeshFilterDecision(False, "missing-coordinates") + + if event.is_object(): + previous = self.last_objects.get(uid) + current = (event.tactical_name, event.lat, event.lon) + if previous is None: + self.last_objects[uid] = current + return MeshFilterDecision(True, "new-object") + if previous != current: + self.last_objects[uid] = current + return MeshFilterDecision(True, "object-changed") + return MeshFilterDecision(False, "object-refresh") + + previous = self.last_positions.get(uid) + if previous is None: + self.last_positions[uid] = (event.lat, event.lon, event.timestamp) + return MeshFilterDecision(True, "new-station") + + prev_lat, prev_lon, prev_ts = previous + time_delta = event.timestamp - prev_ts + distance = _haversine_meters(prev_lat, prev_lon, event.lat, event.lon) + if time_delta >= self.min_position_interval_seconds: + self.last_positions[uid] = (event.lat, event.lon, event.timestamp) + return MeshFilterDecision(True, "time-threshold") + if distance >= self.min_position_distance_meters: + self.last_positions[uid] = (event.lat, event.lon, event.timestamp) + return MeshFilterDecision(True, "distance-threshold") + return MeshFilterDecision(False, "position-throttled") + + def allow_message(self, event: MessageEvent) -> MeshFilterDecision: + return MeshFilterDecision(True, "message-pass") + + +def _haversine_meters(lat1: float, lon1: float, lat2: float, lon2: float) -> float: + radius = 6_371_000.0 + phi1 = math.radians(lat1) + phi2 = math.radians(lat2) + d_phi = math.radians(lat2 - lat1) + d_lambda = math.radians(lon2 - lon1) + a = ( + math.sin(d_phi / 2) ** 2 + + math.cos(phi1) * math.cos(phi2) * math.sin(d_lambda / 2) ** 2 + ) + c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a)) + return radius * c diff --git a/sartrack/__init__.py b/sartrack/__init__.py new file mode 100644 index 0000000..808c805 --- /dev/null +++ b/sartrack/__init__.py @@ -0,0 +1,20 @@ +from sartrack.adapter import SARTrackAPRSServerClient, SARTrackAPRSServerConfig, SARTrackTCPConfig, SARTrackTCPServer +from sartrack.parser import ( + SARTrackParseError, + canonical_to_sartrack_event, + parse_line, + parse_sartrack_canonical, + parse_sartrack_event, +) + +__all__ = [ + "SARTrackAPRSServerClient", + "SARTrackAPRSServerConfig", + "SARTrackParseError", + "SARTrackTCPConfig", + "SARTrackTCPServer", + "canonical_to_sartrack_event", + "parse_line", + "parse_sartrack_canonical", + "parse_sartrack_event", +] diff --git a/sartrack/adapter.py b/sartrack/adapter.py new file mode 100644 index 0000000..a76ccee --- /dev/null +++ b/sartrack/adapter.py @@ -0,0 +1,9 @@ +from sartrack.aprs_server_client import SARTrackAPRSServerClient, SARTrackAPRSServerConfig +from sartrack.tcp_server import SARTrackTCPConfig, SARTrackTCPServer + +__all__ = [ + "SARTrackAPRSServerClient", + "SARTrackAPRSServerConfig", + "SARTrackTCPConfig", + "SARTrackTCPServer", +] diff --git a/sartrack/aprs_server_client.py b/sartrack/aprs_server_client.py index 31eb155..8a9b291 100644 --- a/sartrack/aprs_server_client.py +++ b/sartrack/aprs_server_client.py @@ -4,7 +4,8 @@ import logging from dataclasses import dataclass -from aprs.parser import APRSParseError, parse_aprs_frame +from aprs.parser import APRSParseError, parse_aprs_event +from core.model import Source from core.router import EventRouter from core.state import StateStore @@ -124,14 +125,16 @@ async def _read_loop(self, router: EventRouter) -> None: continue try: - event = parse_aprs_frame(raw) + event = parse_aprs_event( + raw, + source=Source.SARTRACK, + ingress_adapter="sartrack_aprs", + raw_context={"sartrack_aprs_line": raw}, + ) except APRSParseError as exc: self.log.warning("RX | unsupported | adapter=sartrack.aprs error=%s", exc) continue - if isinstance(event.raw, dict): - event.raw["ingress_adapter"] = "sartrack_aprs" - event.raw["sartrack_aprs_line"] = raw await router.publish(event) diff --git a/sartrack/parser.py b/sartrack/parser.py index 043ef47..d38614f 100644 --- a/sartrack/parser.py +++ b/sartrack/parser.py @@ -14,14 +14,25 @@ class SARTrackParseError(ValueError): pass -def parse_line(line: str) -> PositionEvent | MessageEvent | None: +def parse_sartrack_event( + line: str, + *, + source: Source = Source.SARTRACK, + ingress_adapter: str | None = None, + raw_context: dict[str, object] | None = None, +) -> PositionEvent | MessageEvent | None: raw = line.strip() if not raw: return None try: canonical = parse_sartrack_canonical(raw) - event = canonical_to_sartrack_event(canonical) + event = canonical_to_sartrack_event( + canonical, + source=source, + ingress_adapter=ingress_adapter, + raw_context=raw_context, + ) log.debug( "SARTrack canonical -> %s source=%s subtype=%s target=%r", event.kind, @@ -35,6 +46,10 @@ def parse_line(line: str) -> PositionEvent | MessageEvent | None: return None +def parse_line(line: str) -> PositionEvent | MessageEvent | None: + return parse_sartrack_event(line) + + def parse_sartrack_canonical(line: str) -> dict[str, object]: upper = line.upper() @@ -188,7 +203,13 @@ def parse_sartrack_canonical(line: str) -> dict[str, object]: raise SARTrackParseError(f"unknown record type in line: {line}") -def canonical_to_sartrack_event(canonical: dict[str, object]) -> PositionEvent | MessageEvent: +def canonical_to_sartrack_event( + canonical: dict[str, object], + *, + source: Source = Source.SARTRACK, + ingress_adapter: str | None = None, + raw_context: dict[str, object] | None = None, +) -> PositionEvent | MessageEvent: identity = canonical["identity"] position = canonical["position"] text = canonical["text"] @@ -196,13 +217,19 @@ def canonical_to_sartrack_event(canonical: dict[str, object]) -> PositionEvent | if not source_id: raise SARTrackParseError("canonical SARTrack event missing source id") + raw_payload: dict[str, object] = {"canonical": canonical} + if raw_context: + raw_payload.update(raw_context) + if ingress_adapter: + raw_payload["ingress_adapter"] = ingress_adapter + if text["text"]: return MessageEvent( id=str(source_id), message=str(text["text"]), target=str(canonical["sartrack"]["receiver"]) if canonical["sartrack"]["receiver"] else None, - source=Source.SARTRACK, - raw={"canonical": canonical}, + source=source, + raw=raw_payload, ) if position["lat"] is None or position["lon"] is None: @@ -213,8 +240,8 @@ def canonical_to_sartrack_event(canonical: dict[str, object]) -> PositionEvent | lat=float(position["lat"]), lon=float(position["lon"]), message=str(text["comment"]) if text["comment"] else None, - source=Source.SARTRACK, - raw={"canonical": canonical}, + source=source, + raw=raw_payload, ) diff --git a/sartrack/tcp_server.py b/sartrack/tcp_server.py index 348bfc5..0239185 100644 --- a/sartrack/tcp_server.py +++ b/sartrack/tcp_server.py @@ -6,7 +6,7 @@ from core.router import EventRouter from core.state import StateStore -from sartrack.parser import parse_line +from sartrack.parser import parse_sartrack_event @dataclass(slots=True) @@ -127,11 +127,13 @@ async def _handle_client( if not raw: continue - event = parse_line(raw) + event = parse_sartrack_event( + raw, + ingress_adapter="sartrack_tcp", + raw_context={"sartrack_tcp_peer": str(peer)}, + ) if event is None: continue - if isinstance(event.raw, dict): - event.raw["ingress_adapter"] = "sartrack_tcp" await router.publish(event) except (ConnectionResetError, BrokenPipeError) as exc: diff --git a/tests/test_cot_mesh_adapters.py b/tests/test_cot_mesh_adapters.py new file mode 100644 index 0000000..e9a9781 --- /dev/null +++ b/tests/test_cot_mesh_adapters.py @@ -0,0 +1,174 @@ +from __future__ import annotations + +import unittest + +from cot.cot_encoder import encode_event_to_cot_xml +from cot.cot_parser import canonical_to_cot_event, parse_cot_canonical, parse_cot_xml +from core.model import MessageEvent, PositionEvent, Source +from mesh.mesh_decoder import decode_mesh_packet, decode_mesh_text_payload +from mesh.mesh_encoder import encode_position_payload, encode_text_payload +from mesh.mesh_filter import MeshFilter + + +class MeshDecoderTests(unittest.TestCase): + def test_decode_native_mesh_position_packet(self) -> None: + packet = { + "fromId": "!abcd1234", + "decoded": { + "portnum": "POSITION_APP", + "position": { + "latitudeI": 500647000, + "longitudeI": 199450000, + "altitude": 200, + }, + }, + } + + decoded = decode_mesh_packet(packet, source_id="N0CALL-10") + + self.assertIsInstance(decoded, PositionEvent) + assert isinstance(decoded, PositionEvent) + self.assertEqual(decoded.id, "N0CALL-10") + self.assertEqual(decoded.entity_kind, "station") + self.assertAlmostEqual(decoded.lat or 0.0, 50.0647, places=4) + self.assertAlmostEqual(decoded.lon or 0.0, 19.9450, places=4) + self.assertEqual(decoded.alt, 200) + + def test_roundtrip_object_position_payload(self) -> None: + event = PositionEvent( + id="OBJ1", + entity_kind="object", + type="object", + lat=50.1, + lon=19.1, + tactical_name="pin-1", + message="marker", + object_name="OBJ1", + owner_id="N0CALL-10", + source=Source.SARTRACK, + ) + + payload = encode_position_payload(event) + decoded = decode_mesh_text_payload(payload, source_id="N0CALL-10") + + self.assertIsInstance(decoded, PositionEvent) + assert isinstance(decoded, PositionEvent) + self.assertEqual(decoded.entity_kind, "object") + self.assertEqual(decoded.type, "object") + self.assertEqual(decoded.object_name, "OBJ1") + self.assertEqual(decoded.owner_id, "N0CALL-10") + self.assertEqual(decoded.tactical_name, "pin-1") + self.assertEqual(decoded.message, "marker") + + def test_roundtrip_aprs_text_message_payload(self) -> None: + event = MessageEvent( + id="N0CALL-10", + target="SP9PET-10", + message="test", + message_id="01", + source=Source.SARTRACK, + ) + + payload = encode_text_payload(event) + decoded = decode_mesh_text_payload(payload, source_id="fallback") + + self.assertIsInstance(decoded, MessageEvent) + assert isinstance(decoded, MessageEvent) + self.assertEqual(decoded.id, "N0CALL-10") + self.assertEqual(decoded.target, "SP9PET-10") + self.assertEqual(decoded.message, "test") + self.assertEqual(decoded.message_id, "01") + + +class MeshFilterTests(unittest.TestCase): + def test_station_position_is_throttled_when_unchanged(self) -> None: + filt = MeshFilter(min_position_interval_seconds=180, min_position_distance_meters=75.0) + event = PositionEvent( + id="N0CALL-10", + entity_kind="station", + type="position", + lat=50.0, + lon=19.0, + timestamp=1000, + source=Source.SARTRACK, + ) + + first = filt.allow_position(event) + second = filt.allow_position(event) + + self.assertTrue(first.allowed) + self.assertEqual(first.reason, "new-station") + self.assertFalse(second.allowed) + self.assertEqual(second.reason, "position-throttled") + + def test_object_refresh_is_filtered(self) -> None: + filt = MeshFilter() + event = PositionEvent( + id="OBJ1", + entity_kind="object", + type="object", + lat=50.1, + lon=19.1, + tactical_name="pin-1", + object_name="OBJ1", + owner_id="N0CALL-10", + source=Source.SARTRACK, + ) + + first = filt.allow_position(event) + second = filt.allow_position(event) + + self.assertTrue(first.allowed) + self.assertEqual(first.reason, "new-object") + self.assertFalse(second.allowed) + self.assertEqual(second.reason, "object-refresh") + + +class CoTParserTests(unittest.TestCase): + def test_cot_canonical_roundtrip_object(self) -> None: + xml = ( + '' + '' + 'APRS object OBJ1 | owner N0CALL-10 | marker' + "" + ) + + canonical = parse_cot_canonical(xml) + decoded = canonical_to_cot_event(canonical) + + self.assertEqual(canonical["meta"]["source_format"], "cot_xml") + self.assertEqual(canonical["meta"]["source_subtype"], "object") + self.assertEqual(decoded.entity_kind, "object") + self.assertEqual(decoded.object_name, "OBJ1") + self.assertEqual(decoded.owner_id, "N0CALL-10") + self.assertEqual(decoded.message, "marker") + self.assertEqual(decoded.alt, 123.0) + + def test_roundtrip_object_cot(self) -> None: + event = PositionEvent( + id="OBJ1", + entity_kind="object", + type="object", + lat=50.1, + lon=19.1, + tactical_name="pin-1", + message="marker", + object_name="OBJ1", + owner_id="N0CALL-10", + source=Source.SARTRACK, + ) + + xml = encode_event_to_cot_xml(event) + decoded = parse_cot_xml(xml) + + self.assertEqual(decoded.entity_kind, "object") + self.assertEqual(decoded.type, "object") + self.assertEqual(decoded.id, "OBJ1") + self.assertEqual(decoded.object_name, "OBJ1") + self.assertEqual(decoded.owner_id, "N0CALL-10") + self.assertEqual(decoded.message, "marker") + + +if __name__ == "__main__": + unittest.main() From 7c00ad7ef70bba0e942877d5cacabe6c40fae3dd Mon Sep 17 00:00:00 2001 From: DjGreen Date: Thu, 16 Apr 2026 03:00:15 +0200 Subject: [PATCH 2/6] Refactor adapter structure to use one consistent RX/TX layout and keep model.py as the single runtime event language. Changes: - unified RX entrypoints so each protocol now uses a single parser module - unified TX entrypoints so each protocol now uses a single encoder module - removed legacy transitional files: - aprs/generator.py - cot/cot_parser.py - cot/cot_encoder.py - mesh/decoder.py - mesh/mesh_decoder.py - mesh/mesh_encoder.py - aprs/decoder.py - sartrack/decoder.py - cot/decoder.py - updated imports across adapters, tests and package exports - verified TX paths do not build outbound payloads from raw RX data - removed raw propagation from TX-side event clones where it was unnecessary - improved docstrings/comments around model, state, parser and GUI layers - unified operator-facing event naming in GUI/state store - updated README to match the current repository structure and event pipeline Architecture after this change: - model.py is the single runtime language for router/adapters - canonical.py remains a normalization helper for parser/encoder layers - parser.py handles RX normalization - encoder.py handles TX shaping - adapter.py handles transport/session logic Validation: - unittest suite passes (24 tests) - py_compile checks pass --- .../pytak-7.3.0.dist-info/INSTALLER | 1 + .../pytak-7.3.0.dist-info/METADATA | 76 ++ .../pytak-7.3.0.dist-info/RECORD | 26 + .../pytak-7.3.0.dist-info/REQUESTED | 0 .../site-packages/pytak-7.3.0.dist-info/WHEEL | 5 + .../pytak-7.3.0.dist-info/licenses/LICENSE | 202 ++++ .../pytak-7.3.0.dist-info/top_level.txt | 1 + .venv/Lib/site-packages/pytak/VERSION | 1 + .venv/Lib/site-packages/pytak/__init__.py | 85 ++ .../Lib/site-packages/pytak/asyncio_dgram.py | 342 +++++++ .venv/Lib/site-packages/pytak/classes.py | 660 +++++++++++++ .../site-packages/pytak/client_functions.py | 588 +++++++++++ .venv/Lib/site-packages/pytak/commands.py | 30 + .venv/Lib/site-packages/pytak/constants.py | 111 +++ .../Lib/site-packages/pytak/crypto_classes.py | 927 ++++++++++++++++++ .../site-packages/pytak/crypto_functions.py | 152 +++ .venv/Lib/site-packages/pytak/functions.py | 348 +++++++ README.md | 34 +- aprs/__init__.py | 3 +- aprs/agwpe.py | 12 +- aprs/agwpe_server.py | 12 +- aprs/encoder.py | 111 ++- aprs/generator.py | 110 --- aprs/is_server.py | 2 +- aprs/parser.py | 22 +- config.py | 16 +- core/canonical.py | 6 + core/model.py | 25 + core/state.py | 64 ++ cot/__init__.py | 3 +- cot/cot.py | 171 +++- cot/cot_encoder.py | 82 -- cot/cot_parser.py | 133 --- cot/encoder.py | 118 ++- cot/parser.py | 152 ++- gui/server.py | 16 +- main.py | 2 +- mesh/__init__.py | 14 +- mesh/adapter.py | 5 +- mesh/decoder.py | 7 - mesh/encoder.py | 61 +- mesh/mesh_decoder.py | 160 --- mesh/mesh_encoder.py | 60 -- mesh/parser.py | 172 +++- requirements.txt | 1 + sartrack/__init__.py | 2 + sartrack/parser.py | 20 +- tests/test_cot_adapter.py | 93 ++ tests/test_cot_mesh_adapters.py | 43 +- 49 files changed, 4641 insertions(+), 646 deletions(-) create mode 100644 .venv/Lib/site-packages/pytak-7.3.0.dist-info/INSTALLER create mode 100644 .venv/Lib/site-packages/pytak-7.3.0.dist-info/METADATA create mode 100644 .venv/Lib/site-packages/pytak-7.3.0.dist-info/RECORD create mode 100644 .venv/Lib/site-packages/pytak-7.3.0.dist-info/REQUESTED create mode 100644 .venv/Lib/site-packages/pytak-7.3.0.dist-info/WHEEL create mode 100644 .venv/Lib/site-packages/pytak-7.3.0.dist-info/licenses/LICENSE create mode 100644 .venv/Lib/site-packages/pytak-7.3.0.dist-info/top_level.txt create mode 100644 .venv/Lib/site-packages/pytak/VERSION create mode 100644 .venv/Lib/site-packages/pytak/__init__.py create mode 100644 .venv/Lib/site-packages/pytak/asyncio_dgram.py create mode 100644 .venv/Lib/site-packages/pytak/classes.py create mode 100644 .venv/Lib/site-packages/pytak/client_functions.py create mode 100644 .venv/Lib/site-packages/pytak/commands.py create mode 100644 .venv/Lib/site-packages/pytak/constants.py create mode 100644 .venv/Lib/site-packages/pytak/crypto_classes.py create mode 100644 .venv/Lib/site-packages/pytak/crypto_functions.py create mode 100644 .venv/Lib/site-packages/pytak/functions.py delete mode 100644 aprs/generator.py delete mode 100644 cot/cot_encoder.py delete mode 100644 cot/cot_parser.py delete mode 100644 mesh/decoder.py delete mode 100644 mesh/mesh_decoder.py delete mode 100644 mesh/mesh_encoder.py create mode 100644 tests/test_cot_adapter.py diff --git a/.venv/Lib/site-packages/pytak-7.3.0.dist-info/INSTALLER b/.venv/Lib/site-packages/pytak-7.3.0.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/.venv/Lib/site-packages/pytak-7.3.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/.venv/Lib/site-packages/pytak-7.3.0.dist-info/METADATA b/.venv/Lib/site-packages/pytak-7.3.0.dist-info/METADATA new file mode 100644 index 0000000..0080be7 --- /dev/null +++ b/.venv/Lib/site-packages/pytak-7.3.0.dist-info/METADATA @@ -0,0 +1,76 @@ +Metadata-Version: 2.4 +Name: pytak +Version: 7.3.0 +Summary: PyTAK is a Python package for rapid TAK integration. +Home-page: https://github.com/snstac/pytak +Author-email: Greg Albrecht +Maintainer: Greg Albrecht +Maintainer-email: oss@undef.net +License: Apache 2.0 +Keywords: Cursor on Target,CoT,ATAK,TAK,WinTAK,TAK,TAK Server +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Intended Audience :: Developers +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Development Status :: 5 - Production/Stable +Classifier: Operating System :: POSIX +Classifier: Operating System :: MacOS :: MacOS X +Classifier: Operating System :: Microsoft :: Windows +Classifier: Operating System :: OS Independent +Requires-Python: <4,>=3.6 +Description-Content-Type: text/markdown +License-File: LICENSE +Provides-Extra: with-crypto +Requires-Dist: cryptography>=39.0.0; extra == "with-crypto" +Provides-Extra: with-takproto +Requires-Dist: takproto>=2.0.0; extra == "with-takproto" +Provides-Extra: with-aiohttp +Requires-Dist: aiohttp>=3.8.0; extra == "with-aiohttp" +Provides-Extra: test +Requires-Dist: pytest-asyncio; extra == "test" +Requires-Dist: pytest-cov; extra == "test" +Requires-Dist: pylint; extra == "test" +Requires-Dist: flake8; extra == "test" +Requires-Dist: black; extra == "test" +Requires-Dist: cryptography; extra == "test" +Requires-Dist: aiohttp; extra == "test" +Requires-Dist: takproto; extra == "test" +Dynamic: license-file + +![ATAK Screenshot with PyTAK Logo.](https://pytak.readthedocs.io/en/stable/media/atak_screenshot_with_pytak_logo-x25.jpg) + +# Python Team Awareness Kit + +PyTAK is a Python Module for creating Team Awareness Kit ([TAK](https://tak.gov)) clients, servers & gateways. + +## Features + +- **TAK Protocol Support**: Connect with ATAK, WinTAK, iTAK, and TAK Server. +- **Data Handling**: Manage TAK, Cursor on Target (CoT), and non-CoT data. +- **Data Parsing and Serialization**: Parse and serialize TAK and CoT data. +- **Network Communication**: Send and receive TAK and CoT data over a network. +- **No External Dependencies**: Batteries-included Pythonic architecture requires no external libraries or modules. + +## Documentation + +See [PyTAK documentation](https://pytak.rtfd.io/) for instructions on getting +started with PyTAK, examples, configuration & troubleshooting options. + +## License & Copyright + +Copyright Sensors & Signals LLC https://www.snstac.com + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +asyncio_dgram is Copyright (c) 2019 Justin Bronder and is licensed under the MIT +License, see pytak/asyncio_dgram/LICENSE for details. + diff --git a/.venv/Lib/site-packages/pytak-7.3.0.dist-info/RECORD b/.venv/Lib/site-packages/pytak-7.3.0.dist-info/RECORD new file mode 100644 index 0000000..9fea08a --- /dev/null +++ b/.venv/Lib/site-packages/pytak-7.3.0.dist-info/RECORD @@ -0,0 +1,26 @@ +pytak-7.3.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +pytak-7.3.0.dist-info/METADATA,sha256=cZSuhFeaeOqNfdPViERZ8c2XBpdQtGt5Rx3RAb8R2Wg,3181 +pytak-7.3.0.dist-info/RECORD,, +pytak-7.3.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +pytak-7.3.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91 +pytak-7.3.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358 +pytak-7.3.0.dist-info/top_level.txt,sha256=N81yEQruCBBn_NPDQtqlaZe03b9QThW9rtEZ5lheUMs,6 +pytak/VERSION,sha256=nnkGTHSaN7LDNpZiPxMrkcKbXpQUE9v7E_p4UeKlMUM,6 +pytak/__init__.py,sha256=oEoYg-ehjCm1JtvBpVfwhu9PzH3WtQQoF-bgFoifXss,2057 +pytak/__pycache__/__init__.cpython-314.pyc,, +pytak/__pycache__/asyncio_dgram.cpython-314.pyc,, +pytak/__pycache__/classes.cpython-314.pyc,, +pytak/__pycache__/client_functions.cpython-314.pyc,, +pytak/__pycache__/commands.cpython-314.pyc,, +pytak/__pycache__/constants.cpython-314.pyc,, +pytak/__pycache__/crypto_classes.cpython-314.pyc,, +pytak/__pycache__/crypto_functions.cpython-314.pyc,, +pytak/__pycache__/functions.cpython-314.pyc,, +pytak/asyncio_dgram.py,sha256=XSnT_Kvm68gtg87fYVpvRRm1YFrclil-aJGcSkNJxm4,11084 +pytak/classes.py,sha256=Cffs1SQofYaXnOd0ov4T_2CL7E31MSg7blqXJKndl0Y,22899 +pytak/client_functions.py,sha256=FBNMIhHJEonTIfZaV0lJu1D7cl5acv9-AFCeMht6brU,18642 +pytak/commands.py,sha256=8DNfeSLaRjAJdSz37gU0Eexpw9MWwoMVaF8yQVmjMzQ,921 +pytak/constants.py,sha256=QpDpbdkVQEwSR2kx9PNvGwJdH6xUJWb6p4IKWWq6vPE,3785 +pytak/crypto_classes.py,sha256=k0VGEI0ZWVnFowOjp74NiUIy15fPvfn5jm5SwYnEEWE,34674 +pytak/crypto_functions.py,sha256=lwkE8ngoXl9BdBrJicW1UA5E4g3oi6StZw6FqmjMjR8,4867 +pytak/functions.py,sha256=c07APuh5NUfmsrSY3293L8QAPkPOI-tWjt2tOCNB9FM,11451 diff --git a/.venv/Lib/site-packages/pytak-7.3.0.dist-info/REQUESTED b/.venv/Lib/site-packages/pytak-7.3.0.dist-info/REQUESTED new file mode 100644 index 0000000..e69de29 diff --git a/.venv/Lib/site-packages/pytak-7.3.0.dist-info/WHEEL b/.venv/Lib/site-packages/pytak-7.3.0.dist-info/WHEEL new file mode 100644 index 0000000..14a883f --- /dev/null +++ b/.venv/Lib/site-packages/pytak-7.3.0.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: setuptools (82.0.1) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/.venv/Lib/site-packages/pytak-7.3.0.dist-info/licenses/LICENSE b/.venv/Lib/site-packages/pytak-7.3.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/.venv/Lib/site-packages/pytak-7.3.0.dist-info/licenses/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/.venv/Lib/site-packages/pytak-7.3.0.dist-info/top_level.txt b/.venv/Lib/site-packages/pytak-7.3.0.dist-info/top_level.txt new file mode 100644 index 0000000..1d1ac78 --- /dev/null +++ b/.venv/Lib/site-packages/pytak-7.3.0.dist-info/top_level.txt @@ -0,0 +1 @@ +pytak diff --git a/.venv/Lib/site-packages/pytak/VERSION b/.venv/Lib/site-packages/pytak/VERSION new file mode 100644 index 0000000..1502020 --- /dev/null +++ b/.venv/Lib/site-packages/pytak/VERSION @@ -0,0 +1 @@ +7.3.0 diff --git a/.venv/Lib/site-packages/pytak/__init__.py b/.venv/Lib/site-packages/pytak/__init__.py new file mode 100644 index 0000000..0325f4e --- /dev/null +++ b/.venv/Lib/site-packages/pytak/__init__.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# __init__.py from https://github.com/snstac/pytak +# +# Copyright Sensors & Signals LLC https://www.snstac.com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Python Team Awareness Kit (PyTAK) Module.""" + +from .constants import ( # NOQA + LOG_LEVEL, + LOG_FORMAT, + DEFAULT_COT_PORT, + DEFAULT_BACKOFF, + DEFAULT_SLEEP, + DEFAULT_ATAK_PORT, + DEFAULT_BROADCAST_PORT, + DEFAULT_COT_STALE, + DEFAULT_FIPS_CIPHERS, + W3C_XML_DATETIME, + DEFAULT_TC_TOKEN_URL, + DEFAULT_COT_URL, + DEFAULT_TLS_PARAMS_OPT, + DEFAULT_TLS_PARAMS_REQ, + DEFAULT_HOST_ID, + BOOLEAN_TRUTH, + DEFAULT_XML_DECLARATION, + DEFAULT_IMPORT_OTHER_CONFIGS, + DEFAULT_TAK_PROTO, + DEFAULT_PYTAK_MULTICAST_LOCAL_ADDR, + DEFAULT_COT_ACCESS, + DEFAULT_COT_CAVEAT, + DEFAULT_COT_RELTO, + DEFAULT_COT_QOS, + DEFAULT_COT_OPEX, + DEFAULT_COT_VAL, + DEFAULT_MAX_OUT_QUEUE, + DEFAULT_MAX_IN_QUEUE, + ISO_8601_UTC, + DEFAULT_TLS_ENROLLMENT_CERT_PASSPHRASE_LENGTH +) + +from .classes import ( # NOQA + Worker, + TXWorker, + RXWorker, + QueueWorker, + CLITool, + SimpleCOTEvent, + COTEvent, + TAKDataPackage +) + +from .functions import ( # NOQA + split_host, + parse_url, + hello_event, + cot_time, + gen_cot, + gen_cot_xml, + cot2xml, + enroll_tak, + decode_response +) + +from .client_functions import ( # NOQA + create_udp_client, + protocol_factory, + txworker_factory, + rxworker_factory, + cli, + read_pref_package, +) + +from . import asyncio_dgram # NOQA diff --git a/.venv/Lib/site-packages/pytak/asyncio_dgram.py b/.venv/Lib/site-packages/pytak/asyncio_dgram.py new file mode 100644 index 0000000..71aa3d1 --- /dev/null +++ b/.venv/Lib/site-packages/pytak/asyncio_dgram.py @@ -0,0 +1,342 @@ +# MIT License +# +# Copyright (c) 2019 Justin Bronder +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import sys +import asyncio +import pathlib +import socket +import warnings + +__all__ = ("TransportClosed", "bind", "connect", "from_socket", "DatagramClient") + + +class TransportClosed(Exception): + """ + Raised when the asyncio.DatagramTransport underlying a DatagramStream is + closed. + """ + + +class DatagramStream: + """ + Representation of a Datagram socket attached via either bind() or + connect() returned to consumers of this module. Provides simple + wrappers around sending and receiving bytes. + + Due to the stateless nature of datagram protocols, errors are not + immediately available to this class at the point an action was performed + that will generate it. Rather, successive calls will raise exceptions if + there are any. Checking for exceptions can be done explicitly by using the + exception property. + + For instance, failure to connect to a remote endpoint will not be noticed + until some point in time later, at which point ConnectionRefused will be + raised. + """ + + def __init__(self, transport, recvq, excq, drained): + """ + @param transport - asyncio transport + @param recvq - asyncio queue that gets populated by the + DatagramProtocol with received datagrams. + @param excq - asyncio queue that gets populated with any errors + detected by the DatagramProtocol. + @param drained - asyncio event that is unset when writing is + paused and set otherwise. + """ + self._transport = transport + self._recvq = recvq + self._excq = excq + self._drained = drained + + def __del__(self): + self._transport.close() + + @property + def exception(self): + """ + If the underlying protocol detected an error, raise the first + unconsumed exception it noticed, otherwise returns None. + """ + try: + exc = self._excq.get_nowait() + raise exc + except asyncio.queues.QueueEmpty: + pass + + @property + def sockname(self): + """ + The associated socket's own address + """ + r = self._transport.get_extra_info("sockname") + return None if r == "" else r + + @property + def peername(self): + """ + The address the associated socket is connected to + """ + r = self._transport.get_extra_info("peername") + return None if r == "" else r + + @property + def socket(self): + """ + The socket instance used by the stream. In python <3.8 this is a + socket.socket instance, after it is an asyncio.TransportSocket + instance. + """ + return self._transport.get_extra_info("socket") + + def close(self): + """ + Close the underlying transport. + """ + self._transport.close() + + async def send(self, data, addr=None): + """ + @param data - bytes to send + @param addr - remote address to send data to, if unspecified then the + underlying socket has to have been been connected to a + remote address previously. + + @raises TransportClosed - DatagramTransport closed. + """ + if self._transport.is_closing(): + raise TransportClosed() + + _ = self.exception + self._transport.sendto(data, addr) + await self._drained.wait() + + async def recv(self): + """ + Receive data on the local socket. + + @return - tuple of the bytes received and the address (ip, port) that + the data was received from. + + @raises TransportClosed - DatagramTransport closed. + """ + if self._transport.is_closing(): + raise TransportClosed() + + _ = self.exception + data, addr = await self._recvq.get() + if data is None: + raise TransportClosed() + + return data, addr + + +class DatagramServer(DatagramStream): + """ + Datagram socket bound to an address on the local machine. + """ + + async def send(self, data, addr): + """ + @param data - bytes to send + @param addr - remote address to send data to. + """ + await super().send(data, addr) + + +class DatagramClient(DatagramStream): + """ + Datagram socket connected to a remote address. + """ + + async def send(self, data): + """ + @param data - bytes to send + """ + await super().send(data) + + +class Protocol(asyncio.DatagramProtocol): + """ + asyncio.DatagramProtocol for feeding received packets into the + Datagram{Client,Server} which handles converting the lower level callback + based asyncio into higher level coroutines. + """ + + def __init__(self, recvq, excq, drained): + """ + @param recvq - asyncio.Queue for new datagrams + @param excq - asyncio.Queue for exceptions + @param drained - asyncio.Event set when the write buffer is below the + high watermark. + """ + self._recvq = recvq + self._excq = excq + self._drained = drained + + self._drained.set() + + # Transports are connected at the time a connection is made. + self._transport = None + + def connection_made(self, transport): + if self._transport is not None: + old_peer = self._transport.get_extra_info("peername") + new_peer = transport.get_extra_info("peername") + warnings.warn( + "Reinitializing transport connection from %s to %s", old_peer, new_peer + ) + + self._transport = transport + + def connection_lost(self, exc): + if exc is not None: + self._excq.put_nowait(exc) + + self._recvq.put_nowait((None, None)) + + if self._transport is not None: + self._transport.close() + self._transport = None + + def datagram_received(self, data, addr): + self._recvq.put_nowait((data, addr)) + + def error_received(self, exc): + self._excq.put_nowait(exc) + + def pause_writing(self): + self._drained.clear() + super().pause_writing() + + def resume_writing(self): + self._drained.set() + super().resume_writing() + + +async def bind(addr): + """ + Bind a socket to a local address for datagrams. The socket will be either + AF_INET, AF_INET6 or AF_UNIX depending upon the type of address specified. + + @param addr - For AF_INET or AF_INET6, a tuple with the the host and port to + to bind; port may be set to 0 to get any free port. + For AF_UNIX the path at which to bind (with a leading \0 for + abstract sockets). + @return - A DatagramServer instance + """ + loop = asyncio.get_event_loop() + recvq = asyncio.Queue() + excq = asyncio.Queue() + drained = asyncio.Event() + + if not isinstance(addr, tuple): + family = socket.AF_UNIX + if isinstance(addr, pathlib.Path): + addr = str(addr) + else: + family = 0 + + transport, protocol = await loop.create_datagram_endpoint( + lambda: Protocol(recvq, excq, drained), + local_addr=addr, + family=family, + ) + + return DatagramServer(transport, recvq, excq, drained) + + +async def connect(addr, local_addr=None, allow_broadcast=None): + """ + Connect a socket to a remote address for datagrams. The socket will be + either AF_INET, AF_INET6 or AF_UNIX depending upon the type of host + specified. + + @param addr - For AF_INET or AF_INET6, a tuple with the the host and port to + to connect to. + For AF_UNIX the path at which to connect (with a leading \0 + for abstract sockets). + @return - A DatagramClient instance + """ + loop = asyncio.get_event_loop() + recvq = asyncio.Queue() + excq = asyncio.Queue() + drained = asyncio.Event() + + if not isinstance(addr, tuple): + family = socket.AF_UNIX + if isinstance(addr, pathlib.Path): + addr = str(addr) + else: + family = 0 + + transport, protocol = await loop.create_datagram_endpoint( + lambda: Protocol(recvq, excq, drained), + remote_addr=addr, + family=family, + local_addr=local_addr, + allow_broadcast=allow_broadcast, + ) + + return DatagramClient(transport, recvq, excq, drained) + + +async def from_socket(sock): + """ + Create a DatagramStream from a socket. This is meant to be used in cases + where the defaults set by `bind()` and `connect()` are not desired and/or + sufficient. If `socket.connect()` was previously called on the socket, + then an instance of DatagramClient will be returned, otherwise an instance + of DatagramServer. + + @param sock - socket to use in the DatagramStream. + @return - A DatagramClient for connected sockets, otherwise a + DatagramServer. + """ + loop = asyncio.get_event_loop() + recvq = asyncio.Queue() + excq = asyncio.Queue() + drained = asyncio.Event() + + supported_families = tuple((socket.AF_INET, socket.AF_INET6)) + if not sys.platform == "win32": + supported_families += (socket.AF_UNIX,) + + if sock.family not in supported_families: + raise TypeError( + "socket family not one of %s" + % (", ".join(str(f) for f in supported_families)) + ) + + if sock.type != socket.SOCK_DGRAM: + raise TypeError("socket type must be %s" % (socket.SOCK_DGRAM,)) + + transport, protocol = await loop.create_datagram_endpoint( + lambda: Protocol(recvq, excq, drained), sock=sock + ) + + if transport.get_extra_info("peername") is not None: + # Workaround transport ignoring the peer address of the socket. + transport._address = transport.get_extra_info("peername") + return DatagramClient(transport, recvq, excq, drained) + else: + return DatagramServer(transport, recvq, excq, drained) diff --git a/.venv/Lib/site-packages/pytak/classes.py b/.venv/Lib/site-packages/pytak/classes.py new file mode 100644 index 0000000..066375f --- /dev/null +++ b/.venv/Lib/site-packages/pytak/classes.py @@ -0,0 +1,660 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# classes.py from https://github.com/snstac/pytak +# +# Copyright Sensors & Signals LLC https://www.snstac.com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""PyTAK Class Definitions.""" + +import abc +import asyncio +import ipaddress +import logging +import multiprocessing as mp +import random + + +import os +import uuid +import zipfile +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import List, Optional, Dict, Any +import argparse +import sys + + +import xml.etree.ElementTree as ET + +from dataclasses import dataclass +from typing import Set, Union + +from configparser import ConfigParser, SectionProxy + +import pytak + +try: + import takproto # type: ignore +except ImportError: + takproto = None + + +# Optimized: Shared logger configuration to avoid duplication +def _setup_logger(logger: logging.Logger, level: int = None) -> logging.Logger: + """Configure a logger with standard PyTAK formatting.""" + if not logger.handlers: + log_level = level or pytak.LOG_LEVEL + logger.setLevel(log_level) + console_handler = logging.StreamHandler() + console_handler.setLevel(log_level) + console_handler.setFormatter(pytak.LOG_FORMAT) + logger.addHandler(console_handler) + logger.propagate = False + return logger + + +class Worker: + """Meta class for all other Worker Classes.""" + + _logger = _setup_logger(logging.getLogger(__name__)) + logging.getLogger("asyncio").setLevel(pytak.LOG_LEVEL) + + def __init__( + self, + queue: Union[asyncio.Queue, mp.Queue], + config: Union[None, SectionProxy, dict] = None, + ) -> None: + """Initialize a Worker instance.""" + self.queue: Union[asyncio.Queue, mp.Queue] = queue + if config: + self.config = config + else: + config_p = ConfigParser({}) + config_p.add_section("pytak") + self.config = config_p["pytak"] or {} + + if bool(self.config.get("DEBUG")): + for handler in self._logger.handlers: + handler.setLevel(logging.DEBUG) + + tak_proto_version = int(self.config.get("TAK_PROTO") or pytak.DEFAULT_TAK_PROTO) + + if tak_proto_version > 0 and takproto is None: + self._logger.warning( + "TAK_PROTO is set to '%s', but the 'takproto' Python module is not installed.\n" + "Try: python -m pip install pytak[with_takproto]\n" + "See Also: https://pytak.rtfd.io/en/latest/compatibility/#tak-protocol-payload-version-1-protobuf", + tak_proto_version, + ) + + self.use_protobuf = tak_proto_version > 0 and takproto is not None + + async def fts_compat(self) -> None: + """Apply FreeTAKServer (FTS) compatibility. + + If the FTS_COMPAT (or PYTAK_SLEEP) config options are set, will async sleep for + either a given (PYTAK_SLEEP) or random (FTS_COMPAT) time. + """ + pytak_sleep: int = int(self.config.get("PYTAK_SLEEP") or 0) + if bool(self.config.get("FTS_COMPAT") or pytak_sleep): + sleep_period: int = int( + pytak_sleep or (int(pytak.DEFAULT_SLEEP) * random.random()) + ) + self._logger.debug("COMPAT: Sleeping for %ss", sleep_period) + await asyncio.sleep(sleep_period) + + @abc.abstractmethod + async def handle_data(self, data: bytes) -> None: + """Handle data (placeholder method, please override).""" + pass + + async def _handle_full_queue(self, queue: Union[asyncio.Queue, mp.Queue]) -> None: + """Handle a full queue by removing oldest item. Optimized to reduce code duplication.""" + self._logger.warning( + "Queue full, dropping oldest data. Consider raising MAX_IN_QUEUE or MAX_OUT_QUEUE see https://pytak.rtfd.io/" + ) + if isinstance(queue, asyncio.Queue): + await queue.get() + else: + queue.get_nowait() + + async def run_once(self) -> None: + """Reads Data from Queue & passes data to next Handler.""" + data = await self.queue.get() + await self.handle_data(data) + await self.fts_compat() + + async def run(self, _=-1) -> None: + """Run this Thread - calls run_once() in a loop.""" + self._logger.info("Running: %s", self.__class__.__name__) + while True: + await self.run_once() + await asyncio.sleep(0) # make sure other tasks have a chance to run + + +class TXWorker(Worker): + """Works data queue and hands off to Protocol Workers. + + You should create an TXWorker Instance using the `pytak.txworker_factory()` + Function. + + Data is put onto the Queue using a `pytak.QueueWorker()` instance. + """ + + def __init__( + self, + queue: Union[asyncio.Queue, mp.Queue], + config: Union[None, SectionProxy, dict], + writer: asyncio.Protocol, + ) -> None: + """Initialize a TXWorker instance.""" + super().__init__(queue, config) + self.writer: asyncio.Protocol = writer + + async def handle_data(self, data: bytes) -> None: + """Accept CoT event from CoT event queue and process for writing.""" + # self._logger.debug("TX (%s): %s", self.config.get('name'), data) + await self.send_data(data) + + async def send_data(self, data: bytes) -> None: + """Send Data using the appropriate Protocol method.""" + if data is None: + self._logger.warning("send_data called with None data, skipping send.") + return + + if self.use_protobuf: + host, _ = pytak.parse_url(self.config.get("COT_URL", pytak.DEFAULT_COT_URL)) + is_multicast: bool = False + + try: + is_multicast = ipaddress.ip_address(host).is_multicast + except ValueError: + # It's probably not an ip address... + pass + + if is_multicast: + proto = takproto.TAKProtoVer.MESH + else: + proto = takproto.TAKProtoVer.STREAM + + try: + data = takproto.xml2proto(data, proto) + except ET.ParseError as exc: + self._logger.warning(exc) + self._logger.warning("Could not convert XML to Proto.") + + if hasattr(self.writer, "send"): + await self.writer.send(data) + else: + if hasattr(self.writer, "write"): + self.writer.write(data) + if hasattr(self.writer, "drain"): + await self.writer.drain() + if hasattr(self.writer, "flush"): + # FIXME: This should be an asyncio.Future?: + self.writer.flush() + + +class RXWorker(Worker): + """Async receive (input) queue worker. + + Reads events from a `pytak.protocol_factory()` reader and adds them to + an `rx_queue`. + + Most implementations use this to drain an RX buffer on a socket. + + pytak([asyncio.Protocol]->[pytak.EventReceiver]->[queue.Queue]) + """ + + def __init__( + self, + queue: Union[asyncio.Queue, mp.Queue], + config: Union[None, SectionProxy, dict], + reader: asyncio.Protocol, + ) -> None: + """Initialize a RXWorker instance.""" + super().__init__(queue, config) + self.reader: asyncio.Protocol = reader + self.reader_queue = None + + @abc.abstractmethod + async def handle_data(self, data: bytes) -> None: + """Handle data (placeholder method, please override).""" + pass + + async def readcot(self): + """Read CoT from the wire until we hit an event boundary.""" + cot = None + try: + if hasattr(self.reader, "readuntil"): + cot = await self.reader.readuntil("".encode("UTF-8")) + elif hasattr(self.reader, "recv"): + cot, _ = await self.reader.recv() + + if self.use_protobuf: + tak_v1 = takproto.parse_proto(cot) + if tak_v1 != -1: + cot = tak_v1 # .SerializeToString() + return cot + except asyncio.IncompleteReadError: + return None + + async def run_once(self) -> None: + """Run this worker once.""" + if self.reader: + data: bytes = await self.readcot() + if data: + self._logger.debug("RX data: %s", data) + self.queue.put_nowait(data) + + async def run(self, _=-1) -> None: + """Run this worker.""" + self._logger.info("Running: %s", self.__class__.__name__) + while True: + await self.run_once() + await asyncio.sleep(0) # make sure other tasks have a chance to run + + +class QueueWorker(Worker): + """Read non-CoT Messages from an async network client. + + (`asyncio.Protocol` or similar async network client) + Serializes it as COT, and puts it onto an `asyncio.Queue`. + + Implementations should handle serializing messages as COT Events, and + putting them onto the `event_queue`. + + The `event_queue` is handled by the `pytak.EventWorker` Class. + + pytak([asyncio.Protocol]->[pytak.MessageWorker]->[asyncio.Queue]) + """ + + def __init__( + self, + queue: Union[asyncio.Queue, mp.Queue], + config: Union[None, SectionProxy, dict], + ) -> None: + super().__init__(queue, config) + self._logger.info("Using COT_URL='%s'", self.config.get("COT_URL")) + + @abc.abstractmethod + async def handle_data(self, data: bytes) -> None: + """Handle data (placeholder method, please override).""" + pass + + async def put_queue( + self, data: bytes, queue_arg: Union[asyncio.Queue, mp.Queue, None] = None + ) -> None: + """Put Data onto the Queue.""" + _queue = queue_arg or self.queue + self._logger.debug("Queue size=%s", _queue.qsize()) + + # Optimized: Check for full queue once and handle uniformly + if _queue.full(): + await self._handle_full_queue(_queue) + + if isinstance(_queue, asyncio.Queue): + await _queue.put(data) + else: + _queue.put_nowait(data) + + +class CLITool: + """Wrapper Object for CLITools.""" + + _logger = _setup_logger(logging.getLogger(__name__)) + logging.getLogger("asyncio").setLevel(pytak.LOG_LEVEL) + + def __init__( + self, + config: Union[ConfigParser, SectionProxy], + tx_queue: Union[asyncio.Queue, mp.Queue, None] = None, + rx_queue: Union[asyncio.Queue, mp.Queue, None] = None, + ) -> None: + """Initialize CLITool instance.""" + self.tasks: Set = set() + self.running_tasks: Set = set() + self._config = config + self.queues: dict = {} + + self.max_in_queue = int( + self._config.get("MAX_IN_QUEUE") or pytak.DEFAULT_MAX_IN_QUEUE + ) + self.max_out_queue = int( + self._config.get("MAX_OUT_QUEUE") or pytak.DEFAULT_MAX_OUT_QUEUE + ) + self.tx_queue: Union[asyncio.Queue, mp.Queue] = tx_queue or asyncio.Queue( + self.max_out_queue + ) + self.rx_queue: Union[asyncio.Queue, mp.Queue] = rx_queue or asyncio.Queue( + self.max_in_queue + ) + + if isinstance(self._config, SectionProxy) and bool(self._config.get("DEBUG")): + for handler in self._logger.handlers: + handler.setLevel(logging.DEBUG) + + @property + def config(self): + """Return the config object.""" + return self._config + + @config.setter + def config(self, val): + """Set the config object.""" + self._config = val + + async def create_workers(self, i_config): + """ + Create and run queue workers with specified config parameters. + + Parameters + ---------- + i_config : `configparser.SectionProxy` + Configuration options & values. + """ + tx_queue = asyncio.Queue(self.max_out_queue) + rx_queue = asyncio.Queue(self.max_in_queue) + if len(self.queues) == 0: + # If the queue list is empty, make this the default. + self.tx_queue = tx_queue + self.rx_queue = rx_queue + self.queues[i_config.name] = {"tx_queue": tx_queue, "rx_queue": rx_queue} + + reader, writer = await pytak.protocol_factory(i_config) + write_worker = pytak.TXWorker(tx_queue, i_config, writer) + read_worker = pytak.RXWorker(rx_queue, i_config, reader) + self.add_task(write_worker) + self.add_task(read_worker) + + async def setup(self) -> None: + """Set up CLITool. + + Creates protocols, queue workers and adds them to our task list. + """ + # Create our TX & RX Protocol Worker + reader, writer = await pytak.protocol_factory(self.config) + write_worker = pytak.TXWorker(self.tx_queue, self.config, writer) + read_worker = pytak.RXWorker(self.rx_queue, self.config, reader) + self.add_task(write_worker) + self.add_task(read_worker) + + async def hello_event(self): + """Send a 'hello world' style event to the Queue.""" + hello = pytak.hello_event(self.config.get("COT_HOST_ID")) + if hello: + self.tx_queue.put_nowait(hello) + + def add_task(self, task): + """Add the given task to our coroutine task list.""" + self._logger.debug("Add Task: %s", task) + self.tasks.add(task) + + def add_tasks(self, tasks): + """Add the given list or set of tasks to our couroutine task list.""" + for task in tasks: + self.add_task(task) + + def run_task(self, task): + """Run the given coroutine task.""" + self._logger.debug("Run Task: %s", task) + self.running_tasks.add(asyncio.ensure_future(task.run())) + # self.running_tasks.add(run_coroutine_in_thread(task.run())) + + def run_tasks(self, tasks=None): + """Run the given list or set of couroutine tasks.""" + tasks = tasks or self.tasks + for task in tasks: + self.run_task(task) + self.tasks.clear() + + async def run(self): + """Run this Thread and its associated coroutine tasks.""" + self._logger.info("Run: %s", self.__class__.__name__) + + if not self.config.get("PYTAK_NO_HELLO", False): + await self.hello_event() + + self.run_tasks() + + done, _ = await asyncio.wait( + self.running_tasks, return_when=asyncio.FIRST_COMPLETED + ) + + for task in done: + self._logger.info("Complete: %s", task) + + +@dataclass +class SimpleCOTEvent: + """CoT Event Dataclass.""" + + lat: Union[bytes, str, float, None] = None + lon: Union[bytes, str, float, None] = None + uid: Union[str, None] = None + stale: Union[float, int, None] = None + cot_type: Union[str, None] = None + + def __str__(self) -> str: + """Return a formatted string representation of the dataclass.""" + event = self.to_xml() + return ET.tostring(event, encoding="unicode") + + def to_bytes(self) -> bytes: + """Return the class as bytes.""" + event = self.to_xml() + return ET.tostring(event, encoding="utf-8") + + def to_xml(self) -> ET.Element: + """Return a CoT Event as an XML string.""" + cotevent = COTEvent( + lat=self.lat, + lon=self.lon, + uid=self.uid, + stale=self.stale, + cot_type=self.cot_type, + le=pytak.DEFAULT_COT_VAL, + ce=pytak.DEFAULT_COT_VAL, + hae=pytak.DEFAULT_COT_VAL, + ) + event = pytak.cot2xml(cotevent) + return event + + +@dataclass +class COTEvent(SimpleCOTEvent): + """COT Event Dataclass.""" + + ce: Union[bytes, str, float, int, None] = None + hae: Union[bytes, str, float, int, None] = None + le: Union[bytes, str, float, int, None] = None + + def to_xml(self) -> ET.Element: + """Return a CoT Event as an XML string.""" + cotevent = COTEvent( + lat=self.lat, + lon=self.lon, + uid=self.uid, + stale=self.stale, + cot_type=self.cot_type, + le=self.le, + ce=self.ce, + hae=self.hae, + ) + event = pytak.cot2xml(cotevent) + return event + + +class TAKDataPackage: + """ + Generator for TAK Data Package formatted zip files. + """ + + def __init__(self, name: str, uid: Optional[str] = None, on_receive_delete: bool = False): + """ + Initialize TAK Data Package generator. + + Args: + name: Display name for the data package + uid: Unique identifier (auto-generated if None) + on_receive_delete: Whether to delete package after import + """ + self.name = name + self.uid = uid or str(uuid.uuid4()) + self.on_receive_delete = on_receive_delete + self.files: List[Dict[str, Any]] = [] + + def add_file(self, file_path: str, ignore: bool = False, zip_entry_name: Optional[str] = None): + """ + Add a file to the data package. + + Args: + file_path: Path to the file to include + ignore: Whether to ignore this file during import + zip_entry_name: Custom name for the file in the zip (uses filename if None) + """ + if not os.path.exists(file_path): + raise FileNotFoundError(f"File not found: {file_path}") + + entry_name = zip_entry_name or os.path.basename(file_path) + + self.files.append({ + 'path': file_path, + 'zip_entry': entry_name, + 'ignore': ignore + }) + + def add_directory(self, dir_path: str, recursive: bool = True, ignore_pattern: Optional[str] = None): + """ + Add all files from a directory to the data package. + + Args: + dir_path: Path to the directory + recursive: Whether to include subdirectories + ignore_pattern: File pattern to ignore (simple wildcard matching) + """ + if not os.path.exists(dir_path): + raise FileNotFoundError(f"Directory not found: {dir_path}") + + dir_path = Path(dir_path) + + if recursive: + files = dir_path.rglob('*') + else: + files = dir_path.glob('*') + + for file_path in files: + if file_path.is_file(): + # Simple pattern matching for ignore_pattern + if ignore_pattern and ignore_pattern in file_path.name: + continue + + # Calculate relative path for zip entry + relative_path = file_path.relative_to(dir_path) + self.add_file(str(file_path), zip_entry_name=str(relative_path)) + + def _generate_manifest_xml(self) -> str: + """ + Generate the manifest.xml content based on current configuration. + + Returns: + XML string for the manifest + """ + # Create root element + root = ET.Element("MissionPackageManifest", version="2") + + # Configuration section + config = ET.SubElement(root, "Configuration") + + # Add parameters + uid_param = ET.SubElement(config, "Parameter") + uid_param.set("name", "uid") + uid_param.set("value", self.uid) + + name_param = ET.SubElement(config, "Parameter") + name_param.set("name", "name") + name_param.set("value", self.name) + + delete_param = ET.SubElement(config, "Parameter") + delete_param.set("name", "onReceiveDelete") + delete_param.set("value", str(self.on_receive_delete).lower()) + + # Contents section + contents = ET.SubElement(root, "Contents") + + for file_info in self.files: + content = ET.SubElement(contents, "Content") + content.set("ignore", str(file_info['ignore']).lower()) + content.set("zipEntry", file_info['zip_entry']) + + # Format XML with proper indentation + self._indent_xml(root) + + # Convert to string + xml_str = ET.tostring(root, encoding='unicode', xml_declaration=True) + return xml_str + + def _indent_xml(self, elem, level=0): + """Add proper indentation to XML elements.""" + indent = "\n" + level * " " + if len(elem): + if not elem.text or not elem.text.strip(): + elem.text = indent + " " + if not elem.tail or not elem.tail.strip(): + elem.tail = indent + for child in elem: + self._indent_xml(child, level + 1) + if not child.tail or not child.tail.strip(): + child.tail = indent + else: + if level and (not elem.tail or not elem.tail.strip()): + elem.tail = indent + + def create_package(self, output_path: str, use_dpk_extension: bool = False, include_manifest: bool = True): + """ + Create the TAK Data Package zip file. + + Args: + output_path: Path where to save the package + use_dpk_extension: Use .dpk extension instead of .zip + include_manifest: Whether to include the manifest (if False, files are imported serially) + """ + if not self.files: + raise ValueError("No files added to the package") + + # Ensure proper extension + if use_dpk_extension and not output_path.endswith('.dpk'): + output_path = output_path.rsplit('.', 1)[0] + '.dpk' + elif not use_dpk_extension and not output_path.endswith('.zip'): + output_path = output_path.rsplit('.', 1)[0] + '.zip' + + # Create the zip file + with zipfile.ZipFile(output_path, 'w', zipfile.ZIP_DEFLATED) as zipf: + # Add all files + for file_info in self.files: + zipf.write(file_info['path'], file_info['zip_entry']) + print(f"Added: {file_info['zip_entry']}") + + # Add manifest if requested + if include_manifest: + manifest_xml = self._generate_manifest_xml() + + # Create MANIFEST directory and add manifest.xml + zipf.writestr('MANIFEST/manifest.xml', manifest_xml) + print("Added: MANIFEST/manifest.xml") + + print(f"\nTAK Data Package created: {output_path}") + print(f"Package UID: {self.uid}") + print(f"Files included: {len(self.files)}") + diff --git a/.venv/Lib/site-packages/pytak/client_functions.py b/.venv/Lib/site-packages/pytak/client_functions.py new file mode 100644 index 0000000..60d5b3d --- /dev/null +++ b/.venv/Lib/site-packages/pytak/client_functions.py @@ -0,0 +1,588 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# client_functions.py from https://github.com/snstac/pytak +# +# Copyright Sensors & Signals LLC https://www.snstac.com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""PyTAK functions for creating network & CLI TAK clients.""" + +import argparse +import asyncio +import importlib +import ipaddress +import logging +import os +import platform +import pprint +import secrets +import socket +import ssl +import struct +import sys +import warnings +import tempfile + +from asyncio import get_running_loop +from configparser import ConfigParser, SectionProxy +from urllib.parse import ParseResult, urlparse +from typing import Any, Tuple, Union + +import pytak + +from pytak.functions import unzip_file, find_file, load_preferences, connectString2url + +from pytak.asyncio_dgram import ( + DatagramClient, + connect as dgconnect, + from_socket, +) + +from pytak.crypto_functions import convert_cert + + +def get_cot_url(config) -> ParseResult: + """Verify and parse a raw COT_URL.""" + raw_cot_url: str = config.get("COT_URL", pytak.DEFAULT_COT_URL) + + if "://" not in raw_cot_url: + warnings.warn(f"Invalid COT_URL={raw_cot_url}", SyntaxWarning) + raise SyntaxError( + "Specify COT_URL as a full URL. For example: tcp://tak.example.com:1234" + ) + + cot_url: ParseResult = urlparse(raw_cot_url) + return cot_url + + +async def protocol_factory( # NOQA pylint: disable=too-many-locals,too-many-branches,too-many-statements + config: SectionProxy, +) -> Any: + """Create input, output, or input-output clients for network and file protocols. + + Parameters + ---------- + config : `SectionProxy` + Configuration parameters & values. + + Returns + ------- + `Any` + Varies by input-output protocol. + """ + reader: Any = None + writer: Any = None + + cot_url: ParseResult = get_cot_url(config) + scheme: str = cot_url.scheme.lower() + + # TCP + if scheme in ["tcp"]: + host, port = pytak.parse_url(cot_url) + reader, writer = await asyncio.open_connection(host, port) + + # TLS + elif scheme in ["tls", "ssl"]: + reader, writer = await create_tls_client(config, cot_url) + + # UDP + elif "udp" in scheme: + # Support Linux hosts with no default gateway defined with local addr: + local_addr = ( + config.get( + "PYTAK_MULTICAST_LOCAL_ADDR", pytak.DEFAULT_PYTAK_MULTICAST_LOCAL_ADDR + ), + 0, + ) + multicast_ttl = config.get("PYTAK_MULTICAST_TTL", 1) + reader, writer = await pytak.create_udp_client( + cot_url, local_addr, multicast_ttl + ) + + # LOG + elif "log" in scheme: + if cot_url.hostname: + dest: str = cot_url.hostname.lower() + if "stderr" in dest: + writer = sys.stderr.buffer + else: + writer = sys.stdout.buffer + # File output + elif "file" in scheme: + path = cot_url.netloc + cot_url.path + file_path = Path(path) + file_path.parent.mkdir(parents=True, exist_ok=True) + writer = open(file_path, 'wb+') + + # Default + if not reader and not writer: + raise SyntaxError( + "Invalid COT_URL protocol specified. " + "See: https://pytak.rtfd.io/en/stable/configuration/" + ) + + return reader, writer + + +async def create_udp_client( + url: ParseResult, local_addr=None, multicast_ttl=1 +) -> Tuple[Union[DatagramClient, None], DatagramClient]: + """Create an AsyncIO UDP network client for Unicast, Broadcast & Multicast. + + Parameters + ---------- + url : `ParseResult` + A parsed fully-qualified URL parsed with `urllib.parse.urlparse()`. + An input to urparse() would be: udp://tak.example.com:4242 + + Returns + ------- + `DatagramClient` + An AsyncIO UDP network stream client. + """ + reader: Union[DatagramClient, None] = None + rsock: Union[socket.socket, None] = None + + host, port = pytak.parse_url(url) + + local_addr = local_addr or "0.0.0.0" + + is_write_only: bool = "+wo" in url.scheme + is_broadcast: bool = "broadcast" in url.scheme + is_multicast: bool = "multicast" in url.scheme + + # Optimized: Single try-catch for IP address validation + if not is_multicast: + try: + is_multicast = ipaddress.ip_address(host).is_multicast + except ValueError: + # It's probably not an ip address... + pass + + # Create the Writer + writer: DatagramClient = await dgconnect( + (host, port), local_addr=local_addr, allow_broadcast=is_broadcast + ) + + if is_broadcast: + writer.socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + + if is_multicast: + writer.socket.setsockopt( + socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, struct.pack("b", multicast_ttl) + ) + + if is_write_only: + return reader, writer + + # Create the Reader + rsock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + + bindall = sys.platform == "win32" + rsock.bind(("" if bindall else host, port)) + + reader = await from_socket(rsock) + + if not reader: + return reader, writer + + if is_broadcast: + # SO_BROADCAST + reader.socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + + if is_broadcast or is_multicast: + # SO_REUSEADDR + reader.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + # SO_REUSEPORT + reader.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + except AttributeError: + # Some systems don't support SO_REUSEPORT + pass + + # Create Multicast Reader + if is_multicast: + ip = ( + socket.INADDR_ANY + if local_addr[0] is None + else int(ipaddress.IPv4Address(local_addr[0])) + ) + group = int(ipaddress.IPv4Address(host)) + mreq = struct.pack("!LL", group, ip) + reader.socket.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) + + return reader, writer + + +def get_tls_config(config: SectionProxy) -> SectionProxy: + """Get the TLS config and ensures required TLS params are set. + + Parameters + ---------- + config : `SectionProxy` + Configuration parameters & values. + + Returns + ------- + `SectionProxy` + A PyTAK TLS configuration. + """ + tls_config_req: dict = dict( + zip( + pytak.DEFAULT_TLS_PARAMS_REQ, + [config.get(x) for x in pytak.DEFAULT_TLS_PARAMS_REQ], + ) + ) + + if not all(tls_config_req.values()): + raise SyntaxError( + f"Not all required TLS Params specified: {pytak.DEFAULT_TLS_PARAMS_REQ}" + ) + + tls_config_opt: dict = dict( + zip( + pytak.DEFAULT_TLS_PARAMS_OPT, + [config.get(x) for x in pytak.DEFAULT_TLS_PARAMS_OPT], + ) + ) + + tls_config_req.update(tls_config_opt) + + return ConfigParser(dict(filter(lambda x: x[1], tls_config_req.items())))["DEFAULT"] + + +async def create_tls_client( + config, cot_url +) -> Tuple[asyncio.StreamReader, asyncio.StreamWriter]: + """Create a two-way TLS socket. + + Establishing a socket requires: + 1. Enabling or disabling TLS Verifications. + 2. Establishing a TLS Context. + 3. Configuring an async TCP read-write socket. + + Parameters + ---------- + config : `SectionProxy` + Configuration parameters for this socket. + cot_url : `str` + The COT_URL as a string (un-parsed). + """ + + reader, writer = None, None + host, port = pytak.parse_url(cot_url) + tls_config: SectionProxy = get_tls_config(config) + + if tls_config.get("PYTAK_TLS_CERT_ENROLLMENT_USERNAME") and tls_config.get( + "PYTAK_TLS_CERT_ENROLLMENT_PASSWORD" + ): + from pytak.crypto_classes import CertificateEnrollment + + enrollment = CertificateEnrollment() + + cert_enrollment_username = tls_config.get("PYTAK_TLS_CERT_ENROLLMENT_USERNAME") + cert_enrollment_password = tls_config.get("PYTAK_TLS_CERT_ENROLLMENT_PASSWORD") + cert_enrollment_url = tls_config.get("PYTAK_TLS_CERT_ENROLLMENT_URL", host) + + cert_enrollment_passphrase = tls_config.get( + "PYTAK_TLS_CERT_ENROLLMENT_PASSPHRASE" + ) + if not cert_enrollment_passphrase: + # Generate a random passphrase for the PKCS#12 file. + cert_enrollment_passphrase = secrets.token_urlsafe(16) + print( + f"Using generated passphrase for enrollment: {cert_enrollment_passphrase}" + ) + tls_config["PYTAK_TLS_CERT_ENROLLMENT_PASSPHRASE"] = ( + cert_enrollment_passphrase + ) + + with tempfile.NamedTemporaryFile(suffix=".p12", delete=False) as tmpfile: + output_path = tmpfile.name + + await enrollment.begin_enrollment( + domain=host, + username=cert_enrollment_username, + password=cert_enrollment_password, + output_path=output_path, + passphrase=cert_enrollment_passphrase, + ) + # Update TLS config with the output path of the cert enrollment. + tls_config["PYTAK_TLS_CLIENT_CERT"] = output_path + + ssl_ctx = get_ssl_ctx(tls_config) + + if ssl_ctx.check_hostname: + expected_server_hostname = tls_config.get("PYTAK_TLS_SERVER_EXPECTED_HOSTNAME") + else: + expected_server_hostname = None + + try: + reader, writer = await asyncio.open_connection( + host, port, ssl=ssl_ctx, server_hostname=expected_server_hostname + ) + except ssl.SSLCertVerificationError as exc: + raise SyntaxError( + ( + "Could not verify TLS Certificate for TAK Server." + "Bypass with PYTAK_TLS_DONT_CHECK_HOSTNAME=1 or PYTAK_TLS_DONT_VERIFY=1" + "See: https://pytak.rtfd.io/en/stable/configuration" + ) + ) from exc + + return reader, writer + + +def get_ssl_ctx(tls_config: SectionProxy) -> ssl.SSLContext: + """Configure a TLS socket context.""" + + client_cert = tls_config.get("PYTAK_TLS_CLIENT_CERT") + client_key = tls_config.get("PYTAK_TLS_CLIENT_KEY") + client_cafile = tls_config.get("PYTAK_TLS_CLIENT_CAFILE") + client_password = tls_config.get( + "PYTAK_TLS_CERT_ENROLLMENT_PASSPHRASE", + tls_config.get("PYTAK_TLS_CLIENT_PASSWORD"), + ) + + client_ciphers = tls_config.get("PYTAK_TLS_CLIENT_CIPHERS") or "ALL" + + # Do not verify CA against our trust store. + dont_verify = tls_config.getboolean("PYTAK_TLS_DONT_VERIFY") + + dont_check_hostname = dont_verify or tls_config.getboolean( + "PYTAK_TLS_DONT_CHECK_HOSTNAME" + ) + + # Cert is always required. + if client_cert: + if not os.path.exists(client_cert): + raise SyntaxError( + f"Resource not found: PYTAK_TLS_CLIENT_CERT={client_cert}" + ) + else: + raise SyntaxError("Missing value: PYTAK_TLS_CLIENT_CERT") + + if client_key: + if not os.path.exists(client_key): + raise SyntaxError(f"Resource not found: PYTAK_TLS_CLIENT_KEY={client_key}") + + # SSL Context + ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ssl_ctx.options |= ssl.OP_NO_TLSv1 + ssl_ctx.options |= ssl.OP_NO_TLSv1_1 + ssl_ctx.set_ciphers(client_ciphers) + # Checks & Verifications + ssl_ctx.check_hostname = True + ssl_ctx.verify_mode = ssl.VerifyMode.CERT_REQUIRED + + # PCKS#12 + if client_cert.endswith(".p12"): + cert_paths = convert_cert(client_cert, client_password) + client_cert = cert_paths["cert_pem_path"] + client_key = cert_paths["pk_pem_path"] + if not os.path.exists(client_cert) and os.path.exists(client_key): + raise SystemError( + f"Missing PKCS#12 extracted {client_cert} & {client_key}." + ) + + try: + ssl_ctx.load_cert_chain( + client_cert, keyfile=client_key, password=client_password + ) + except Exception as exc: + raise ValueError( + f"Error opening resource. Using: PYTAK_TLS_CLIENT_CERT={client_cert} " + f"[PYTAK_TLS_CLIENT_KEY={client_key}] Using " + f"Password: {bool(client_password)}?" + ) from exc + + # CA File + if client_cafile: + ssl_ctx.load_verify_locations(cafile=client_cafile) + + # Disables TLS Server Common Name Verification + if dont_check_hostname: + warnings.warn("Disabled TLS Server Common Name Verification") + ssl_ctx.check_hostname = False + + # Disables TLS Server Certificate Verification + if dont_verify: + warnings.warn("Disabled TLS Server Certificate Verification") + ssl_ctx.verify_mode = ssl.CERT_NONE + + return ssl_ctx + + +async def txworker_factory( + queue: asyncio.Queue, config: SectionProxy +) -> pytak.TXWorker: + """Create a PyTAK TXWorker based on URL parameters. + + :param cot_url: URL to COT Destination. + :param event_queue: asyncio.Queue worker to get events from. + :return: EventWorker or asyncio Protocol + """ + _, writer = await protocol_factory(config) + return pytak.TXWorker(queue, config, writer) + + +async def rxworker_factory( + queue: asyncio.Queue, config: SectionProxy +) -> pytak.RXWorker: + """Create a PyTAK TXWorker based on URL parameters. + + :param cot_url: URL to COT Destination. + :param event_queue: asyncio.Queue worker to get events from. + :return: EventWorker or asyncio Protocol + """ + reader, _ = await protocol_factory(config) + return pytak.RXWorker(queue, config, reader) + + +async def main(app_name: str, config: SectionProxy, full_config: ConfigParser) -> None: + """ + Abstract implementation of an async main function. + + Parameters + ---------- + app_name : `str` + Name of the app calling this function. + config : `SectionProxy` + A dict of configuration parameters & values. + full_config : `ConfigParser` + A full dict of configuration parameters & values. + """ + app = importlib.__import__(app_name) + clitool: pytak.CLITool = pytak.CLITool(config) + create_tasks = getattr(app, "create_tasks") + await clitool.create_workers(config) + if bool(config.get("IMPORT_OTHER_CONFIGS", pytak.DEFAULT_IMPORT_OTHER_CONFIGS)): + try: + for i in full_config.sections()[1:]: + await clitool.create_workers(full_config[i]) + except EOFError: + logging.warning("No more configs to create workers for!") + # await clitool.setup() + clitool.add_tasks(create_tasks(config, clitool)) + await clitool.run() + + +def read_pref_package(pref_package: str) -> dict: + """Read a pref package / data package of preferences.""" + pref_config = { + "COT_URL": "", + "PYTAK_TLS_CLIENT_CERT": None, + "PYTAK_TLS_CLIENT_KEY": None, + "PYTAK_TLS_CLIENT_CAFILE": None, + } + + dp_path: str = unzip_file(pref_package) + pref_file: str = find_file(dp_path, "*.pref") + prefs: dict = load_preferences(pref_file, dp_path) + + connect_string: str = prefs.get("connect_string", "") + assert connect_string + pref_config["COT_URL"] = connectString2url(connect_string) + + cert_location: str = prefs.get("certificate_location", "") + assert os.path.exists(cert_location) + + client_password: str = prefs.get("client_password", "") + assert client_password + + import pytak.crypto_functions + + pem_certs: dict = pytak.crypto_functions.convert_cert( + cert_location, client_password + ) + pref_config["PYTAK_TLS_CLIENT_CERT"] = pem_certs.get("cert_pem_path") + pref_config["PYTAK_TLS_CLIENT_KEY"] = pem_certs.get("pk_pem_path") + pref_config["PYTAK_TLS_CLIENT_CAFILE"] = pem_certs.get("ca_pem_path") + + assert all(pref_config) + return pref_config + + +def cli(app_name: str) -> None: + """Abstract implementation of a Command Line Interface (CLI). + + Parameters + ---------- + app_name : `str` + Name of the app calling this function. + """ + app = importlib.__import__(app_name) + + parser = argparse.ArgumentParser() + parser.add_argument( + "-c", + "--CONFIG_FILE", + dest="CONFIG_FILE", + default="config.ini", + type=str, + help="Optional configuration file. Default: config.ini", + ) + parser.add_argument( + "-p", + "--PREF_PACKAGE", + dest="PREF_PACKAGE", + required=False, + type=str, + help="Optional connection preferences package zip file (aka data package).", + ) + namespace = parser.parse_args() + cli_args = {k: v for k, v in vars(namespace).items() if v is not None} + + # Read config: + env_vars = os.environ + + # Remove env vars that contain '%', which ConfigParser or pprint barf on: + env_vars = {key: val for key, val in env_vars.items() if "%" not in val} + + env_vars["COT_URL"] = env_vars.get("COT_URL", pytak.DEFAULT_COT_URL) + env_vars["COT_HOST_ID"] = f"{app_name}@{platform.node()}" + env_vars["COT_STALE"] = getattr(app, "DEFAULT_COT_STALE", pytak.DEFAULT_COT_STALE) + env_vars["TAK_PROTO"] = env_vars.get("TAK_PROTO", pytak.DEFAULT_TAK_PROTO) + + orig_config: ConfigParser = ConfigParser(env_vars) + + config_file = cli_args.get("CONFIG_FILE", "") + if os.path.exists(config_file): + logging.info("Reading configuration from %s", config_file) + orig_config.read(config_file) + else: + orig_config.add_section(app_name) + + config: SectionProxy = orig_config[app_name] + full_config: ConfigParser = orig_config + + pref_package: str = config.get("PREF_PACKAGE", cli_args.get("PREF_PACKAGE")) + if pref_package and os.path.exists(pref_package): + pref_config = read_pref_package(pref_package) + config.update(pref_config) + + debug = config.getboolean("DEBUG") + if debug: + print(f"Showing Config: {config_file}") + print("=" * 10) + pprint.pprint(dict(config)) + print("=" * 10) + + if sys.version_info[:2] >= (3, 7): + asyncio.run(main(app_name, config, full_config), debug=debug) + else: + loop = get_running_loop() + try: + loop.run_until_complete(main(app_name, config, full_config)) + finally: + loop.close() diff --git a/.venv/Lib/site-packages/pytak/commands.py b/.venv/Lib/site-packages/pytak/commands.py new file mode 100644 index 0000000..d8596ec --- /dev/null +++ b/.venv/Lib/site-packages/pytak/commands.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# commands.py from https://github.com/snstac/pytak +# +# Copyright Sensors & Signals LLC https://www.snstac.com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""PyTAK Command Line.""" + +import pytak + + +def main() -> None: + """Boilerplate main function.""" + # PyTAK CLI tool boilerplate: + pytak.cli(__name__.split(".", maxsplit=1)[0]) + + +if __name__ == "__main__": + main() diff --git a/.venv/Lib/site-packages/pytak/constants.py b/.venv/Lib/site-packages/pytak/constants.py new file mode 100644 index 0000000..916c46f --- /dev/null +++ b/.venv/Lib/site-packages/pytak/constants.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# constants.py from https://github.com/snstac/pytak +# +# Copyright Sensors & Signals LLC https://www.snstac.com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""PyTAK Constants.""" + +import logging +import os +import platform + +from typing import Optional + + +LOG_LEVEL: int = logging.INFO +LOG_FORMAT: logging.Formatter = logging.Formatter( + ("%(asctime)s pytak %(levelname)s - %(message)s") +) + +if os.environ.get("INVOCATION_ID"): + LOG_LEVEL = logging.INFO + LOG_FORMAT = logging.Formatter(("[%(levelname)s] %(message)s")) + logging.debug("Systemd format logging enabled via INVOCATION_ID env var.") + +if bool(os.environ.get("DEBUG")): + LOG_LEVEL = logging.DEBUG + LOG_FORMAT = logging.Formatter( + ( + "%(asctime)s pytak %(levelname)s %(name)s.%(funcName)s:%(lineno)d - " + "%(message)s" + ) + ) + logging.debug("pytak Debugging Enabled via DEBUG Environment Variable.") + +DEFAULT_COT_URL: str = "udp+wo://239.2.3.1:6969" # ATAK Default multicast +DEFAULT_COT_STALE: str = "120" # Config wants all values as strings, we'll cast later. +DEFAULT_HOST_ID: str = f"pytak@{platform.node()}" +DEFAULT_COT_PORT: str = "8087" +DEFAULT_ATAK_PORT: str = "4242" +DEFAULT_BROADCAST_PORT: str = "6969" + +DEFAULT_BACKOFF: str = "120" +DEFAULT_SLEEP: str = "5" +DEFAULT_FIPS_CIPHERS: str = ( + "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384" +) + +W3C_XML_DATETIME: str = "%Y-%m-%dT%H:%M:%S.%fZ" +ISO_8601_UTC = W3C_XML_DATETIME # Issue 51: Not technically correct. + +TC_TOKEN_URL = "https://app-api.parteamconnect.com/api/v1/auth/token" +DEFAULT_TC_TOKEN_URL = os.getenv("TC_TOKEN_URL", TC_TOKEN_URL) + +DEFAULT_TLS_PARAMS_REQ: tuple = () + +# Optimized: Use tuple instead of list for immutable constant (faster access) +DEFAULT_TLS_PARAMS_OPT: tuple = ( + "PYTAK_TLS_CLIENT_CERT", + "PYTAK_TLS_CLIENT_KEY", + "PYTAK_TLS_CLIENT_CAFILE", + "PYTAK_TLS_CLIENT_CIPHERS", + "PYTAK_TLS_DONT_CHECK_HOSTNAME", + "PYTAK_TLS_DONT_VERIFY", + "PYTAK_TLS_CLIENT_PASSWORD", + "PYTAK_TLS_SERVER_EXPECTED_HOSTNAME", + "PYTAK_TLS_CERT_ENROLLMENT_USERNAME", + "PYTAK_TLS_CERT_ENROLLMENT_PASSWORD", + "PYTAK_TLS_CERT_ENROLLMENT_PASSPHRASE", +) + +DEFAULT_IMPORT_OTHER_CONFIGS: str = "0" + +# Optimized: Use tuple for immutable truth values (faster lookups) +BOOLEAN_TRUTH: tuple = ("true", "yes", "y", "on", "1") +DEFAULT_COT_VAL: str = "9999999.0" + +# TAK Protocol to use for CoT output, one of: 0 (XML, default), 1 (Mesh/Stream). +# Doesn't always work with iTAK. Recommend sticking with 0 (XML). +DEFAULT_TAK_PROTO: str = "0" + +# Python <3.8 has no way of including XML Declaration in ET.tostring(): +DEFAULT_XML_DECLARATION: bytes = ( + b'' +) + +# Multicast +DEFAULT_PYTAK_MULTICAST_LOCAL_ADDR: str = "0.0.0.0" + +# See MIL-STD-6090. +DEFAULT_COT_ACCESS: Optional[str] = os.getenv("COT_ACCESS", "UNCLASSIFIED") +DEFAULT_COT_CAVEAT: Optional[str] = os.getenv("COT_CAVEAT", "") +DEFAULT_COT_RELTO: Optional[str] = os.getenv("COT_RELTO", "") +DEFAULT_COT_QOS: Optional[str] = os.getenv("COT_QOS", "") +DEFAULT_COT_OPEX: Optional[str] = os.getenv("COT_OPEX", "") + +DEFAULT_MAX_OUT_QUEUE = 100 +DEFAULT_MAX_IN_QUEUE = 500 + +DEFAULT_TLS_ENROLLMENT_CERT_PASSPHRASE_LENGTH: int = 16 \ No newline at end of file diff --git a/.venv/Lib/site-packages/pytak/crypto_classes.py b/.venv/Lib/site-packages/pytak/crypto_classes.py new file mode 100644 index 0000000..887201b --- /dev/null +++ b/.venv/Lib/site-packages/pytak/crypto_classes.py @@ -0,0 +1,927 @@ +#!/usr/bin/env python3 +""" +Certificate Enrollment Module + +Performs certificate enrollment operations similar to the Android Java implementation. +This module handles certificate signing requests, key generation, and certificate processing. +Uses async/await for improved performance and modern Python patterns. +""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import logging +import os +import secrets +from typing import Dict, List, Optional, Tuple +import xml.etree.ElementTree as ET +from pathlib import Path + +import urllib +import warnings + +from pytak.crypto_functions import INSTALL_MSG + + +USE_CRYPTOGRAPHY = False +try: + from cryptography import x509 + from cryptography.hazmat.primitives import serialization, hashes + from cryptography.hazmat.primitives.asymmetric import rsa + from cryptography.hazmat.primitives.serialization import pkcs12 + USE_CRYPTOGRAPHY = True +except ImportError as exc: + warnings.warn(str(exc)) + +USE_AIOHTTP = False +try: + import aiohttp + from aiohttp import BasicAuth, ClientTimeout + USE_AIOHTTP = True +except ImportError as exc: + warnings.warn(str(exc)) + + +class CertificateEnrollment: + """ + Perform certificate enrollment operations using async/await. + + This class handles the certificate enrollment process including: + - Key generation + - Certificate Signing Request (CSR) creation + - Certificate retrieval and processing + - Keystore creation + """ + + def __init__( + self, + trust_store_path: Optional[str] = None, + ): + """ + Initialize the certificate enrollment class. + + Args: + trust_store_path: Optional path to the trust store certificate file + """ + if not USE_CRYPTOGRAPHY: + raise ValueError(INSTALL_MSG) + + if not USE_AIOHTTP: + raise ValueError( + "This module requires aiohttp for asynchronous HTTP requests. " + "Please install it using 'pip install aiohttp'. " + "See https://pytak.rtfd.io/ for more details." + ) + + self.logger = logging.getLogger(__name__) + self.trust_store_path = trust_store_path + self._setup_logging() + + def _setup_logging(self): + """Setup logging configuration.""" + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) + # Enable debug logging for this module when verbose is needed + # Uncomment the next line to see more detailed parsing info + # self.logger.setLevel(logging.DEBUG) + + async def begin_enrollment( + self, + domain: str, + username: str, + password: str, + trust_all: bool = False, + use_v2: bool = True, + output_path: Optional[str] = None, + passphrase: Optional[str] = None, + ) -> None: + """ + Begin the certificate enrollment process. + + Args: + domain: The domain for certificate enrollment + username: Username for authentication + password: Password for authentication + trust_all: If True, skip certificate verification (NOT for production) + use_v2: If True, use the v2 endpoint that returns JSON (default: True) + output_path: Custom path for the output certificate file. If None, + defaults to ~/Downloads/clientCert.p12 + """ + private_key = self._generate_key() + if private_key is None: + self.logger.error("Failed to generate private key") + return + + self.logger.info("Private key generated successfully") + + # Run enrollment asynchronously + await self._enrollment_process( + domain, + username, + password, + trust_all, + private_key, + use_v2, + output_path, + passphrase, + ) + + async def _enrollment_process( + self, + domain: str, + username: str, + password: str, + trust_all: bool, + private_key, + use_v2: bool = True, + output_path: Optional[str] = None, + passphrase: Optional[str] = None, + ) -> None: + """ + Async function to handle the enrollment process. + + Args: + domain: The domain for certificate enrollment + username: Username for authentication + password: Password for authentication + trust_all: If True, skip certificate verification + private_key: The generated private key + use_v2: If True, use the v2 endpoint that returns JSON + output_path: Custom path for the output certificate file + """ + try: + async with self._create_session(trust_all) as session: + csr = await self._generate_csr( + username, password, session, domain, private_key + ) + if csr: + await self._process_csr( + username, + password, + session, + domain, + csr, + private_key, + use_v2, + output_path, + passphrase=passphrase, + ) + except Exception as e: + self.logger.error(f"Error in enrollment process: {e}") + + def _generate_key(self, key_size: int = 4096) -> Optional[rsa.RSAPrivateKey]: + """ + Generate a private RSA key. + + Args: + key_size: Size of the RSA key (default: 4096) + + Returns: + RSA private key or None if generation fails + """ + try: + private_key = rsa.generate_private_key( + public_exponent=65537, key_size=key_size + ) + return private_key + except Exception as e: + self.logger.error(f"Failed to generate private key: {e}") + return None + + #def _create_session(self, trust_all: bool) -> aiohttp.ClientSession: + def _create_session(self, trust_all: bool): + """ + Create an aiohttp session with appropriate SSL configuration. + + Args: + trust_all: If True, disable SSL verification + + Returns: + Configured aiohttp ClientSession + """ + # Configure timeout + timeout = ClientTimeout(total=30) + + # Configure SSL context + ssl_context = None + trust_all = True + if trust_all: + ssl_context = False + self.logger.warning("SSL verification disabled - NOT for production use!") + elif self.trust_store_path and os.path.exists(self.trust_store_path): + # For aiohttp, we would need to create a custom SSL context + # For now, we'll use the default verification + ssl_context = None + + # Create connector with retry configuration + # Don't pass loop parameter to avoid event loop issues + connector = aiohttp.TCPConnector(limit=10, ssl=ssl_context) + + return aiohttp.ClientSession(connector=connector, timeout=timeout) + + async def _generate_csr( + self, + username: str, + password: str, + session: aiohttp.ClientSession, + domain: str, + private_key, + ) -> Optional[str]: + """ + Generate a Certificate Signing Request. + + Args: + username: Username for authentication + password: Password for authentication + session: aiohttp ClientSession to use + domain: Domain for the request + private_key: Private key for the CSR + + Returns: + CSR string or None if generation fails + """ + try: + url = f"https://{domain}:8446/Marti/api/tls/config" + auth = BasicAuth(username, password) + + async with session.get(url, auth=auth) as response: + response.raise_for_status() + response_text = await response.text() + + self.logger.info(f"Received config response: {response_text}") + + # Parse XML response + csr_config = self._parse_config_xml(response_text) + if not csr_config: + self.logger.error("Failed to parse config XML") + return None + + # Add username as CN + csr_config["CN"] = username + + # Generate CSR + csr = self._create_csr_from_config(csr_config, private_key) + self.logger.info("CSR generated successfully") + return csr + + except Exception as e: + self.logger.error(f"Error generating CSR: {e}") + return None + + def _parse_config_xml(self, xml_content: str) -> Dict[str, str]: + """ + Parse the XML configuration response. + + Args: + xml_content: XML content string + + Returns: + Dictionary of configuration parameters + """ + try: + root = ET.fromstring(xml_content) + config = {} + + # Handle different XML structures - try both formats + # First try the namespaced format (newer TAK servers) + # The namespace might be different, so we'll search by tag name regardless of namespace + for elem in root.iter(): + if elem.tag.endswith("nameEntry") or elem.tag == "nameEntry": + name = elem.get("name", "") + value = elem.get("value", "") + if name and value: + config[name] = value + + # If no entries found, try the older format with simple "entry" elements + if not config: + for elem in root.iter(): + if elem.tag.endswith("entry") or elem.tag == "entry": + name = elem.get("name", "") + value = elem.get("value", "") + if name and value: + config[name] = value + + self.logger.debug(f"Parsed config: {config}") + return config + except Exception as e: + self.logger.error(f"Error parsing XML config: {e}") + return {} + + def _create_csr_from_config(self, config: Dict[str, str], private_key) -> str: + """ + Create a CSR from configuration parameters. + + Args: + config: Configuration dictionary + private_key: Private key for the CSR + + Returns: + PEM-encoded CSR string + """ + # Build subject name from config + subject_components = [] + + if "CN" in config: + subject_components.append( + x509.NameAttribute(x509.NameOID.COMMON_NAME, config["CN"]) + ) + if "O" in config: + subject_components.append( + x509.NameAttribute(x509.NameOID.ORGANIZATION_NAME, config["O"]) + ) + if "OU" in config: + subject_components.append( + x509.NameAttribute(x509.NameOID.ORGANIZATIONAL_UNIT_NAME, config["OU"]) + ) + if "C" in config: + subject_components.append( + x509.NameAttribute(x509.NameOID.COUNTRY_NAME, config["C"]) + ) + if "ST" in config: + subject_components.append( + x509.NameAttribute(x509.NameOID.STATE_OR_PROVINCE_NAME, config["ST"]) + ) + if "L" in config: + subject_components.append( + x509.NameAttribute(x509.NameOID.LOCALITY_NAME, config["L"]) + ) + + subject = x509.Name(subject_components) + + # Create CSR + csr = ( + x509.CertificateSigningRequestBuilder() + .subject_name(subject) + .sign(private_key, hashes.SHA256()) + ) + + # Return PEM-encoded CSR + return csr.public_bytes(serialization.Encoding.PEM).decode("utf-8") + + async def _process_csr( + self, + username: str, + password: str, + session: aiohttp.ClientSession, + domain: str, + csr: str, + private_key, + use_v2: bool = True, + output_path: Optional[str] = None, + passphrase: Optional[str] = None, + ) -> None: + """ + Process the CSR by sending it for signing and handling the response. + + Args: + username: Username for authentication + password: Password for authentication + session: aiohttp ClientSession to use + domain: Domain for the request + csr: Certificate Signing Request + private_key: Private key corresponding to the CSR + use_v2: If True, use the v2 endpoint that returns JSON + output_path: Custom path for the output certificate file + """ + try: + # Choose endpoint based on version + if use_v2: + url = ( + f"https://{domain}:8446/Marti/api/tls/signClient/v2?clientUid=test" + ) + content_type = "application/pkcs10" + else: + url = f"https://{domain}:8446/Marti/api/tls/signClient?clientUid=test" + content_type = "application/pkcs10" + + auth = BasicAuth(username, password) + + async with session.post( + url, + auth=auth, + data=csr.encode("utf-8"), + headers={"Content-Type": content_type}, + ) as response: + self.logger.info(f"Response code: {response.status}") + + if response.status == 200: + if use_v2: + # v2 endpoint returns JSON (but might have incorrect content-type) + try: + response_data = await response.json() + except Exception as json_error: + # If JSON parsing fails due to content-type, try parsing text as JSON + self.logger.warning( + f"JSON parsing failed ({json_error}), trying to parse text as JSON" + ) + response_text = await response.text() + response_data = json.loads(response_text) + + await self._process_json_certificate_response( + response_data, private_key, output_path, passphrase + ) + else: + # Original endpoint returns PKCS12 + response_content = await response.read() + await self._process_pkcs12_certificate_response( + response_content, private_key, output_path, passphrase + ) + else: + self.logger.error( + f"Certificate signing failed with status: {response.status}" + ) + # If v2 fails, try fallback to v1 + if use_v2: + self.logger.info("Trying fallback to v1 endpoint...") + await self._process_csr( + username, + password, + session, + domain, + csr, + private_key, + use_v2=False, + output_path=output_path, + passphrase=passphrase, + ) + + except Exception as e: + self.logger.error(f"Error processing CSR: {e}") + # If v2 fails with exception, try fallback to v1 + if use_v2: + self.logger.info( + "v2 endpoint failed, trying fallback to v1 endpoint..." + ) + try: + await self._process_csr( + username, + password, + session, + domain, + csr, + private_key, + use_v2=False, + output_path=output_path, + passphrase=passphrase, + ) + except Exception as fallback_error: + self.logger.error( + f"Both v2 and v1 endpoints failed: {fallback_error}" + ) + + async def _process_json_certificate_response( + self, + response_data: dict, + private_key, + output_path: Optional[str] = None, + passphrase: Optional[str] = None, + ) -> None: + """ + Process the JSON certificate response from v2 endpoint. + + Args: + response_data: JSON response containing certificate data + private_key: Private key for the certificate + output_path: Custom path for the output certificate file + """ + try: + self.logger.info("Processing JSON certificate response from v2 endpoint") + self.logger.debug(f"Response data keys: {list(response_data.keys())}") + + # Extract the signed certificate + signed_cert_pem = response_data.get("signedCert") + if not signed_cert_pem: + self.logger.error("No 'signedCert' found in JSON response") + return + + # Extract CA certificates - they might be named ca0, ca1, ca2, etc. + ca_pems = [] + ca_index = 0 + while f"ca{ca_index}" in response_data: + ca_pem = response_data[f"ca{ca_index}"] + if ca_pem: + ca_pems.append(ca_pem) + ca_index += 1 + + self.logger.info( + f"Found signed certificate and {len(ca_pems)} CA certificates" + ) + + # Debug: Log certificate formats to help diagnose PEM issues + self.logger.debug(f"Signed cert starts with: {signed_cert_pem[:50]}...") + if ca_pems: + self.logger.debug(f"CA cert 0 starts with: {ca_pems[0][:50]}...") + + # Debug: Log raw certificate lengths and check for obvious issues + self.logger.info(f"Raw signed cert length: {len(signed_cert_pem)}") + self.logger.info( + f"Raw CA cert lengths: {[len(ca_pem) for ca_pem in ca_pems]}" + ) + + # Check if certificates have proper PEM structure or are headerless (v2 format) + if not signed_cert_pem.startswith("-----BEGIN"): + self.logger.info( + "Signed certificate appears to be headerless (v2 format) - will add headers" + ) + else: + self.logger.info("Signed certificate has PEM headers") + + for i, ca_pem in enumerate(ca_pems): + if not ca_pem.startswith("-----BEGIN"): + self.logger.info( + f"CA certificate {i} appears to be headerless (v2 format) - will add headers" + ) + else: + self.logger.info(f"CA certificate {i} has PEM headers") + + # Fix PEM formatting if needed + self.logger.info("Applying PEM formatting fixes...") + try: + signed_cert_pem = self._fix_pem_formatting(signed_cert_pem) + ca_pems = [self._fix_pem_formatting(ca_pem) for ca_pem in ca_pems] + + # Debug: Log post-fix certificate info + self.logger.info(f"Fixed signed cert length: {len(signed_cert_pem)}") + self.logger.debug( + f"Fixed signed cert preview:\n{signed_cert_pem[:200]}..." + ) + + # Validate that certificates can be parsed before trying to create PKCS12 + self.logger.info("Pre-validating certificate parsing...") + from cryptography import x509 + + test_cert = x509.load_pem_x509_certificate( + signed_cert_pem.encode("utf-8") + ) + self.logger.info(f"✅ Signed certificate validation passed") + + for i, ca_pem in enumerate(ca_pems): + test_ca = x509.load_pem_x509_certificate(ca_pem.encode("utf-8")) + self.logger.info(f"✅ CA certificate {i} validation passed") + + except Exception as validation_error: + self.logger.error( + f"❌ Certificate validation failed: {validation_error}" + ) + self.logger.error( + "This appears to be a server-side certificate format issue." + ) + self.logger.info( + "Will attempt to save raw certificate data for manual inspection..." + ) + + # Save raw certificate data for debugging + debug_dir = Path.home() / "Downloads" / "cert_debug" + debug_dir.mkdir(exist_ok=True) + + with open(debug_dir / "raw_signed_cert.pem", "w") as f: + f.write(signed_cert_pem) + with open(debug_dir / "raw_ca_certs.txt", "w") as f: + for i, ca_pem in enumerate(ca_pems): + f.write(f"=== CA Certificate {i} ===\n") + f.write(ca_pem) + f.write("\n\n") + + self.logger.info(f"Raw certificate data saved to: {debug_dir}") + return + + # Create client certificate file + self._create_client_certificate( + signed_cert_pem, ca_pems, private_key, output_path, passphrase + ) + + except Exception as e: + self.logger.error(f"Error processing JSON certificate response: {e}") + import traceback + + self.logger.error(f"Full traceback: {traceback.format_exc()}") + + async def _process_pkcs12_certificate_response( + self, response_content: bytes, private_key, output_path: Optional[str] = None + ) -> None: + """ + Process the certificate response and create client keystore. + + Args: + response_content: Raw response content (PKCS12 data) + private_key: Private key for the certificate + output_path: Custom path for the output certificate file + """ + try: + self.logger.info( + f"Processing certificate response, size: {len(response_content)} bytes" + ) + + # First, let's try to load it as PKCS12 using the original method + try: + pkcs12_data = pkcs12.load_key_and_certificates( + response_content, b"atakatak" # Default password from original code + ) + private_key_from_p12, certificate, additional_certificates = pkcs12_data + + self.logger.info( + f"PKCS12 data loaded - Certificate: {certificate is not None}, " + f"Additional certs: {len(additional_certificates) if additional_certificates else 0}" + ) + + # Handle the case where the main certificate might be None + # but we have certificates in additional_certificates + if certificate is None and additional_certificates: + self.logger.info( + "Main certificate is None, checking additional certificates" + ) + # Look for the signed certificate - it might be the first one or have a specific pattern + certificate = additional_certificates[0] + additional_certificates = ( + additional_certificates[1:] + if len(additional_certificates) > 1 + else [] + ) + self.logger.info( + "Using first additional certificate as main certificate" + ) + + if certificate is None: + self.logger.error("No certificate found in PKCS12 response") + return + + # Convert certificate to PEM + cert_pem = certificate.public_bytes(serialization.Encoding.PEM).decode( + "utf-8" + ) + + # Convert CA certificates to PEM + ca_pems = [] + if additional_certificates: + for ca_cert in additional_certificates: + ca_pem = ca_cert.public_bytes( + serialization.Encoding.PEM + ).decode("utf-8") + ca_pems.append(ca_pem) + + self.logger.info("Certificate processing completed successfully") + self.logger.info(f"Found {len(ca_pems)} CA certificates") + + # Create client certificate file + self._create_client_certificate( + cert_pem, ca_pems, private_key, output_path + ) + + except Exception as pkcs12_error: + self.logger.error(f"Failed to load as PKCS12: {pkcs12_error}") + # Try to save the raw response for debugging + debug_file = Path.home() / "Downloads" / "server_response.p12" + with open(debug_file, "wb") as f: + f.write(response_content) + self.logger.info( + f"Saved raw server response to {debug_file} for debugging" + ) + raise + + except Exception as e: + self.logger.error(f"Error processing certificate response: {e}") + # Log additional debugging information + import traceback + + self.logger.error(f"Full traceback: {traceback.format_exc()}") + + def _create_client_certificate( + self, + cert_pem: str, + ca_pems: List[str], + private_key, + output_path: Optional[str] = None, + passphrase: Optional[bytes] = None, + ) -> None: + """ + Create a client certificate file (PKCS12 format). + + Args: + cert_pem: Client certificate in PEM format + ca_pems: CA certificates in PEM format + private_key: Private key for the certificate + output_path: Custom path for the output certificate file. If None, + defaults to ~/Downloads/clientCert.p12 + passphrase: Optional password for PKCS12 encryption (bytes). + """ + try: + self.logger.info("Creating client certificate from PEM data...") + + # Parse the certificate + self.logger.debug("Parsing signed certificate...") + try: + certificate = x509.load_pem_x509_certificate(cert_pem.encode("utf-8")) + self.logger.info("✅ Signed certificate parsed successfully") + except Exception as cert_error: + self.logger.error( + f"❌ Failed to parse signed certificate: {cert_error}" + ) + self.logger.error( + f"Signed cert content (first 300 chars):\n{cert_pem[:300]}" + ) + raise + + # Parse CA certificates + ca_certificates = [] + for i, ca_pem in enumerate(ca_pems): + self.logger.debug(f"Parsing CA certificate {i}...") + try: + ca_cert = x509.load_pem_x509_certificate(ca_pem.encode("utf-8")) + ca_certificates.append(ca_cert) + self.logger.info(f"✅ CA certificate {i} parsed successfully") + except Exception as ca_error: + self.logger.error( + f"❌ Failed to parse CA certificate {i}: {ca_error}" + ) + self.logger.error( + f"CA cert {i} content (first 300 chars):\n{ca_pem[:300]}" + ) + raise + + # Create PKCS12 data + pkcs12_data = pkcs12.serialize_key_and_certificates( + name=b"TAK Client Cert", + key=private_key, + cert=certificate, + cas=ca_certificates if ca_certificates else None, + encryption_algorithm=serialization.BestAvailableEncryption( + passphrase.encode("utf-8") + ), + ) + + # Save to file + if output_path: + cert_file = Path(output_path) + # Create parent directories if they don't exist + cert_file.parent.mkdir(parents=True, exist_ok=True) + else: + downloads_dir = Path.home() / "Downloads" + downloads_dir.mkdir( + parents=True, exist_ok=True + ) # Ensure Downloads directory exists + cert_file = downloads_dir / "clientCert.p12" + + with open(cert_file, "wb") as f: + f.write(pkcs12_data) + + self.logger.info(f"Client certificate saved to: {cert_file}") + + except Exception as e: + self.logger.error(f"Error creating client certificate: {e}") + + def _fix_pem_formatting(self, pem_content: str) -> str: + """ + Fix PEM certificate formatting issues. + + Some servers may return PEM certificates with incorrect line breaks + or formatting that causes parsing issues. The v2 endpoint specifically + returns PEM strings WITHOUT headers/footers and may not end with newlines. + + Args: + pem_content: Raw PEM content that may have formatting issues + + Returns: + Properly formatted PEM content with headers/footers + """ + if not pem_content: + return pem_content + + # First, handle JSON-escaped newlines (common in JSON responses) + content = pem_content.replace("\\n", "\n").replace("\\r", "\r") + + # Normalize line breaks + content = content.replace("\r\n", "\n").replace("\r", "\n").strip() + + # Check if this content lacks PEM headers/footers (v2 endpoint behavior) + if not content.startswith("-----BEGIN") and not content.endswith("-----"): + # This is likely a headerless PEM from v2 endpoint + self.logger.debug( + "Detected headerless PEM data from v2 endpoint, adding headers/footers" + ) + + # Remove any existing line breaks and whitespace + cert_data = content.replace("\n", "").replace(" ", "").replace("\t", "") + + # Format as proper PEM with headers/footers + formatted_lines = ["-----BEGIN CERTIFICATE-----"] + + # Add certificate data in 64-character lines (standard PEM format) + for i in range(0, len(cert_data), 64): + formatted_lines.append(cert_data[i : i + 64]) + + formatted_lines.append("-----END CERTIFICATE-----") + + result = "\n".join(formatted_lines) + "\n" + self.logger.debug( + f"Added PEM headers/footers: {len(content)} -> {len(result)} chars" + ) + return result + + # Check if this is a single-line certificate with headers (less common) + if ( + "-----BEGIN" in content + and "-----END" in content + and "\n" not in content.strip() + ): + # Split on the boundaries to extract parts + begin_match = content.find("-----BEGIN") + end_match = content.find("-----END") + + if begin_match != -1 and end_match != -1: + # Extract header + header_end = content.find("-----", begin_match + 5) + 5 + header = content[begin_match:header_end] + + # Extract footer + footer_start = content.find("-----END") + footer_end = content.find("-----", footer_start + 5) + 5 + footer = content[footer_start:footer_end] + + # Extract certificate data (between header and footer) + cert_data = content[header_end:footer_start].strip() + + # Format properly + formatted_lines = [header] + + # Add certificate data in 64-character lines + for i in range(0, len(cert_data), 64): + formatted_lines.append(cert_data[i : i + 64]) + + formatted_lines.append(footer) + + result = "\n".join(formatted_lines) + "\n" + self.logger.debug( + f"Fixed single-line PEM: {len(content)} -> {len(result)} chars" + ) + return result + + # Handle multi-line case with headers (original logic) + lines = content.split("\n") + + # Remove empty lines and strip whitespace + lines = [line.strip() for line in lines if line.strip()] + + # Find the header and footer + header_line = None + footer_line = None + cert_data_lines = [] + + for i, line in enumerate(lines): + if line.startswith("-----BEGIN"): + header_line = line + elif line.startswith("-----END"): + footer_line = line + elif header_line and not footer_line: + # This is certificate data + cert_data_lines.append(line) + + if not header_line or not footer_line: + self.logger.warning("PEM header or footer not found, returning as-is") + return pem_content + + # Reconstruct properly formatted PEM + formatted_lines = [header_line] + + # Add certificate data in 64-character lines (standard PEM format) + cert_data = "".join(cert_data_lines) + for i in range(0, len(cert_data), 64): + formatted_lines.append(cert_data[i : i + 64]) + + formatted_lines.append(footer_line) + + result = "\n".join(formatted_lines) + "\n" + + self.logger.debug( + f"Fixed multi-line PEM formatting: {len(pem_content)} -> {len(result)} chars" + ) + return result + + +async def main(): + """Example usage of the CertificateEnrollment class.""" + # Example usage + enrollment = CertificateEnrollment() + + # For testing with self-signed certificates (NOT for production) + domain = "example.com" + username = "testuser" + password = "testpass" + + # Use v2 endpoint by default (returns JSON) + # Example with custom output path + await enrollment.begin_enrollment( + domain, + username, + password, + trust_all=True, + use_v2=True, + output_path="./my_client_cert.p12", + ) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/.venv/Lib/site-packages/pytak/crypto_functions.py b/.venv/Lib/site-packages/pytak/crypto_functions.py new file mode 100644 index 0000000..b139d05 --- /dev/null +++ b/.venv/Lib/site-packages/pytak/crypto_functions.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# crypto_functions.py from https://github.com/snstac/pytak +# +# Copyright Sensors & Signals LLC https://www.snstac.com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""PyTAK Crypto (as in cryptography) Functions.""" + +import os +import tempfile +import warnings +import ssl + +from typing import Union,Tuple + + +INSTALL_MSG = ( + "Python cryptography module not installed. Install with: " + " python3 -m pip install cryptography" +) + +# Check if cryptography is installed +USE_CRYPTOGRAPHY = False +try: + from cryptography.hazmat.primitives import serialization + from cryptography.hazmat.primitives.serialization import pkcs12, Encoding, PrivateFormat, NoEncryption + from cryptography.x509 import Certificate + from cryptography.hazmat.primitives.asymmetric import rsa + + USE_CRYPTOGRAPHY = True +except ImportError as exc: + warnings.warn(str(exc)) + + +def save_pem(pem: bytes, dest: Union[str, None] = None) -> str: + """Save PEM data to dest.""" + if dest: + with open(dest, "wb+") as dest_fd: + dest_fd.write(pem) + pem_path: str = dest + else: + pem_fd, pem_path = tempfile.mkstemp(suffix=".pem") + with os.fdopen(pem_fd, "wb+") as pfd: + pfd.write(pem) + + assert os.path.exists(pem_path) + return pem_path + + +def load_cert( + cert_path: str, cert_pass: str +): # -> Set[_RSAPrivateKey, Certificate, Certificate]: + """Load RSA Keys & Certs from a pkcs12 ().p12) file.""" + if not USE_CRYPTOGRAPHY: + raise ValueError(INSTALL_MSG) + + with open(cert_path, "br+") as cp_fd: + p12_data = cp_fd.read() + + res = pkcs12.load_key_and_certificates(p12_data, str.encode(cert_pass)) + assert len(res) == 3 + return res + + +def convert_cert(cert_path: str, cert_pass: str) -> dict: + """Convert a P12 cert to PEM.""" + if not USE_CRYPTOGRAPHY: + raise ValueError(INSTALL_MSG) + + cert_paths = { + "pk_pem_path": None, + "cert_pem_path": None, + "ca_pem_path": None, + } + + private_key, cert, additional_certificates = load_cert(cert_path, cert_pass) + + # Load privkey + pk_pem = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ) + cert_paths["pk_pem_path"] = save_pem(pk_pem) + + cert_pem = cert.public_bytes(encoding=serialization.Encoding.PEM) + cert_paths["cert_pem_path"] = save_pem(cert_pem) + + ca_cert: Certificate = additional_certificates[0] + ca_pem = ca_cert.public_bytes(encoding=serialization.Encoding.PEM) + cert_paths["ca_pem_path"] = save_pem(ca_pem) + + assert all(cert_paths) + return cert_paths + + + +def convert_p12_to_pem(output_path: str, passphrase: str) -> Tuple[str, str]: + # Convert .p12 to PEM + with open(output_path, "rb") as p12_file: + p12_data = p12_file.read() + private_key, cert, additional_certs = pkcs12.load_key_and_certificates( + p12_data, passphrase.encode() + ) + + # Write PEM files + pem_key_path = output_path + ".key.pem" + pem_cert_path = output_path + ".cert.pem" + + with open(pem_key_path, "wb") as key_file: + key_file.write( + private_key.private_bytes( + Encoding.PEM, + PrivateFormat.TraditionalOpenSSL, + NoEncryption() + ) + ) + + with open(pem_cert_path, "wb") as cert_file: + cert_file.write(cert.public_bytes(Encoding.PEM)) + if additional_certs: + for ca in additional_certs: + cert_file.write(ca.public_bytes(Encoding.PEM)) + + return pem_key_path, pem_cert_path + + +def create_ssl_context(output_path, passphrase): + """Creates an SSL Context from a PKCS#12 certificate container.""" + # Convert the .p12 file to PEM format + pem_key_path, pem_cert_path = convert_p12_to_pem(output_path, passphrase) + print(f"Converted PKCS#12 to PEM. Key path: {pem_key_path}, Cert path: {pem_cert_path}") + + # Create an SSL context using the PEM files + # ssl_context = ssl.create_default_context() + ssl_context = ssl._create_unverified_context() + ssl_context.load_cert_chain(certfile=pem_cert_path, keyfile=pem_key_path) + ssl_context.check_hostname = False + ssl_context.verify_mode = ssl.CERT_NONE + + return ssl_context \ No newline at end of file diff --git a/.venv/Lib/site-packages/pytak/functions.py b/.venv/Lib/site-packages/pytak/functions.py new file mode 100644 index 0000000..99c3d96 --- /dev/null +++ b/.venv/Lib/site-packages/pytak/functions.py @@ -0,0 +1,348 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# functions.py from https://github.com/snstac/pytak +# +# Copyright Sensors & Signals LLC https://www.snstac.com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""PyTAK Functions.""" + +import datetime +import warnings +import xml.etree.ElementTree as ET +import secrets +import tempfile +import zipfile +import os + +import json + +from pathlib import Path +from typing import Optional, Tuple, Union +from urllib.parse import ParseResult, urlparse + +import pytak +import pytak.crypto_classes # pylint: disable=cyclic-import + + +def split_host(host: str, port: Union[int, None] = None) -> Tuple[str, int]: + """Split a host:port string or host, port params into a host,port tuple.""" + addr: str + _port: str + + if ":" in host: + addr, _port = host.split(":") + port = int(_port) + elif port: + addr = host + port = int(port) + else: + addr = host + port = int(pytak.DEFAULT_COT_PORT) + return addr, int(port) + + +def parse_url(url: Union[str, ParseResult]) -> Tuple[str, int]: + """Parse a CoT destination URL.""" + if isinstance(url, str): + _url: ParseResult = urlparse(url) + elif isinstance(url, ParseResult): + _url = url + + assert isinstance(_url, ParseResult) + + port: Union[int, str] = pytak.DEFAULT_BROADCAST_PORT + host: str = _url.netloc + + if ":" in _url.netloc: + host, port = _url.netloc.split(":") + else: + if "broadcast" in _url.scheme: + port = pytak.DEFAULT_BROADCAST_PORT + elif "multicast" in _url.scheme: + warnings.warn( + "You no longer need to specify '+multicast' in the COT_URL.", + DeprecationWarning, + ) + port = pytak.DEFAULT_BROADCAST_PORT + else: + port = pytak.DEFAULT_COT_PORT + + return host, int(port) + + +def cot_time(cot_stale: Union[int, None] = None) -> str: + """Get the current UTC datetime as a W3C XML Schema dateTime primitive. + + See: https://www.w3.org/TR/xmlschema-2/#dateTime + + Parameters + ---------- + cot_stale : `Union[int, None]` + Time in seconds to add to the current time, for use with Cursor on Target + 'stale' attributes. + + Returns + ------- + `str` + Current UTC datetime in W3C XML Schema dateTime format. + """ + time = datetime.datetime.now(datetime.timezone.utc) + if cot_stale: + time = time + datetime.timedelta(seconds=int(cot_stale)) + return time.strftime(pytak.W3C_XML_DATETIME) + + +def hello_event(uid: Optional[bytes] = None) -> bytes: + """Generate a Hello CoT Event.""" + uid = uid or "takPing" + return gen_cot(uid=uid, cot_type="t-x-d-d") + + +def unzip_file(zip_src: bytes, zip_dest: Union[bytes, None] = None) -> str: + """Unzips a given zip file, returning the destination path.""" + _zip_dest: str = zip_dest or tempfile.mkdtemp(prefix="pytak_dp_") + with zipfile.ZipFile(zip_src, "r") as zip_ref: + zip_ref.extractall(_zip_dest) + assert os.path.exists(_zip_dest) + return _zip_dest + + +def find_file(search_dir: str, glob: str) -> str: + """Find the first file for a given glob in the search directory.""" + try: + files = list(Path(search_dir).rglob(glob)) + assert len(files) > 0 + return str(files[0]) + except Exception as exc: + raise EOFError(f"Could not find file: {glob}") from exc + + +def find_cert(search_dir: str, cert_path: str) -> str: + """Find a cert file within a search dir after extracting the basename.""" + cert_file: str = os.path.basename(cert_path) + assert cert_file + return find_file(search_dir, cert_file) + + +def load_preferences(pref_path: str, search_dir: str) -> dict: + """Load preferences file into a dict.""" + with open(pref_path, "rb+") as pref_fd: + pref_data: bytes = pref_fd.read() + + root: ET.Element = ET.fromstring(pref_data) + entries: list = root.findall(".//entry") + + prefs: dict = { + "connect_string": None, + "client_password": None, + "certificate_location": None, + } + + # Determine the COT URL, client certificate and password + for entry in entries: + if entry.attrib["key"] == "connectString0": + prefs["connect_string"] = entry.text + if entry.attrib["key"] == "clientPassword": + prefs["client_password"] = entry.text + if entry.attrib["key"] == "certificateLocation": + prefs["certificate_location"] = find_cert(search_dir, entry.text) + + return prefs + + +def connectString2url(conn_str: str) -> str: # pylint: disable=invalid-name + """Convert a TAK-style connectString into a URL.""" + uri_parts = conn_str.split(":") + return f"{uri_parts[2]}://{uri_parts[0]}:{uri_parts[1]}" + + +def cot2xml(event: pytak.COTEvent) -> ET.Element: + """Generate a minimum COT Event as an XML object.""" + # Optimized: Reduce redundant str() calls and or operations + lat = str(event.lat) if event.lat is not None else "0.0" + lon = str(event.lon) if event.lon is not None else "0.0" + uid = event.uid or pytak.DEFAULT_HOST_ID + stale = int(event.stale or pytak.DEFAULT_COT_STALE) + cot_type = event.cot_type or "a-u-G" + le = str(event.le) if event.le is not None else pytak.DEFAULT_COT_VAL + hae = str(event.hae) if event.hae is not None else pytak.DEFAULT_COT_VAL + ce = str(event.ce) if event.ce is not None else pytak.DEFAULT_COT_VAL + + xevent = ET.Element("event") + xevent.set("version", "2.0") + xevent.set("type", cot_type) + xevent.set("uid", uid) + xevent.set("how", "m-g") + xevent.set("time", pytak.cot_time()) + xevent.set("start", pytak.cot_time()) + xevent.set("stale", pytak.cot_time(stale)) + + point = ET.Element("point") + point.set("lat", lat) + point.set("lon", lon) + point.set("le", le) + point.set("hae", hae) + point.set("ce", ce) + + # Optimized: Pre-compute flow tag name once + flow_tags = ET.Element("_flow-tags_") + _ft_tag: str = f"{pytak.DEFAULT_HOST_ID}-pytak".replace("@", "-") + flow_tags.set(_ft_tag, pytak.cot_time()) + + detail = ET.Element("detail") + detail.append(flow_tags) + + xevent.append(point) + xevent.append(detail) + + return xevent + + +def gen_cot_xml( + lat: Union[bytes, str, float, None] = None, + lon: Union[bytes, str, float, None] = None, + ce: Union[bytes, str, float, int, None] = None, + hae: Union[bytes, str, float, int, None] = None, + le: Union[bytes, str, float, int, None] = None, + uid: Union[str, None] = None, + stale: Union[float, int, None] = None, + cot_type: Union[str, None] = None, + callsign: Optional[str] = None, +) -> Optional[ET.Element]: + """Generate a minimum CoT Event as an XML object.""" + # Optimized: Use default values directly instead of redundant or operators + lat = str(lat) if lat is not None else "0.0" + lon = str(lon) if lon is not None else "0.0" + ce = str(ce) if ce is not None else pytak.DEFAULT_COT_VAL + hae = str(hae) if hae is not None else pytak.DEFAULT_COT_VAL + le = str(le) if le is not None else pytak.DEFAULT_COT_VAL + uid = uid or pytak.DEFAULT_HOST_ID + stale = int(stale or pytak.DEFAULT_COT_STALE) + cot_type = cot_type or "a-u-G" + + event = ET.Element("event") + event.set("version", "2.0") + event.set("type", cot_type) + event.set("uid", uid) + event.set("how", "m-g") + event.set("time", pytak.cot_time()) + event.set("start", pytak.cot_time()) + event.set("stale", pytak.cot_time(stale)) + + point = ET.Element("point") + point.set("lat", lat) + point.set("lon", lon) + point.set("le", le) + point.set("hae", hae) + point.set("ce", ce) + + # Optimized: Pre-compute flow tag name once + flow_tags = ET.Element("_flow-tags_") + _ft_tag: str = f"{pytak.DEFAULT_HOST_ID}-pytak".replace("@", "-") + flow_tags.set(_ft_tag, pytak.cot_time()) + + detail = ET.Element("detail") + detail.append(flow_tags) + + if callsign: + contact = ET.Element("contact") + contact.set("callsign", callsign) + detail.append(contact) + + event.append(point) + event.append(detail) + + return event + + +def gen_cot( + lat: Union[bytes, float, None] = None, + lon: Union[bytes, float, None] = None, + ce: Union[bytes, float, int, None] = None, + hae: Union[bytes, float, int, None] = None, + le: Union[bytes, float, int, None] = None, + uid: Optional[str] = None, + stale: Union[float, int, None] = None, + cot_type: Optional[str] = None, + callsign: Optional[str] = None, +) -> Optional[bytes]: + """Generate a minimum CoT Event as an XML string [gen_cot_xml() wrapper].""" + cot: Union[ET.Element, bytes, None] = gen_cot_xml( + lat, lon, ce, hae, le, uid, stale, cot_type, callsign + ) + if isinstance(cot, ET.Element): + # Optimized: Pre-allocate bytearray for better performance + return pytak.DEFAULT_XML_DECLARATION + b"\n" + ET.tostring(cot) + return cot + + +def tak_pong() -> bytes: + """Generate a takPong CoT Event.""" + event = ET.Element("event") + event.set("version", "2.0") + event.set("type", "t-x-d-d") + event.set("uid", "takPong") + event.set("how", "m-g") + + # Optimized: Compute time once and reuse + current_time: str = pytak.cot_time() + event.set("time", current_time) + event.set("start", current_time) + event.set("stale", pytak.cot_time(3600)) + + return ET.tostring(event) + + +async def enroll_tak(host: str, username: str, password: str, passphrase: Optional[str] = None) -> Tuple[str, str]: + enrollment = pytak.crypto_classes.CertificateEnrollment() + passphrase = secrets.token_urlsafe(pytak.DEFAULT_TLS_ENROLLMENT_CERT_PASSPHRASE_LENGTH) if passphrase is None else passphrase + + # Create a temporary file to store the certificate + output_path = None + + # Use tempfile to create a temp file for the cert + with tempfile.NamedTemporaryFile(suffix=".p12", delete=False) as tmpfile: + output_path = tmpfile.name + + await enrollment.begin_enrollment( + domain=host, + username=username, + password=password, + output_path=output_path, + passphrase=passphrase, + trust_all=True + ) + + return output_path, passphrase + + +async def decode_response(response): + """ + Decode the response from the server. + If the response is JSON, it will return a dictionary. + If the response is not JSON, it will return the text content. + """ + try: + response_data = await response.json() + except Exception as json_error: + # If JSON parsing fails due to content-type, try parsing text as JSON + response_text = await response.text() + try: + response_data = json.loads(response_text) + except json.JSONDecodeError: + # If still fails, return the text content + response_data = response_text + print(f"Failed to decode JSON: {json_error}. Returning text content instead.") + print(f"Response text: {response_text}") + return response_data \ No newline at end of file diff --git a/README.md b/README.md index 52539c5..20c5733 100644 --- a/README.md +++ b/README.md @@ -56,16 +56,17 @@ Main project areas: - AGWPE server adapter - APRS-IS style local server - APRS parser - - APRS packet generators + - APRS packet encoder - `sartrack/` - SARTrack TCP server - SARTrack APRS client - `mesh/` - Meshtastic TCP adapter - - Meshtastic packet normalization/parser + - Meshtastic RX parser + - Meshtastic TX encoder - `cot/` - CoT stub adapter - - XML generation helpers + - CoT XML parser and encoder - `gui/` - FastAPI-based operator panel - `utils/` @@ -82,18 +83,14 @@ Preferred public module layout: - `parser.py` for `raw -> canonical -> internal event` - `mesh/` - `adapter.py` for Meshtastic transport integration - - `decoder.py` for `raw -> internal event` + - `parser.py` for `raw -> canonical -> internal event` - `encoder.py` for `internal event -> mesh payload` - `filter.py` for outbound throttling/dedup shaping - - `parser.py` for lower-level packet normalization into canonical form - `cot/` - `adapter.py` for CoT-facing adapter entrypoints - - `encoder.py` for `internal event -> CoT XML/canonical` + - `encoder.py` for `internal event -> CoT XML` - `parser.py` for `CoT XML -> canonical -> internal event` -Historical implementation modules still exist under names like `generator.py`, `cot_encoder.py`, `mesh_encoder.py` or `mesh_decoder.py`. -They are intentionally kept as implementation backends for now, while the newer `adapter/parser/encoder/decoder/filter` modules are the preferred human-facing entrypoints when extending the repo. - Main runtime entrypoint: - [main.py](main.py) @@ -106,12 +103,12 @@ Main operator configuration: The preferred mental model for the gateway is: -`RX protocol -> parser/decoder -> canonical data -> internal event -> router -> encoder -> TX protocol` +`RX protocol -> parser -> canonical data -> internal event -> router -> encoder -> TX protocol` In plain language: 1. An ingress adapter receives raw network or radio-facing data. -2. A protocol parser or decoder translates that payload into a canonical/internal shape. +2. A protocol parser translates that payload into a canonical/internal shape. 3. The router deduplicates and dispatches the resulting event. 4. Egress adapters encode that internal event into their own transport format. @@ -121,7 +118,7 @@ Text diagram: APRS / SARTrack / Mesh / CoT RX | v - parser / decoder layer + parser layer | v canonical structure @@ -141,8 +138,8 @@ APRS / SARTrack / Mesh / CoT RX This is the main rule to keep in mind while extending the codebase: -- decoding belongs in `parser.py` or `decoder.py` -- event mapping belongs close to the parser/decoder layer +- decoding belongs in `parser.py` +- event mapping belongs close to the parser layer - routing belongs in `core/router.py` - transport-specific output shaping belongs in `encoder.py` - socket/session handling belongs in `adapter.py` @@ -274,14 +271,16 @@ CoT remains in the repo as a future-facing integration path rather than the main Implemented today: - canonical event shaping for CoT -- XML generation stub +- XML generation +- PyTAK-backed transport adapter for local TAK-compatible clients - different treatment for: - `station` - `object` Current state: -- not a full TAK Server integration +- local CoT transport can be driven by `PyTAK` URLs such as `udp://127.0.0.1:4242` +- not yet a full TAK Server integration - not a custom ATAK plugin effort - intended mainly as a future extension point @@ -500,6 +499,7 @@ Important runtime dependencies include: - `fastapi` - `uvicorn` - `paho-mqtt` +- `pytak` Python `3.11+` is recommended. @@ -610,7 +610,7 @@ Good starting points when extending the gateway: - [sartrack/adapter.py](sartrack/adapter.py) - [sartrack/parser.py](sartrack/parser.py) - [mesh/adapter.py](mesh/adapter.py) -- [mesh/decoder.py](mesh/decoder.py) +- [mesh/parser.py](mesh/parser.py) - [mesh/encoder.py](mesh/encoder.py) - [cot/adapter.py](cot/adapter.py) - [cot/parser.py](cot/parser.py) diff --git a/aprs/__init__.py b/aprs/__init__.py index 0449ad0..15d152a 100644 --- a/aprs/__init__.py +++ b/aprs/__init__.py @@ -1,6 +1,6 @@ from aprs.adapter import AGWPEAdapter, AGWPEConfig, AGWPEServerAdapter, AGWPEServerConfig, APRSISServerAdapter, APRSISServerConfig from aprs.encoder import build_message_packet, build_object_packet, build_position_packet -from aprs.parser import APRSParseError, canonical_to_aprs_event, parse_aprs_canonical, parse_aprs_event, parse_aprs_frame +from aprs.parser import APRSParseError, canonical_to_aprs_event, decode_aprs_frame, parse_aprs_canonical, parse_aprs_event, parse_aprs_frame __all__ = [ "AGWPEAdapter", @@ -13,6 +13,7 @@ "build_message_packet", "build_object_packet", "build_position_packet", + "decode_aprs_frame", "canonical_to_aprs_event", "parse_aprs_canonical", "parse_aprs_event", diff --git a/aprs/agwpe.py b/aprs/agwpe.py index c887a41..8f64f98 100644 --- a/aprs/agwpe.py +++ b/aprs/agwpe.py @@ -6,7 +6,7 @@ import struct from dataclasses import dataclass -from aprs.generator import build_message_packet, build_object_packet, build_position_packet +from aprs.encoder import build_message_packet, build_object_packet, build_position_packet from aprs.parser import APRSParseError, parse_aprs_event from core.model import MessageEvent, PositionEvent, Source from core.router import EventRouter @@ -200,7 +200,6 @@ async def send(self, event: PositionEvent | MessageEvent) -> None: source=event.source, timestamp=event.timestamp, ttl=event.ttl, - raw=event.raw, ) if _is_object_position(tx_event): frame = build_object_packet( @@ -227,7 +226,6 @@ async def send(self, event: PositionEvent | MessageEvent) -> None: source=event.source, timestamp=event.timestamp, ttl=event.ttl, - raw=event.raw, ) frame = build_message_packet( tx_event, @@ -410,14 +408,6 @@ def _select_tx_callsign(self, event: PositionEvent | MessageEvent) -> str | None def _extract_owner_callsign(event: PositionEvent | MessageEvent) -> str | None: if isinstance(event, PositionEvent) and event.owner_id: return event.owner_id - if isinstance(event.raw, dict): - canonical = event.raw.get("canonical") - if isinstance(canonical, dict): - custom = canonical.get("sartrack", {}).get("custom") - if isinstance(custom, dict): - owner = custom.get("owner_callsign") - if owner: - return str(owner) return None @staticmethod diff --git a/aprs/agwpe_server.py b/aprs/agwpe_server.py index ec01721..dc94cb9 100644 --- a/aprs/agwpe_server.py +++ b/aprs/agwpe_server.py @@ -6,7 +6,7 @@ import struct from dataclasses import dataclass -from aprs.generator import build_message_packet, build_object_packet, build_position_packet +from aprs.encoder import build_message_packet, build_object_packet, build_position_packet from aprs.parser import APRSParseError, parse_aprs_event from core.model import MessageEvent, PositionEvent, Source from core.router import EventRouter @@ -197,7 +197,6 @@ async def send(self, event: PositionEvent | MessageEvent) -> None: source=event.source, timestamp=event.timestamp, ttl=event.ttl, - raw=event.raw, ) if _is_object_position(tx_event): frame = build_object_packet( @@ -224,7 +223,6 @@ async def send(self, event: PositionEvent | MessageEvent) -> None: source=event.source, timestamp=event.timestamp, ttl=event.ttl, - raw=event.raw, ), destination=self.config.tx_destination, path=self.config.tx_path, @@ -585,14 +583,6 @@ def _select_tx_callsign(self, event: PositionEvent | MessageEvent) -> str | None def _extract_owner_callsign(event: PositionEvent | MessageEvent) -> str | None: if isinstance(event, PositionEvent) and event.owner_id: return event.owner_id - if isinstance(event.raw, dict): - canonical = event.raw.get("canonical") - if isinstance(canonical, dict): - custom = canonical.get("sartrack", {}).get("custom") - if isinstance(custom, dict): - owner = custom.get("owner_callsign") - if owner: - return str(owner) return None @staticmethod diff --git a/aprs/encoder.py b/aprs/encoder.py index 215c8ec..5ac91ab 100644 --- a/aprs/encoder.py +++ b/aprs/encoder.py @@ -1,4 +1,113 @@ -from aprs.generator import build_message_packet, build_object_packet, build_position_packet +from __future__ import annotations + +"""APRS packet encoder for position, object and message TX.""" + +import time + +from core.model import MessageEvent, PositionEvent + + +DEFAULT_SYMBOL_TABLE = "/" +DEFAULT_SYMBOL = ">" + + +def _coalesce_symbol_table(event: PositionEvent) -> str: + return event.symbol_table or DEFAULT_SYMBOL_TABLE + + +def _coalesce_symbol_code(event: PositionEvent) -> str: + return event.symbol_code or DEFAULT_SYMBOL + + +def _format_lat(lat: float) -> str: + hemi = "N" if lat >= 0 else "S" + lat = abs(lat) + degrees = int(lat) + minutes = (lat - degrees) * 60 + return f"{degrees:02d}{minutes:05.2f}{hemi}" + + +def _format_lon(lon: float) -> str: + hemi = "E" if lon >= 0 else "W" + lon = abs(lon) + degrees = int(lon) + minutes = (lon - degrees) * 60 + return f"{degrees:03d}{minutes:05.2f}{hemi}" + + +def _build_header(source: str, destination: str, path: str | None = None) -> str: + header = f"{source}>{destination}" + if path: + cleaned = path.strip() + if cleaned: + header += f",{cleaned}" + return header + + +def _format_object_timestamp(timestamp: int | float) -> str: + utc = time.gmtime(timestamp) + return time.strftime("%d%H%Mz", utc) + + +def build_position_packet( + event: PositionEvent, + destination: str = "APRS", + path: str | None = None, +) -> str: + if event.lat is None or event.lon is None: + raise ValueError("Cannot build APRS position packet without lat/lon") + + comment = (event.message or "")[:67] + payload = ( + f"!{_format_lat(event.lat)}{_coalesce_symbol_table(event)}" + f"{_format_lon(event.lon)}{_coalesce_symbol_code(event)}{comment}" + ) + header = _build_header(event.id, destination, path) + return f"{header}:{payload}" + + +def build_object_packet( + event: PositionEvent, + *, + source: str, + destination: str = "APRS", + path: str | None = None, +) -> str: + if event.lat is None or event.lon is None: + raise ValueError("Cannot build APRS object packet without lat/lon") + + object_name = (event.object_name or event.id or "").strip()[:9].ljust(9) + comment = event.message or "" + if event.tactical_name: + comment = f"{comment}[:{event.tactical_name}" if comment else f"[:{event.tactical_name}" + comment = comment[:67] + + payload = ( + f";{object_name}*{_format_object_timestamp(event.timestamp)}" + f"{_format_lat(event.lat)}{_coalesce_symbol_table(event)}" + f"{_format_lon(event.lon)}{_coalesce_symbol_code(event)}{comment}" + ) + header = _build_header(source, destination, path) + return f"{header}:{payload}" + + +def build_message_packet( + event: MessageEvent, + destination: str = "APRS", + path: str | None = None, +) -> str: + if not event.target: + raise ValueError("Cannot build APRS message packet without target") + if not event.message: + raise ValueError("Cannot build APRS message packet without message text") + + target = event.target.strip().upper()[:9].ljust(9) + text = event.message[:67] + payload = f":{target}:{text}" + if event.message_id: + payload += f"{{{event.message_id[:5]}" + header = _build_header(event.id, destination, path) + return f"{header}:{payload}" __all__ = [ "build_message_packet", diff --git a/aprs/generator.py b/aprs/generator.py deleted file mode 100644 index c6d10d5..0000000 --- a/aprs/generator.py +++ /dev/null @@ -1,110 +0,0 @@ -from __future__ import annotations - -"""Legacy APRS packet builder implementation behind aprs.encoder.""" - -import time - -from core.model import MessageEvent, PositionEvent - - -DEFAULT_SYMBOL_TABLE = "/" -DEFAULT_SYMBOL = ">" - - -def _coalesce_symbol_table(event: PositionEvent) -> str: - return event.symbol_table or DEFAULT_SYMBOL_TABLE - - -def _coalesce_symbol_code(event: PositionEvent) -> str: - return event.symbol_code or DEFAULT_SYMBOL - - -def _format_lat(lat: float) -> str: - hemi = "N" if lat >= 0 else "S" - lat = abs(lat) - degrees = int(lat) - minutes = (lat - degrees) * 60 - return f"{degrees:02d}{minutes:05.2f}{hemi}" - - -def _format_lon(lon: float) -> str: - hemi = "E" if lon >= 0 else "W" - lon = abs(lon) - degrees = int(lon) - minutes = (lon - degrees) * 60 - return f"{degrees:03d}{minutes:05.2f}{hemi}" - - -def _build_header(source: str, destination: str, path: str | None = None) -> str: - header = f"{source}>{destination}" - if path: - cleaned = path.strip() - if cleaned: - header += f",{cleaned}" - return header - - -def _format_object_timestamp(timestamp: int | float) -> str: - utc = time.gmtime(timestamp) - return time.strftime("%d%H%Mz", utc) - - -def build_position_packet( - event: PositionEvent, - destination: str = "APRS", - path: str | None = None, -) -> str: - if event.lat is None or event.lon is None: - raise ValueError("Cannot build APRS position packet without lat/lon") - - comment = (event.message or "")[:67] - payload = ( - f"!{_format_lat(event.lat)}{_coalesce_symbol_table(event)}" - f"{_format_lon(event.lon)}{_coalesce_symbol_code(event)}{comment}" - ) - header = _build_header(event.id, destination, path) - return f"{header}:{payload}" - - -def build_object_packet( - event: PositionEvent, - *, - source: str, - destination: str = "APRS", - path: str | None = None, -) -> str: - if event.lat is None or event.lon is None: - raise ValueError("Cannot build APRS object packet without lat/lon") - - object_name = (event.object_name or event.id or "").strip()[:9].ljust(9) - comment = event.message or "" - if event.tactical_name: - comment = f"{comment}[:{event.tactical_name}" if comment else f"[:{event.tactical_name}" - comment = comment[:67] - - payload = ( - f";{object_name}*{_format_object_timestamp(event.timestamp)}" - f"{_format_lat(event.lat)}{_coalesce_symbol_table(event)}" - f"{_format_lon(event.lon)}{_coalesce_symbol_code(event)}{comment}" - ) - header = _build_header(source, destination, path) - return f"{header}:{payload}" - - -def build_message_packet( - event: MessageEvent, - destination: str = "APRS", - path: str | None = None, -) -> str: - if not event.target: - raise ValueError("Cannot build APRS message packet without target") - if not event.message: - raise ValueError("Cannot build APRS message packet without message text") - - target = event.target.strip().upper()[:9].ljust(9) - text = event.message[:67] - payload = f":{target}:{text}" - if event.message_id: - payload += f"{{{event.message_id[:5]}" - header = _build_header(event.id, destination, path) - return f"{header}:{payload}" diff --git a/aprs/is_server.py b/aprs/is_server.py index 43e0190..40938ac 100644 --- a/aprs/is_server.py +++ b/aprs/is_server.py @@ -4,7 +4,7 @@ import logging from dataclasses import dataclass -from aprs.generator import build_message_packet, build_object_packet, build_position_packet +from aprs.encoder import build_message_packet, build_object_packet, build_position_packet from aprs.parser import APRSParseError, parse_aprs_event from core.model import MessageEvent, PositionEvent, Source from core.router import EventRouter diff --git a/aprs/parser.py b/aprs/parser.py index 07183fe..14f3ad0 100644 --- a/aprs/parser.py +++ b/aprs/parser.py @@ -1,11 +1,13 @@ from __future__ import annotations +"""APRS text parser producing runtime events plus optional canonical metadata.""" + import logging import re from datetime import UTC, datetime from core.canonical import empty_canonical_event, iso_utc_now -from core.model import MessageEvent, PositionEvent, Source +from core.model import MessageEvent, PositionEvent, Source, build_raw_payload log = logging.getLogger("aprs.parser") @@ -51,6 +53,7 @@ def parse_aprs_event( ingress_adapter: str | None = None, raw_context: dict[str, object] | None = None, ) -> PositionEvent | MessageEvent: + """Parse an APRS frame into the runtime model.""" canonical = parse_aprs_canonical(frame) event = canonical_to_aprs_event( canonical, @@ -72,6 +75,11 @@ def parse_aprs_frame(frame: str) -> PositionEvent | MessageEvent: return parse_aprs_event(frame) +def decode_aprs_frame(frame: str) -> PositionEvent | MessageEvent: + """Operator-friendly alias for APRS RX decoding.""" + return parse_aprs_event(frame) + + def parse_aprs_canonical(frame: str) -> dict[str, object]: frame = frame.strip() match = APRS_FRAME_RE.match(frame) @@ -151,11 +159,11 @@ def canonical_to_aprs_event( if not source_id: raise APRSParseError("canonical APRS event missing source callsign") - raw_payload: dict[str, object] = {"canonical": canonical} - if raw_context: - raw_payload.update(raw_context) - if ingress_adapter: - raw_payload["ingress_adapter"] = ingress_adapter + raw_payload = build_raw_payload( + canonical=canonical, + ingress_adapter=ingress_adapter, + raw_context=raw_context, + ) if text["text"]: return MessageEvent( @@ -176,7 +184,6 @@ def canonical_to_aprs_event( type=str(meta["source_subtype"] or "unit"), lat=float(position["lat"]), lon=float(position["lon"]), - source=Source.APRS, tactical_name=str(identity["short_name"]) if identity["short_name"] else None, message=str(text["comment"]) if text["comment"] else None, symbol_table=str(canonical["comms"]["symbol_table"]) if canonical["comms"]["symbol_table"] else None, @@ -187,6 +194,7 @@ def canonical_to_aprs_event( owner_id=str(canonical["sartrack"]["custom"].get("owner_callsign")) if isinstance(canonical["sartrack"]["custom"], dict) and canonical["sartrack"]["custom"].get("owner_callsign") else None, + source=source, raw=raw_payload, ) diff --git a/config.py b/config.py index 6398374..16121f6 100644 --- a/config.py +++ b/config.py @@ -53,7 +53,11 @@ SARTRACK_APRS = {"enabled": False, "host": "127.0.0.1", "port": 10152, "reconnect_delay": 5.0} -COT = {'enabled': False} +COT = {'enabled': True, + 'cot_url': 'udp://127.0.0.1:4242', + 'pytak_no_hello': True, + 'tak_proto': 0, + 'stale_seconds': 300} GUI = { "enabled": True, @@ -74,7 +78,7 @@ "aprs.parser": True, "sartrack.tcp": False, "meshtastic": False, - "cot": False, + "cot": True, "kiss": False, }, } @@ -206,6 +210,10 @@ def load_config() -> GatewayConfig: ), cot=CoTConfig( enabled=bool(COT["enabled"]), + cot_url=str(COT.get("cot_url", "udp://127.0.0.1:4242")), + pytak_no_hello=bool(COT.get("pytak_no_hello", True)), + tak_proto=int(COT.get("tak_proto", 0)), + stale_seconds=int(COT.get("stale_seconds", 300)), ), gui=GuiConfig( enabled=bool(GUI["enabled"]), @@ -293,6 +301,10 @@ def persist_sartrack_aprs_config(config: SARTrackAPRSServerConfig) -> None: def persist_cot_config(config: CoTConfig) -> None: values = { "enabled": config.enabled, + "cot_url": config.cot_url, + "pytak_no_hello": config.pytak_no_hello, + "tak_proto": config.tak_proto, + "stale_seconds": config.stale_seconds, } _replace_top_level_assignments({"COT": values}) diff --git a/core/canonical.py b/core/canonical.py index 91ec8ed..6063351 100644 --- a/core/canonical.py +++ b/core/canonical.py @@ -5,6 +5,12 @@ def empty_canonical_event() -> dict[str, Any]: + """Return the normalization schema used between protocol parsers and model events. + + This dict is not the router runtime language. The router and adapters should exchange + `PositionEvent` / `MessageEvent` from `core.model`, while protocol-specific parsers and + encoders may use this schema as a temporary normalization envelope. + """ return { "meta": { "source_format": None, diff --git a/core/model.py b/core/model.py index c01e489..713b5fc 100644 --- a/core/model.py +++ b/core/model.py @@ -1,5 +1,7 @@ from __future__ import annotations +"""Primary runtime event model used by the router and all adapters.""" + import hashlib import time from dataclasses import dataclass, field @@ -17,6 +19,8 @@ class Source(str, Enum): @dataclass(slots=True) class BaseEvent: + """Shared metadata for all runtime events flowing through the router.""" + id: str timestamp: int = field(default_factory=lambda: int(time.time())) source: Source = Source.INTERNAL @@ -42,6 +46,8 @@ def _dedup_payload(self) -> str: @dataclass(slots=True) class PositionEvent(BaseEvent): + """Normalized location-bearing event for stations, objects and future markers/tasks.""" + entity_kind: str = "station" type: str = "unit" lat: float | None = None @@ -87,6 +93,8 @@ def is_station(self) -> bool: @dataclass(slots=True) class MessageEvent(BaseEvent): + """Normalized text message event routed between protocol adapters.""" + message: str = "" target: str | None = None message_id: str | None = None @@ -103,3 +111,20 @@ def dedup_key(self) -> str: if self.message_id: return f"{self.kind}:{self.id}:{self.target or ''}:{self.message_id}" return f"{self.kind}:{self.id}:{self.target or ''}:{self.message}" + + +def build_raw_payload( + *, + canonical: dict[str, Any] | None = None, + ingress_adapter: str | None = None, + raw_context: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Build a consistent raw payload envelope stored on runtime events.""" + payload: dict[str, Any] = {} + if canonical is not None: + payload["canonical"] = canonical + if raw_context: + payload.update(raw_context) + if ingress_adapter: + payload["ingress_adapter"] = ingress_adapter + return payload diff --git a/core/state.py b/core/state.py index bd0424d..3584f73 100644 --- a/core/state.py +++ b/core/state.py @@ -1,5 +1,7 @@ from __future__ import annotations +"""In-memory telemetry store used by the GUI and operator-facing diagnostics.""" + import asyncio import base64 import logging @@ -27,6 +29,8 @@ class ComponentStatus: class StateStore: + """Keep recent events, logs and component status snapshots for the operator panel.""" + def __init__( self, *, @@ -107,6 +111,8 @@ def record_event( if isinstance(custom, dict): record["object_timestamp_raw"] = custom.get("object_timestamp_raw") record["object_timestamp_utc"] = custom.get("object_timestamp_utc") + record["display_type"] = _event_display_type(record) + record["display_label"] = _event_display_label(record) self._events.append(record) self._counters["events_total"] += 1 if event.kind == "position": @@ -255,3 +261,61 @@ def _json_safe(value: Any) -> Any: pass return repr(value) + + +def _event_display_type(record: dict[str, Any]) -> str: + subtype = str(record.get("event_subtype") or record.get("position_type") or "").strip().lower() + target = str(record.get("target") or "").strip().upper() + message = str(record.get("message") or "").strip().upper() + + if subtype in { + "station", + "object", + "private_message", + "encrypted_message", + "group_message", + "bulletin", + "ack", + "ping", + "query", + "status_response", + "task", + "status", + }: + return subtype.replace("_", " ") + + if record.get("entity_kind") == "object" or record.get("position_type") == "object": + return "object" + + if record.get("kind") == "position": + return "station" + + if record.get("kind") == "message": + if target.startswith("BLN"): + return "bulletin" + if target in {"GRUPA", "GROUP", "ALL"}: + return "group message" + if message.startswith("?"): + return "query" + if message.startswith("PING"): + return "ping" + if target: + return "private message" + return "message" + + return str(record.get("kind") or "event") + + +def _event_display_label(record: dict[str, Any]) -> str: + display_type = _event_display_type(record) + is_object = display_type == "object" + + if is_object: + return str(record.get("tactical_name") or record.get("object_name") or record.get("id") or "-") + + if record.get("kind") == "message": + source_id = str(record.get("id") or "-") + target = str(record.get("target") or "").strip() + return f"{source_id} -> {target}" if target else source_id + + return str(record.get("id") or "-") diff --git a/cot/__init__.py b/cot/__init__.py index 994fd63..60b4459 100644 --- a/cot/__init__.py +++ b/cot/__init__.py @@ -1,6 +1,6 @@ from cot.adapter import CoTAdapter, CoTConfig, build_cot_event from cot.encoder import encode_event_to_cot_xml, position_event_to_canonical -from cot.parser import CoTParseError, canonical_to_cot_event, parse_cot_canonical, parse_cot_xml +from cot.parser import CoTParseError, canonical_to_cot_event, decode_cot_xml, parse_cot_canonical, parse_cot_xml __all__ = [ "CoTAdapter", @@ -8,6 +8,7 @@ "CoTParseError", "build_cot_event", "canonical_to_cot_event", + "decode_cot_xml", "encode_event_to_cot_xml", "parse_cot_canonical", "parse_cot_xml", diff --git a/cot/cot.py b/cot/cot.py index fc04667..7647908 100644 --- a/cot/cot.py +++ b/cot/cot.py @@ -1,16 +1,28 @@ from __future__ import annotations +import asyncio +import configparser import logging from dataclasses import dataclass +from urllib.parse import urlparse from core.model import MessageEvent, PositionEvent, Source from core.state import StateStore -from cot.cot_encoder import encode_event_to_cot_xml, position_event_to_canonical +from cot.encoder import encode_event_to_cot_xml, position_event_to_canonical + +try: + import pytak +except ImportError: # pragma: no cover - exercised indirectly in runtime + pytak = None @dataclass(slots=True) class CoTConfig: enabled: bool = True + cot_url: str = "udp://127.0.0.1:4242" + pytak_no_hello: bool = True + tak_proto: int = 0 + stale_seconds: int = 300 class CoTAdapter: @@ -20,6 +32,9 @@ def __init__(self, config: CoTConfig | None = None) -> None: self.config = config or CoTConfig() self.log = logging.getLogger("cot") self.state_store: StateStore | None = None + self._tool: object | None = None + self._tx_queue: asyncio.Queue[bytes] | None = None + self._run_task: asyncio.Task[None] | None = None def attach_state_store(self, state_store: StateStore) -> None: self.state_store = state_store @@ -27,47 +42,160 @@ def attach_state_store(self, state_store: StateStore) -> None: async def connect(self) -> None: if not self.config.enabled: - self.log.info("CoT adapter disabled") + self.log.info("STATE | disabled | adapter=cot") if self.state_store: self.state_store.update_status("cot", "disabled", "adapter disabled") return - self.log.info("CoT adapter stub ready") - if self.state_store: - self.state_store.update_status("cot", "online", "stub ready") + + if pytak is None: + self.log.warning("STATE | unavailable | adapter=cot reason=pytak-not-installed") + if self.state_store: + self.state_store.update_status("cot", "error", "PyTAK not installed") + return + + await self._close_transport() + + try: + effective_cot_url = _effective_cot_url(self.config.cot_url) + if self.state_store: + self.state_store.update_status("cot", "connecting", effective_cot_url) + tool = pytak.CLITool(_build_pytak_config(self.config)) + await tool.setup() + self._tool = tool + self._tx_queue = tool.tx_queue + self._run_task = asyncio.create_task(self._run_tool(tool)) + self.log.info( + "STATE | online | adapter=cot cot_url=%s effective_cot_url=%s tak_proto=%s no_hello=%s", + self.config.cot_url, + effective_cot_url, + self.config.tak_proto, + self.config.pytak_no_hello, + ) + if self.state_store: + self.state_store.update_status("cot", "online", effective_cot_url) + except Exception as exc: + self.log.warning("STATE | error | adapter=cot cot_url=%s error=%s", self.config.cot_url, exc) + if self.state_store: + self.state_store.update_status("cot", "error", str(exc)) + await self._close_transport() async def send(self, event: PositionEvent | MessageEvent) -> None: if not self.config.enabled: - self.log.debug("CoT send skipped: adapter disabled") + self.log.debug("TX | skip | adapter=cot reason=disabled") return if not isinstance(event, PositionEvent): - self.log.debug("CoT send skipped: unsupported event type %s", type(event).__name__) + self.log.debug("TX | skip | adapter=cot reason=unsupported-event event_type=%s", type(event).__name__) return if not event.is_position(): - self.log.debug("CoT send skipped: no position") + self.log.debug("TX | skip | adapter=cot reason=no-position event_id=%s", event.id) + return + if not self._tx_queue: + self.log.warning("TX | skip | adapter=cot reason=not-connected event_id=%s", event.id) return + + if self.config.stale_seconds > event.ttl: + event = PositionEvent( + id=event.id, + timestamp=event.timestamp, + source=event.source, + ttl=self.config.stale_seconds, + entity_kind=event.entity_kind, + type=event.type, + lat=event.lat, + lon=event.lon, + alt=event.alt, + speed=event.speed, + heading=event.heading, + tactical_name=event.tactical_name, + message=event.message, + symbol_table=event.symbol_table, + symbol_code=event.symbol_code, + object_name=event.object_name, + owner_id=event.owner_id, + ) xml = build_cot_event(event) - self.log.info("CoT stub send: %s", xml) + self.log.info( + "TX | cot | adapter=cot entity=%s id=%s cot_url=%s stale_seconds=%s", + event.entity_kind, + event.id, + self.config.cot_url, + event.ttl, + ) + self.log.debug("TX | cot-xml | adapter=cot payload=%s", xml) + await self._tx_queue.put(xml.encode("utf-8")) def get_runtime_config(self) -> dict[str, object]: return { "enabled": self.config.enabled, + "cot_url": self.config.cot_url, + "pytak_no_hello": self.config.pytak_no_hello, + "tak_proto": self.config.tak_proto, + "stale_seconds": self.config.stale_seconds, } async def apply_runtime_config(self, updates: dict[str, object]) -> dict[str, object]: if "enabled" in updates: self.config.enabled = bool(updates["enabled"]) + if "cot_url" in updates and updates["cot_url"] is not None: + self.config.cot_url = str(updates["cot_url"]) + if "pytak_no_hello" in updates: + self.config.pytak_no_hello = bool(updates["pytak_no_hello"]) + if "tak_proto" in updates and updates["tak_proto"] is not None: + self.config.tak_proto = int(updates["tak_proto"]) + if "stale_seconds" in updates and updates["stale_seconds"] is not None: + self.config.stale_seconds = int(updates["stale_seconds"]) + + await self._close_transport() if self.state_store: - state = "disabled" if not self.config.enabled else "online" + state = "disabled" if not self.config.enabled else "reconnecting" detail = "runtime config updated" if self.config.enabled else "adapter disabled" self.state_store.update_status("cot", state, detail) + if self.config.enabled: + await self.connect() return self.get_runtime_config() async def reconnect_now(self) -> None: + await self._close_transport() if self.state_store: - state = "disabled" if not self.config.enabled else "online" + state = "disabled" if not self.config.enabled else "reconnecting" detail = "manual reconnect requested" if self.config.enabled else "adapter disabled" self.state_store.update_status("cot", state, detail) - self.log.info("CoT reconnect requested (stub)") + if self.config.enabled: + await self.connect() + + async def _run_tool(self, tool: object) -> None: + try: + await tool.run() + except asyncio.CancelledError: + self.log.debug("STATE | stopped | adapter=cot reason=task-cancelled") + raise + except Exception as exc: + self.log.warning("STATE | transport-error | adapter=cot error=%s", exc) + if self.state_store: + self.state_store.update_status("cot", "error", str(exc)) + finally: + if self._tool is tool: + self._tool = None + self._tx_queue = None + self._run_task = None + if self.state_store and self.config.enabled: + self.state_store.update_status("cot", "offline", "transport stopped") + + async def _close_transport(self) -> None: + task = self._run_task + self._run_task = None + self._tool = None + self._tx_queue = None + + if task is not None: + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + if self.state_store and self.config.enabled: + self.state_store.update_status("cot", "offline", "transport stopped") def build_cot_event(event: PositionEvent) -> str: @@ -80,3 +208,22 @@ def build_cot_event(event: PositionEvent) -> str: canonical["tak"]["cot_type"] or "a-f-G-U-C", ) return encode_event_to_cot_xml(event) + + +def _build_pytak_config(config: CoTConfig) -> configparser.SectionProxy: + parser = configparser.ConfigParser() + section_name = "ham-router-cot" + parser.add_section(section_name) + parser.set(section_name, "COT_URL", _effective_cot_url(config.cot_url)) + parser.set(section_name, "PYTAK_NO_HELLO", "1" if config.pytak_no_hello else "0") + parser.set(section_name, "TAK_PROTO", str(config.tak_proto)) + return parser[section_name] + + +def _effective_cot_url(cot_url: str) -> str: + parsed = urlparse(cot_url) + scheme = parsed.scheme.lower() + if "udp" in scheme and "+wo" not in scheme: + effective_scheme = f"{scheme}+wo" + return parsed._replace(scheme=effective_scheme).geturl() + return cot_url diff --git a/cot/cot_encoder.py b/cot/cot_encoder.py deleted file mode 100644 index abcda57..0000000 --- a/cot/cot_encoder.py +++ /dev/null @@ -1,82 +0,0 @@ -from __future__ import annotations - -"""Legacy CoT encoding implementation behind cot.encoder.""" - -import time -import xml.etree.ElementTree as ET - -from core.canonical import empty_canonical_event -from core.model import PositionEvent - - -def encode_event_to_cot_xml(event: PositionEvent) -> str: - canonical = position_event_to_canonical(event) - position = canonical["position"] - tak = canonical["tak"] - identity = canonical["identity"] - text = canonical["text"] - - now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp)) - stale = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp + event.ttl)) - - root = ET.Element( - "event", - { - "version": str(tak["cot_version"] or "2.0"), - "uid": str(identity["uid"] or event.id), - "type": str(tak["cot_type"] or "a-f-G-U-C"), - "time": str(position["timestamp_utc"] or now), - "start": str(tak["start_utc"] or now), - "stale": str(tak["stale_utc"] or stale), - "how": str(tak["how"] or "m-g"), - }, - ) - if position["lat"] is not None and position["lon"] is not None: - ET.SubElement( - root, - "point", - { - "lat": str(position["lat"]), - "lon": str(position["lon"]), - "hae": str(position["alt_m"] or 0), - "ce": str(position["ce_m"] or 9999999), - "le": str(position["le_m"] or 9999999), - }, - ) - detail = ET.SubElement(root, "detail") - remarks = ET.SubElement(detail, "remarks") - remarks.text = str(text["remarks"] or text["comment"] or "") - return ET.tostring(root, encoding="unicode") - - -def position_event_to_canonical(event: PositionEvent) -> dict[str, object]: - canonical = empty_canonical_event() - canonical["meta"]["source_format"] = "cot_xml" - canonical["meta"]["source_subtype"] = "position" - canonical["meta"]["parser_version"] = 1 - canonical["identity"]["uid"] = event.object_name or event.id - canonical["identity"]["callsign"] = event.owner_id or event.id - canonical["identity"]["short_name"] = event.tactical_name or event.object_name or event.id - canonical["identity"]["object_type"] = "unknown" if event.is_object() else "person" - canonical["position"]["lat"] = event.lat - canonical["position"]["lon"] = event.lon - canonical["position"]["alt_m"] = event.alt - canonical["position"]["ce_m"] = 9999999 - canonical["position"]["le_m"] = 9999999 - canonical["position"]["timestamp_utc"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp)) - canonical["text"]["comment"] = event.message - if event.is_object(): - parts = [f"APRS object {event.object_name or event.id}"] - if event.owner_id: - parts.append(f"owner {event.owner_id}") - if event.message: - parts.append(event.message) - canonical["text"]["remarks"] = " | ".join(parts) - else: - canonical["text"]["remarks"] = event.message - canonical["tak"]["cot_version"] = "2.0" - canonical["tak"]["cot_type"] = "b-m-p-s-p-loc" if event.is_object() else "a-f-G-U-C" - canonical["tak"]["how"] = "m-g" - canonical["tak"]["start_utc"] = canonical["position"]["timestamp_utc"] - canonical["tak"]["stale_utc"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp + event.ttl)) - return canonical diff --git a/cot/cot_parser.py b/cot/cot_parser.py deleted file mode 100644 index 72f8921..0000000 --- a/cot/cot_parser.py +++ /dev/null @@ -1,133 +0,0 @@ -from __future__ import annotations - -import re -import xml.etree.ElementTree as ET - -from core.canonical import empty_canonical_event -from core.model import PositionEvent, Source - - -class CoTParseError(ValueError): - pass - - -OBJECT_REMARKS_RE = re.compile( - r"^APRS object (?P[^|]+?)(?: \| owner (?P[^|]+?))?(?: \| (?P.*))?$" -) - - -def parse_cot_xml(xml_text: str) -> PositionEvent: - canonical = parse_cot_canonical(xml_text) - return canonical_to_cot_event(canonical, raw={"cot_xml": xml_text}) - - -def parse_cot_canonical(xml_text: str) -> dict[str, object]: - try: - root = ET.fromstring(xml_text) - except ET.ParseError as exc: - raise CoTParseError(str(exc)) from exc - - if root.tag != "event": - raise CoTParseError(f"unsupported root tag: {root.tag}") - - point = root.find("point") - if point is None: - raise CoTParseError("missing point element") - - uid = root.attrib.get("uid") - if not uid: - raise CoTParseError("missing uid") - - try: - lat = float(point.attrib["lat"]) - lon = float(point.attrib["lon"]) - except (KeyError, ValueError) as exc: - raise CoTParseError("invalid point coordinates") from exc - - detail = root.find("detail") - remarks = None - if detail is not None: - remarks_elem = detail.find("remarks") - if remarks_elem is not None and remarks_elem.text: - remarks = remarks_elem.text.strip() or None - - cot_type = root.attrib.get("type", "") - entity_kind = "object" if cot_type == "b-m-p-s-p-loc" else "station" - canonical = empty_canonical_event() - canonical["meta"]["source_format"] = "cot_xml" - canonical["meta"]["source_subtype"] = "object" if entity_kind == "object" else "position" - canonical["meta"]["parser_version"] = 1 - canonical["meta"]["raw_message"] = xml_text - canonical["identity"]["uid"] = uid - canonical["identity"]["callsign"] = uid - canonical["identity"]["short_name"] = uid - canonical["identity"]["object_type"] = "unknown" if entity_kind == "object" else "person" - canonical["position"]["lat"] = lat - canonical["position"]["lon"] = lon - canonical["position"]["alt_m"] = _optional_float(point.attrib.get("hae")) - canonical["position"]["ce_m"] = _optional_float(point.attrib.get("ce")) - canonical["position"]["le_m"] = _optional_float(point.attrib.get("le")) - canonical["position"]["timestamp_utc"] = root.attrib.get("time") - canonical["text"]["remarks"] = remarks - canonical["text"]["comment"] = remarks - canonical["tak"]["cot_version"] = root.attrib.get("version") - canonical["tak"]["cot_type"] = cot_type or None - canonical["tak"]["how"] = root.attrib.get("how") - canonical["tak"]["start_utc"] = root.attrib.get("start") - canonical["tak"]["stale_utc"] = root.attrib.get("stale") - return canonical - - -def canonical_to_cot_event( - canonical: dict[str, object], - *, - raw: dict[str, object] | None = None, -) -> PositionEvent: - identity = canonical["identity"] - position = canonical["position"] - text = canonical["text"] - tak = canonical["tak"] - - uid = identity["uid"] - if not uid: - raise CoTParseError("canonical CoT event missing uid") - if position["lat"] is None or position["lon"] is None: - raise CoTParseError("canonical CoT event missing coordinates") - - cot_type = str(tak["cot_type"] or "") - entity_kind = "object" if cot_type == "b-m-p-s-p-loc" else "station" - position_type = "object" if entity_kind == "object" else "position" - remarks = str(text["remarks"]) if text["remarks"] else None - object_name = None - owner_id = None - message = remarks - - if entity_kind == "object" and remarks: - match = OBJECT_REMARKS_RE.match(remarks) - if match: - object_name = (match.group("object_name") or "").strip() or None - owner_id = (match.group("owner_id") or "").strip() or None - message = (match.group("message") or "").strip() or None - - return PositionEvent( - id=str(object_name or uid), - entity_kind=entity_kind, - type=position_type, - lat=float(position["lat"]), - lon=float(position["lon"]), - alt=_optional_float(position.get("alt_m")), - message=message, - object_name=object_name, - owner_id=owner_id, - source=Source.COT, - raw=raw or {"canonical": canonical}, - ) - - -def _optional_float(value: object) -> float | None: - if value in (None, ""): - return None - try: - return float(value) - except (TypeError, ValueError): - return None diff --git a/cot/encoder.py b/cot/encoder.py index c2d3adf..76f89fe 100644 --- a/cot/encoder.py +++ b/cot/encoder.py @@ -1,4 +1,120 @@ -from cot.cot_encoder import encode_event_to_cot_xml, position_event_to_canonical +from __future__ import annotations + +"""CoT XML encoder for runtime position events.""" + +import time +import xml.etree.ElementTree as ET + +from core.canonical import empty_canonical_event +from core.model import PositionEvent + + +def encode_event_to_cot_xml(event: PositionEvent) -> str: + canonical = position_event_to_canonical(event) + position = canonical["position"] + tak = canonical["tak"] + identity = canonical["identity"] + text = canonical["text"] + + now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp)) + stale = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp + event.ttl)) + + root = ET.Element( + "event", + { + "version": str(tak["cot_version"] or "2.0"), + "uid": str(identity["uid"] or event.id), + "type": str(tak["cot_type"] or "a-f-G-U-C"), + "time": str(position["timestamp_utc"] or now), + "start": str(tak["start_utc"] or now), + "stale": str(tak["stale_utc"] or stale), + "how": str(tak["how"] or "m-g"), + }, + ) + if position["lat"] is not None and position["lon"] is not None: + ET.SubElement( + root, + "point", + { + "lat": str(position["lat"]), + "lon": str(position["lon"]), + "hae": str(position["alt_m"] or 0), + "ce": str(position["ce_m"] or 9999999), + "le": str(position["le_m"] or 9999999), + }, + ) + detail = ET.SubElement(root, "detail") + ET.SubElement( + detail, + "contact", + { + "callsign": str(tak["contact_callsign"] or identity["short_name"] or identity["uid"] or event.id), + }, + ) + ET.SubElement( + detail, + "__group", + { + "name": str(tak["tak_group_name"] or "Ham Router"), + "role": str(tak["tak_group_role"] or _default_group_role(event)), + }, + ) + if event.is_object(): + object_attrs = { + "name": str(event.object_name or event.id), + } + if event.owner_id: + object_attrs["owner"] = str(event.owner_id) + if event.tactical_name: + object_attrs["tactical_name"] = str(event.tactical_name) + if event.symbol_table: + object_attrs["symbol_table"] = str(event.symbol_table) + if event.symbol_code: + object_attrs["symbol_code"] = str(event.symbol_code) + ET.SubElement(detail, "ham_router_object", object_attrs) + remarks = ET.SubElement(detail, "remarks") + remarks.text = str(text["remarks"] or text["comment"] or "") + return ET.tostring(root, encoding="unicode") + + +def position_event_to_canonical(event: PositionEvent) -> dict[str, object]: + canonical = empty_canonical_event() + canonical["meta"]["source_format"] = "cot_xml" + canonical["meta"]["source_subtype"] = "position" + canonical["meta"]["parser_version"] = 1 + canonical["identity"]["uid"] = event.object_name or event.id + canonical["identity"]["callsign"] = event.owner_id or event.id + canonical["identity"]["short_name"] = event.tactical_name or event.object_name or event.id + canonical["identity"]["object_type"] = "unknown" if event.is_object() else "person" + canonical["position"]["lat"] = event.lat + canonical["position"]["lon"] = event.lon + canonical["position"]["alt_m"] = event.alt + canonical["position"]["ce_m"] = 9999999 + canonical["position"]["le_m"] = 9999999 + canonical["position"]["timestamp_utc"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp)) + canonical["text"]["comment"] = event.message + canonical["text"]["remarks"] = event.message + canonical["tak"]["cot_version"] = "2.0" + canonical["tak"]["cot_type"] = "b-m-p-w" if event.is_object() else "a-f-G-U-C" + canonical["tak"]["how"] = "m-g" + canonical["tak"]["start_utc"] = canonical["position"]["timestamp_utc"] + canonical["tak"]["stale_utc"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp + event.ttl)) + canonical["tak"]["contact_callsign"] = _contact_callsign(event) + canonical["tak"]["tak_group_name"] = "Ham Router" + canonical["tak"]["tak_group_role"] = _default_group_role(event) + return canonical + + +def _contact_callsign(event: PositionEvent) -> str: + if event.is_object(): + return event.tactical_name or event.object_name or event.id + return event.tactical_name or event.id + + +def _default_group_role(event: PositionEvent) -> str: + if event.is_object(): + return "Waypoint" + return "Team Member" __all__ = [ "encode_event_to_cot_xml", diff --git a/cot/parser.py b/cot/parser.py index c7439db..ca79492 100644 --- a/cot/parser.py +++ b/cot/parser.py @@ -1,8 +1,158 @@ -from cot.cot_parser import CoTParseError, canonical_to_cot_event, parse_cot_canonical, parse_cot_xml +from __future__ import annotations + +"""CoT XML parser producing runtime position events plus canonical metadata.""" + +import xml.etree.ElementTree as ET + +from core.canonical import empty_canonical_event +from core.model import PositionEvent, Source, build_raw_payload + + +class CoTParseError(ValueError): + pass + + +OBJECT_COT_TYPES = {"b-m-p-s-p-loc", "b-m-p-w"} + + +def parse_cot_xml(xml_text: str) -> PositionEvent: + canonical = parse_cot_canonical(xml_text) + return canonical_to_cot_event(canonical, raw={"cot_xml": xml_text}) + + +def decode_cot_xml(xml_text: str) -> PositionEvent: + """Operator-friendly alias for CoT RX decoding.""" + return parse_cot_xml(xml_text) + + +def parse_cot_canonical(xml_text: str) -> dict[str, object]: + try: + root = ET.fromstring(xml_text) + except ET.ParseError as exc: + raise CoTParseError(str(exc)) from exc + + if root.tag != "event": + raise CoTParseError(f"unsupported root tag: {root.tag}") + + point = root.find("point") + if point is None: + raise CoTParseError("missing point element") + + uid = root.attrib.get("uid") + if not uid: + raise CoTParseError("missing uid") + + try: + lat = float(point.attrib["lat"]) + lon = float(point.attrib["lon"]) + except (KeyError, ValueError) as exc: + raise CoTParseError("invalid point coordinates") from exc + + detail = root.find("detail") + remarks = None + object_meta: dict[str, str] = {} + if detail is not None: + remarks_elem = detail.find("remarks") + if remarks_elem is not None and remarks_elem.text: + remarks = remarks_elem.text.strip() or None + object_elem = detail.find("ham_router_object") + if object_elem is not None: + object_meta = {str(key): str(value) for key, value in object_elem.attrib.items()} + + cot_type = root.attrib.get("type", "") + entity_kind = "object" if cot_type in OBJECT_COT_TYPES else "station" + canonical = empty_canonical_event() + canonical["meta"]["source_format"] = "cot_xml" + canonical["meta"]["source_subtype"] = "object" if entity_kind == "object" else "position" + canonical["meta"]["parser_version"] = 1 + canonical["meta"]["raw_message"] = xml_text + canonical["identity"]["uid"] = uid + canonical["identity"]["callsign"] = uid + canonical["identity"]["short_name"] = uid + canonical["identity"]["object_type"] = "unknown" if entity_kind == "object" else "person" + canonical["position"]["lat"] = lat + canonical["position"]["lon"] = lon + canonical["position"]["alt_m"] = _optional_float(point.attrib.get("hae")) + canonical["position"]["ce_m"] = _optional_float(point.attrib.get("ce")) + canonical["position"]["le_m"] = _optional_float(point.attrib.get("le")) + canonical["position"]["timestamp_utc"] = root.attrib.get("time") + canonical["text"]["remarks"] = remarks + canonical["text"]["comment"] = remarks + canonical["tak"]["cot_version"] = root.attrib.get("version") + canonical["tak"]["cot_type"] = cot_type or None + canonical["tak"]["how"] = root.attrib.get("how") + canonical["tak"]["start_utc"] = root.attrib.get("start") + canonical["tak"]["stale_utc"] = root.attrib.get("stale") + canonical["sartrack"]["custom"] = object_meta or None + return canonical + + +def canonical_to_cot_event( + canonical: dict[str, object], + *, + raw: dict[str, object] | None = None, +) -> PositionEvent: + identity = canonical["identity"] + position = canonical["position"] + text = canonical["text"] + tak = canonical["tak"] + + uid = identity["uid"] + if not uid: + raise CoTParseError("canonical CoT event missing uid") + if position["lat"] is None or position["lon"] is None: + raise CoTParseError("canonical CoT event missing coordinates") + + cot_type = str(tak["cot_type"] or "") + entity_kind = "object" if cot_type in OBJECT_COT_TYPES else "station" + position_type = "object" if entity_kind == "object" else "position" + remarks = str(text["remarks"]) if text["remarks"] else None + object_name = None + owner_id = None + message = remarks + tactical_name = None + symbol_table = None + symbol_code = None + + custom = canonical.get("sartrack", {}).get("custom") + if entity_kind == "object" and isinstance(custom, dict): + object_name = str(custom.get("name") or "").strip() or None + owner_id = str(custom.get("owner") or "").strip() or None + tactical_name = str(custom.get("tactical_name") or "").strip() or None + symbol_table = str(custom.get("symbol_table") or "").strip() or None + symbol_code = str(custom.get("symbol_code") or "").strip() or None + + return PositionEvent( + id=str(object_name or uid), + entity_kind=entity_kind, + type=position_type, + lat=float(position["lat"]), + lon=float(position["lon"]), + alt=_optional_float(position.get("alt_m")), + message=message, + tactical_name=tactical_name, + object_name=object_name, + owner_id=owner_id, + symbol_table=symbol_table, + symbol_code=symbol_code, + source=Source.COT, + raw=build_raw_payload(canonical=canonical, raw_context=raw), + ) + + +def _optional_float(value: object) -> float | None: + if value in (None, ""): + return None + try: + return float(value) + except (TypeError, ValueError): + return None + __all__ = [ "CoTParseError", "canonical_to_cot_event", + "decode_cot_xml", "parse_cot_canonical", "parse_cot_xml", ] diff --git a/gui/server.py b/gui/server.py index 7628e0a..3a8a038 100644 --- a/gui/server.py +++ b/gui/server.py @@ -1,5 +1,7 @@ from __future__ import annotations +"""FastAPI-based local operator panel for status, recent events and basic test TX.""" + import asyncio import json import logging @@ -48,6 +50,8 @@ class ConnectionControl: class GuiServer: + """Expose operator-friendly HTTP endpoints and the single-page local GUI.""" + def __init__( self, config: GuiConfig, @@ -647,6 +651,10 @@ def _build_index_html(title: str) -> str: ${{renderField('tx_callsign', 'TX Callsign')}} ${{renderField('tx_destination', 'TX Destination')}} ${{renderField('tx_path', 'TX Path')}} + ${{editable.has('pytak_no_hello') ? `` : ''}} + ${{renderField('cot_url', 'CoT URL')}} + ${{renderField('tak_proto', 'TAK Proto', 'number')}} + ${{renderField('stale_seconds', 'Stale Seconds', 'number')}} ${{renderField('destination_id', 'Destination ID')}} ${{renderField('aprs_channel_index', 'Mesh Channel Index', 'number')}} ${{renderField('position_transport_mode', 'Position Mode')}} @@ -694,9 +702,9 @@ def _build_index_html(title: str) -> str: const root = document.getElementById('events'); root.innerHTML = ''; for (const item of items) {{ - const isObject = item.entity_kind === 'object' || item.position_type === 'object'; - const label = classifyEventLabel(item); - const displayId = buildDisplayId(item, isObject); + const isObject = item.display_type === 'object' || item.entity_kind === 'object' || item.position_type === 'object'; + const label = item.display_type || classifyEventLabel(item); + const displayId = item.display_label || buildDisplayId(item, isObject); const objectMeta = isObject ? ` | object=${{item.object_name || item.id}} | owner=${{item.owner_id || '-'}} | symbol=${{item.symbol_table || ''}}${{item.symbol_code || ''}} | ts=${{item.object_timestamp_raw || '-'}}` : ''; @@ -719,7 +727,7 @@ def _build_index_html(title: str) -> str: }} function classifyEventLabel(item) {{ - const subtype = String(item.event_subtype || item.position_type || '').toLowerCase(); + const subtype = String(item.display_type || item.event_subtype || item.position_type || '').toLowerCase(); const target = String(item.target || '').trim().toUpperCase(); const message = String(item.message || '').trim().toUpperCase(); diff --git a/main.py b/main.py index ce43245..d55abc8 100644 --- a/main.py +++ b/main.py @@ -180,7 +180,7 @@ async def update_cot(payload: dict[str, object]) -> dict[str, object]: get_config=cot.get_runtime_config, update_config=update_cot, reconnect=cot.reconnect_now, - editable_fields=("enabled",), + editable_fields=("enabled", "cot_url", "pytak_no_hello", "tak_proto", "stale_seconds"), ), } for mesh_adapter in mesh_adapters: diff --git a/mesh/__init__.py b/mesh/__init__.py index 05cc31d..a385bc1 100644 --- a/mesh/__init__.py +++ b/mesh/__init__.py @@ -1,8 +1,15 @@ from mesh.adapter import MeshtasticAdapter, MeshtasticConfig -from mesh.decoder import decode_mesh_packet, decode_mesh_position_packet, decode_mesh_text_packet from mesh.encoder import build_waypoint_fields, build_waypoint_id, encode_position_payload, encode_text_payload from mesh.filter import MeshFilter, MeshFilterDecision -from mesh.parser import MeshtasticParseError, canonical_to_event, parse_meshtastic_packet +from mesh.parser import ( + MeshtasticParseError, + canonical_to_mesh_event, + decode_mesh_packet, + decode_mesh_position_packet, + decode_mesh_text_packet, + decode_mesh_text_payload, + parse_meshtastic_packet, +) __all__ = [ "MeshtasticAdapter", @@ -12,10 +19,11 @@ "MeshtasticParseError", "build_waypoint_fields", "build_waypoint_id", - "canonical_to_event", + "canonical_to_mesh_event", "decode_mesh_packet", "decode_mesh_position_packet", "decode_mesh_text_packet", + "decode_mesh_text_payload", "encode_position_payload", "encode_text_payload", "parse_meshtastic_packet", diff --git a/mesh/adapter.py b/mesh/adapter.py index cfef67f..eb92fbb 100644 --- a/mesh/adapter.py +++ b/mesh/adapter.py @@ -15,10 +15,9 @@ from core.model import MessageEvent, PositionEvent, Source from core.router import EventRouter from core.state import StateStore -from mesh.mesh_decoder import decode_mesh_packet -from mesh.mesh_encoder import build_waypoint_fields, build_waypoint_id, encode_position_payload, encode_text_payload +from mesh.encoder import build_waypoint_fields, build_waypoint_id, encode_position_payload, encode_text_payload from mesh.mesh_filter import MeshFilter -from mesh.parser import parse_meshtastic_packet +from mesh.parser import decode_mesh_packet, parse_meshtastic_packet APRS_CALLSIGN_RE = re.compile(r"^[A-Z0-9]{1,6}(?:-[0-9]{1,2})?$") diff --git a/mesh/decoder.py b/mesh/decoder.py deleted file mode 100644 index 0c76d6f..0000000 --- a/mesh/decoder.py +++ /dev/null @@ -1,7 +0,0 @@ -from mesh.mesh_decoder import decode_mesh_packet, decode_mesh_position_packet, decode_mesh_text_packet - -__all__ = [ - "decode_mesh_packet", - "decode_mesh_position_packet", - "decode_mesh_text_packet", -] diff --git a/mesh/encoder.py b/mesh/encoder.py index aa2578d..2c451e0 100644 --- a/mesh/encoder.py +++ b/mesh/encoder.py @@ -1,4 +1,63 @@ -from mesh.mesh_encoder import build_waypoint_fields, build_waypoint_id, encode_position_payload, encode_text_payload +from __future__ import annotations + +"""Meshtastic payload encoder for compact APRS-derived mesh transport.""" + +from core.model import MessageEvent, PositionEvent + + +def encode_position_payload(event: PositionEvent) -> str: + parts = [ + "APRS_POS", + f"entity={event.entity_kind}", + f"id={event.id}", + f"name={event.tactical_name or ''}", + f"lat={event.lat}", + f"lon={event.lon}", + f"ts={event.timestamp}", + ] + if event.object_name: + parts.append(f"object={event.object_name}") + if event.owner_id: + parts.append(f"owner={event.owner_id}") + if event.symbol_table: + parts.append(f"symbol_table={event.symbol_table}") + if event.symbol_code: + parts.append(f"symbol_code={event.symbol_code}") + if event.message: + parts.append(f"msg={event.message}") + return "|".join(parts) + + +def encode_text_payload(event: MessageEvent) -> str: + if event.target: + if event.message_id: + return f"[APRS {event.id}->{event.target} #{event.message_id}] {event.message}" + return f"[APRS {event.id}->{event.target}] {event.message}" + + if event.message_id: + return f"[APRS {event.id} #{event.message_id}] {event.message}" + return f"[APRS {event.id}] {event.message}" + + +def build_waypoint_fields(event: PositionEvent) -> tuple[str, str]: + label_source = event.object_name or event.tactical_name or event.id + label = label_source.strip()[:30] + if event.is_object(): + description_parts = [f"Object {event.object_name or event.id}"] + if event.owner_id: + description_parts.append(f"owner {event.owner_id}") + else: + description_parts = [f"Station {event.id}"] + if event.message: + description_parts.append(event.message.strip()) + return label, " | ".join(description_parts)[:120] + + +def build_waypoint_id(source_id: str) -> int: + value = 0 + for char in source_id.upper(): + value = ((value * 33) + ord(char)) % 1_000_000_000 + return value or 1 __all__ = [ "build_waypoint_fields", diff --git a/mesh/mesh_decoder.py b/mesh/mesh_decoder.py deleted file mode 100644 index b64ca02..0000000 --- a/mesh/mesh_decoder.py +++ /dev/null @@ -1,160 +0,0 @@ -from __future__ import annotations - -"""Legacy Meshtastic decoder implementation behind mesh.decoder.""" - -import re -from typing import Any - -from core.model import MessageEvent, PositionEvent, Source -from mesh.parser import canonical_to_event, parse_meshtastic_packet - - -APRS_TEXT_RE = re.compile( - r"^\[APRS (?P.+?)(?:->(?P.+?))?(?: #(?P[^\]]+))?\] (?P.*)$" -) - - -def decode_mesh_packet( - packet: dict[str, Any], - *, - source_id: str, - target_id: str | None = None, -) -> PositionEvent | MessageEvent | None: - canonical = parse_meshtastic_packet(packet) - decoded = packet.get("decoded") or {} - portnum = canonical.get("comms", {}).get("portnum") - - if "position" in decoded or portnum == "POSITION_APP": - native_position = decode_mesh_position_packet(packet, source_id=source_id, canonical=canonical) - if native_position is not None: - return native_position - - text_event = decode_mesh_text_packet( - packet, - source_id=source_id, - target_id=target_id, - ) - if text_event is not None: - return text_event - - return canonical_to_event( - canonical, - source_id=source_id, - target_id=target_id, - raw=packet, - ) - - -def decode_mesh_text_packet( - packet: dict[str, Any], - *, - source_id: str, - target_id: str | None = None, -) -> PositionEvent | MessageEvent | None: - decoded = packet.get("decoded") or {} - text = decoded.get("text") - if not text: - return None - return decode_mesh_text_payload(str(text).strip(), source_id=source_id, target_id=target_id, raw=packet) - - -def decode_mesh_text_payload( - text: str, - *, - source_id: str, - target_id: str | None = None, - raw: dict[str, Any] | None = None, -) -> PositionEvent | MessageEvent | None: - if not text: - return None - - if text.startswith("APRS_POS|"): - values = _parse_pipe_payload(text) - event_id = values.get("id") or source_id - lat = _parse_float(values.get("lat")) - lon = _parse_float(values.get("lon")) - if lat is None or lon is None: - return None - return PositionEvent( - id=event_id, - entity_kind=values.get("entity") or ("object" if values.get("object") else "station"), - type="object" if values.get("entity") == "object" or values.get("object") else "position", - lat=lat, - lon=lon, - tactical_name=values.get("name") or None, - message=values.get("msg") or None, - object_name=values.get("object") or None, - owner_id=values.get("owner") or None, - symbol_table=values.get("symbol_table") or None, - symbol_code=values.get("symbol_code") or None, - source=Source.MESHTASTIC, - raw=raw or {"mesh_text": text}, - ) - - aprs_match = APRS_TEXT_RE.match(text) - if aprs_match: - event_source = (aprs_match.group("source") or source_id).strip() - event_target = (aprs_match.group("target") or target_id or "").strip() or None - event_message = (aprs_match.group("message") or "").strip() - return MessageEvent( - id=event_source, - target=event_target, - message=event_message, - message_id=(aprs_match.group("message_id") or "").strip() or None, - source=Source.MESHTASTIC, - raw=raw or {"mesh_text": text}, - ) - - return MessageEvent( - id=source_id, - target=target_id, - message=text, - source=Source.MESHTASTIC, - raw=raw or {"mesh_text": text}, - ) - - -def decode_mesh_position_packet( - packet: dict[str, Any], - *, - source_id: str, - canonical: dict[str, Any] | None = None, -) -> PositionEvent | None: - position = (canonical or {}).get("position", {}) - if not position: - canonical = canonical or parse_meshtastic_packet(packet) - position = canonical.get("position", {}) - - lat = position.get("lat") - lon = position.get("lon") - if lat is None or lon is None: - return None - return PositionEvent( - id=source_id, - entity_kind="station", - type="position", - lat=float(lat), - lon=float(lon), - alt=position.get("alt_m"), - source=Source.MESHTASTIC, - raw=packet, - ) - - -def _parse_pipe_payload(text: str) -> dict[str, str]: - values: dict[str, str] = {} - for part in text.split("|")[1:]: - if "=" not in part: - continue - key, value = part.split("=", 1) - values[key.strip()] = value.strip() - return values - - -def _parse_float(value: str | None) -> float | None: - if value is None or value == "": - return None - try: - return float(value) - except ValueError: - return None diff --git a/mesh/mesh_encoder.py b/mesh/mesh_encoder.py deleted file mode 100644 index b1bd684..0000000 --- a/mesh/mesh_encoder.py +++ /dev/null @@ -1,60 +0,0 @@ -from __future__ import annotations - -"""Legacy Meshtastic payload encoder implementation behind mesh.encoder.""" - -from core.model import MessageEvent, PositionEvent - - -def encode_position_payload(event: PositionEvent) -> str: - parts = [ - "APRS_POS", - f"entity={event.entity_kind}", - f"id={event.id}", - f"name={event.tactical_name or ''}", - f"lat={event.lat}", - f"lon={event.lon}", - f"ts={event.timestamp}", - ] - if event.object_name: - parts.append(f"object={event.object_name}") - if event.owner_id: - parts.append(f"owner={event.owner_id}") - if event.symbol_table: - parts.append(f"symbol_table={event.symbol_table}") - if event.symbol_code: - parts.append(f"symbol_code={event.symbol_code}") - if event.message: - parts.append(f"msg={event.message}") - return "|".join(parts) - - -def encode_text_payload(event: MessageEvent) -> str: - if event.target: - if event.message_id: - return f"[APRS {event.id}->{event.target} #{event.message_id}] {event.message}" - return f"[APRS {event.id}->{event.target}] {event.message}" - - if event.message_id: - return f"[APRS {event.id} #{event.message_id}] {event.message}" - return f"[APRS {event.id}] {event.message}" - - -def build_waypoint_fields(event: PositionEvent) -> tuple[str, str]: - label_source = event.object_name or event.tactical_name or event.id - label = label_source.strip()[:30] - if event.is_object(): - description_parts = [f"APRS object {event.object_name or event.id}"] - if event.owner_id: - description_parts.append(f"owner {event.owner_id}") - else: - description_parts = [f"APRS station {event.id}"] - if event.message: - description_parts.append(event.message.strip()) - return label, " | ".join(description_parts)[:120] - - -def build_waypoint_id(source_id: str) -> int: - value = 0 - for char in source_id.upper(): - value = ((value * 33) + ord(char)) % 1_000_000_000 - return value or 1 diff --git a/mesh/parser.py b/mesh/parser.py index 2cb15cd..2284426 100644 --- a/mesh/parser.py +++ b/mesh/parser.py @@ -1,15 +1,22 @@ from __future__ import annotations +"""Meshtastic RX parser/decoder producing runtime events plus canonical metadata.""" + import base64 import logging +import re from typing import Any from core.canonical import empty_canonical_event, iso_utc_now, to_iso_utc -from core.model import MessageEvent, PositionEvent, Source +from core.model import MessageEvent, PositionEvent, Source, build_raw_payload log = logging.getLogger("mesh.parser") +APRS_TEXT_RE = re.compile( + r"^\[APRS (?P.+?)(?:->(?P.+?))?(?: #(?P[^\]]+))?\] (?P.*)$" +) + class MeshtasticParseError(ValueError): def __init__(self, message: str, *, packet: dict[str, Any] | None = None) -> None: @@ -70,7 +77,7 @@ def parse_meshtastic_packet(packet: dict[str, Any]) -> dict[str, Any]: return canonical -def canonical_to_event( +def canonical_to_mesh_event( canonical: dict[str, Any], *, source_id: str | None, @@ -92,7 +99,7 @@ def canonical_to_event( message=text["text"], message_id=meta.get("message_id"), source=Source.MESHTASTIC, - raw=raw or canonical, + raw=build_raw_payload(canonical=canonical, raw_context=raw), ) log.debug( "Meshtastic canonical -> %s source=%s subtype=%s target=%r msg_id=%r", @@ -118,7 +125,7 @@ def canonical_to_event( tactical_name=tactical_name, message=message, source=Source.MESHTASTIC, - raw=raw or canonical, + raw=build_raw_payload(canonical=canonical, raw_context=raw), ) log.debug( "Meshtastic canonical -> %s source=%s subtype=%s lat=%s lon=%s", @@ -133,6 +140,133 @@ def canonical_to_event( return None +def decode_mesh_packet( + packet: dict[str, Any], + *, + source_id: str, + target_id: str | None = None, +) -> PositionEvent | MessageEvent | None: + canonical = parse_meshtastic_packet(packet) + decoded = packet.get("decoded") or {} + portnum = canonical.get("comms", {}).get("portnum") + + if "position" in decoded or portnum == "POSITION_APP": + native_position = decode_mesh_position_packet(packet, source_id=source_id, canonical=canonical) + if native_position is not None: + return native_position + + text_event = decode_mesh_text_packet( + packet, + source_id=source_id, + target_id=target_id, + ) + if text_event is not None: + return text_event + + return canonical_to_mesh_event( + canonical, + source_id=source_id, + target_id=target_id, + raw=packet, + ) + + +def decode_mesh_text_packet( + packet: dict[str, Any], + *, + source_id: str, + target_id: str | None = None, +) -> PositionEvent | MessageEvent | None: + decoded = packet.get("decoded") or {} + text = decoded.get("text") + if not text: + return None + return decode_mesh_text_payload(str(text).strip(), source_id=source_id, target_id=target_id, raw=packet) + + +def decode_mesh_text_payload( + text: str, + *, + source_id: str, + target_id: str | None = None, + raw: dict[str, Any] | None = None, +) -> PositionEvent | MessageEvent | None: + if not text: + return None + + if text.startswith("APRS_POS|"): + values = _parse_pipe_payload(text) + event_id = values.get("id") or source_id + lat = _parse_float(values.get("lat")) + lon = _parse_float(values.get("lon")) + if lat is None or lon is None: + return None + return PositionEvent( + id=event_id, + entity_kind=values.get("entity") or ("object" if values.get("object") else "station"), + type="object" if values.get("entity") == "object" or values.get("object") else "position", + lat=lat, + lon=lon, + tactical_name=values.get("name") or None, + message=values.get("msg") or None, + object_name=values.get("object") or None, + owner_id=values.get("owner") or None, + symbol_table=values.get("symbol_table") or None, + symbol_code=values.get("symbol_code") or None, + source=Source.MESHTASTIC, + raw=build_raw_payload(raw_context=raw or {"mesh_text": text}), + ) + + aprs_match = APRS_TEXT_RE.match(text) + if aprs_match: + event_source = (aprs_match.group("source") or source_id).strip() + event_target = (aprs_match.group("target") or target_id or "").strip() or None + event_message = (aprs_match.group("message") or "").strip() + return MessageEvent( + id=event_source, + target=event_target, + message=event_message, + message_id=(aprs_match.group("message_id") or "").strip() or None, + source=Source.MESHTASTIC, + raw=build_raw_payload(raw_context=raw or {"mesh_text": text}), + ) + + return MessageEvent( + id=source_id, + target=target_id, + message=text, + source=Source.MESHTASTIC, + raw=build_raw_payload(raw_context=raw or {"mesh_text": text}), + ) + + +def decode_mesh_position_packet( + packet: dict[str, Any], + *, + source_id: str, + canonical: dict[str, Any] | None = None, +) -> PositionEvent | None: + position = (canonical or {}).get("position", {}) + if not position: + canonical = canonical or parse_meshtastic_packet(packet) + position = canonical.get("position", {}) + + lat = position.get("lat") + lon = position.get("lon") + if lat is None or lon is None: + return None + return PositionEvent( + id=source_id, + entity_kind="station", + type="position", + lat=float(lat), + lon=float(lon), + alt=position.get("alt_m"), + source=Source.MESHTASTIC, + raw=build_raw_payload(canonical=canonical, raw_context=packet), + ) + + def _apply_user_fields(canonical: dict[str, Any], user: Any) -> None: if not isinstance(user, dict): return @@ -260,6 +394,16 @@ def _extract_payload_b64(decoded: dict[str, Any]) -> str | None: return None +def _parse_pipe_payload(text: str) -> dict[str, str]: + values: dict[str, str] = {} + for part in text.split("|")[1:]: + if "=" not in part: + continue + key, value = part.split("=", 1) + values[key.strip()] = value.strip() + return values + + def _coerce_coordinate(decimal_value: Any, integer_value: Any) -> float | None: if decimal_value is not None: try: @@ -276,6 +420,15 @@ def _coerce_coordinate(decimal_value: Any, integer_value: Any) -> float | None: return None +def _parse_float(value: str | None) -> float | None: + if value is None or value == "": + return None + try: + return float(value) + except ValueError: + return None + + def _mps_to_kph(value: Any) -> float | None: try: if value is None: @@ -296,3 +449,14 @@ def _safe_str(value: Any) -> str | None: if value is None: return None return str(value) + + +__all__ = [ + "MeshtasticParseError", + "canonical_to_mesh_event", + "decode_mesh_packet", + "decode_mesh_position_packet", + "decode_mesh_text_packet", + "decode_mesh_text_payload", + "parse_meshtastic_packet", +] diff --git a/requirements.txt b/requirements.txt index 47fcfd3..fd907d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ paho-mqtt>=2.1.0 meshtastic>=2.7.8 fastapi>=0.115.0 uvicorn>=0.30.0 +pytak>=7.3.0 diff --git a/sartrack/__init__.py b/sartrack/__init__.py index 808c805..f625673 100644 --- a/sartrack/__init__.py +++ b/sartrack/__init__.py @@ -2,6 +2,7 @@ from sartrack.parser import ( SARTrackParseError, canonical_to_sartrack_event, + decode_sartrack_line, parse_line, parse_sartrack_canonical, parse_sartrack_event, @@ -14,6 +15,7 @@ "SARTrackTCPConfig", "SARTrackTCPServer", "canonical_to_sartrack_event", + "decode_sartrack_line", "parse_line", "parse_sartrack_canonical", "parse_sartrack_event", diff --git a/sartrack/parser.py b/sartrack/parser.py index d38614f..6318c38 100644 --- a/sartrack/parser.py +++ b/sartrack/parser.py @@ -1,10 +1,12 @@ from __future__ import annotations +"""SARTrack line parser producing runtime events plus optional canonical metadata.""" + import logging from typing import Iterable from core.canonical import empty_canonical_event, iso_utc_now -from core.model import MessageEvent, PositionEvent, Source +from core.model import MessageEvent, PositionEvent, Source, build_raw_payload log = logging.getLogger("sartrack.parser") @@ -21,6 +23,7 @@ def parse_sartrack_event( ingress_adapter: str | None = None, raw_context: dict[str, object] | None = None, ) -> PositionEvent | MessageEvent | None: + """Parse a SARTrack text line into the runtime model.""" raw = line.strip() if not raw: return None @@ -50,6 +53,11 @@ def parse_line(line: str) -> PositionEvent | MessageEvent | None: return parse_sartrack_event(line) +def decode_sartrack_line(line: str) -> PositionEvent | MessageEvent | None: + """Operator-friendly alias for SARTrack RX decoding.""" + return parse_sartrack_event(line) + + def parse_sartrack_canonical(line: str) -> dict[str, object]: upper = line.upper() @@ -217,11 +225,11 @@ def canonical_to_sartrack_event( if not source_id: raise SARTrackParseError("canonical SARTrack event missing source id") - raw_payload: dict[str, object] = {"canonical": canonical} - if raw_context: - raw_payload.update(raw_context) - if ingress_adapter: - raw_payload["ingress_adapter"] = ingress_adapter + raw_payload = build_raw_payload( + canonical=canonical, + ingress_adapter=ingress_adapter, + raw_context=raw_context, + ) if text["text"]: return MessageEvent( diff --git a/tests/test_cot_adapter.py b/tests/test_cot_adapter.py new file mode 100644 index 0000000..be9a288 --- /dev/null +++ b/tests/test_cot_adapter.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import asyncio +import unittest +import xml.etree.ElementTree as ET +from datetime import UTC, datetime + +from core.model import MessageEvent, PositionEvent, Source +from cot.cot import CoTAdapter, CoTConfig, _build_pytak_config, _effective_cot_url + + +class CoTAdapterTests(unittest.IsolatedAsyncioTestCase): + async def test_send_enqueues_xml_payload(self) -> None: + adapter = CoTAdapter(CoTConfig(enabled=True, cot_url="udp://127.0.0.1:4242")) + adapter._tx_queue = asyncio.Queue() + event = PositionEvent( + id="N0CALL-10", + entity_kind="station", + type="position", + lat=50.1, + lon=19.1, + message="hello", + source=Source.SARTRACK, + ) + + await adapter.send(event) + + payload = await adapter._tx_queue.get() + self.assertIsInstance(payload, bytes) + self.assertIn(b" None: + adapter = CoTAdapter(CoTConfig(enabled=True)) + adapter._tx_queue = asyncio.Queue() + event = MessageEvent( + id="N0CALL-10", + target="SP9PET-10", + message="hello", + source=Source.SARTRACK, + ) + + await adapter.send(event) + + self.assertTrue(adapter._tx_queue.empty()) + + def test_build_pytak_config(self) -> None: + config = CoTConfig( + enabled=True, + cot_url="udp://127.0.0.1:4242", + pytak_no_hello=True, + tak_proto=0, + stale_seconds=300, + ) + + section = _build_pytak_config(config) + + self.assertEqual(section["COT_URL"], "udp+wo://127.0.0.1:4242") + self.assertEqual(section["PYTAK_NO_HELLO"], "1") + self.assertEqual(section["TAK_PROTO"], "0") + + def test_effective_udp_cot_url_is_write_only(self) -> None: + self.assertEqual(_effective_cot_url("udp://127.0.0.1:4242"), "udp+wo://127.0.0.1:4242") + self.assertEqual(_effective_cot_url("udp+wo://127.0.0.1:4242"), "udp+wo://127.0.0.1:4242") + self.assertEqual(_effective_cot_url("tcp://127.0.0.1:4242"), "tcp://127.0.0.1:4242") + + async def test_send_uses_configured_stale_seconds(self) -> None: + adapter = CoTAdapter(CoTConfig(enabled=True, stale_seconds=300)) + adapter._tx_queue = asyncio.Queue() + event = PositionEvent( + id="N0CALL-10", + timestamp=1_765_843_200, + entity_kind="station", + type="position", + lat=50.1, + lon=19.1, + message="hello", + source=Source.SARTRACK, + ) + + await adapter.send(event) + + payload = await adapter._tx_queue.get() + root = ET.fromstring(payload.decode("utf-8")) + time_value = datetime.fromisoformat(root.attrib["time"].replace("Z", "+00:00")) + stale_value = datetime.fromisoformat(root.attrib["stale"].replace("Z", "+00:00")) + self.assertEqual(time_value.tzinfo, UTC) + self.assertEqual(stale_value.tzinfo, UTC) + self.assertEqual(int((stale_value - time_value).total_seconds()), 300) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cot_mesh_adapters.py b/tests/test_cot_mesh_adapters.py index e9a9781..3a2f5da 100644 --- a/tests/test_cot_mesh_adapters.py +++ b/tests/test_cot_mesh_adapters.py @@ -2,11 +2,11 @@ import unittest -from cot.cot_encoder import encode_event_to_cot_xml -from cot.cot_parser import canonical_to_cot_event, parse_cot_canonical, parse_cot_xml +from cot.encoder import encode_event_to_cot_xml +from cot.parser import canonical_to_cot_event, parse_cot_canonical, parse_cot_xml from core.model import MessageEvent, PositionEvent, Source -from mesh.mesh_decoder import decode_mesh_packet, decode_mesh_text_payload -from mesh.mesh_encoder import encode_position_payload, encode_text_payload +from mesh.parser import decode_mesh_packet, decode_mesh_text_payload +from mesh.encoder import encode_position_payload, encode_text_payload from mesh.mesh_filter import MeshFilter @@ -125,12 +125,35 @@ def test_object_refresh_is_filtered(self) -> None: class CoTParserTests(unittest.TestCase): + def test_station_cot_contains_contact_and_group(self) -> None: + event = PositionEvent( + id="N0CALL-10", + entity_kind="station", + type="position", + lat=50.0, + lon=19.0, + tactical_name="ALPHA", + message="ready", + source=Source.SARTRACK, + ) + + xml = encode_event_to_cot_xml(event) + + self.assertIn('ready", xml) + def test_cot_canonical_roundtrip_object(self) -> None: xml = ( - '' '' - 'APRS object OBJ1 | owner N0CALL-10 | marker' + '' + '' + '<__group name="Ham Router" role="Waypoint" />' + '' + 'marker' + '' "" ) @@ -143,6 +166,9 @@ def test_cot_canonical_roundtrip_object(self) -> None: self.assertEqual(decoded.object_name, "OBJ1") self.assertEqual(decoded.owner_id, "N0CALL-10") self.assertEqual(decoded.message, "marker") + self.assertEqual(decoded.tactical_name, "pin-1") + self.assertEqual(decoded.symbol_table, "\\") + self.assertEqual(decoded.symbol_code, "z") self.assertEqual(decoded.alt, 123.0) def test_roundtrip_object_cot(self) -> None: @@ -162,12 +188,17 @@ def test_roundtrip_object_cot(self) -> None: xml = encode_event_to_cot_xml(event) decoded = parse_cot_xml(xml) + self.assertIn(' Date: Thu, 16 Apr 2026 05:27:34 +0200 Subject: [PATCH 3/6] =?UTF-8?q?Du=C5=BCe=20post=C4=99py=20w=20nad=20adapte?= =?UTF-8?q?rem=20CoT=20TAK=20-=20jestem=20w=20stanie=20przesy=C5=82a=C4=87?= =?UTF-8?q?=20wiadomo=C5=9Bci=20i=20pozycje=20w=20obu=20kierunkach,=20a=20?= =?UTF-8?q?tak=C5=BCe=20mapowa=C4=87=20symbole=20CoT=20TAK=20na=20symbole?= =?UTF-8?q?=20APRS.=20Nadal=20musz=C4=99=20przetestowa=C4=87=20adapter=20w?= =?UTF-8?q?=20r=C3=B3=C5=BCnych=20scenariuszach=20i=20upewni=C4=87=20si?= =?UTF-8?q?=C4=99,=20=C5=BCe=20jest=20stabilny=20i=20niezawodny,=20ale=20j?= =?UTF-8?q?estem=20zadowolony=20z=20post=C4=99p=C3=B3w,=20kt=C3=B3re=20poc?= =?UTF-8?q?zyni=C5=82em=20do=20tej=20pory.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- aprs_cot_fallback.json | 113 ++++++++++ config.py | 9 + core/router.py | 20 ++ cot/__init__.py | 5 + cot/cot.py | 377 +++++++++++++++++++++++++++++--- cot/encoder.py | 128 ++++++++++- cot/parser.py | 231 ++++++++++++++++--- cot/symbol_mapper.py | 229 +++++++++++++++++++ gui/server.py | 251 +++++++++++++++++++++ main.py | 3 +- tests/test_cot_adapter.py | 30 ++- tests/test_cot_mesh_adapters.py | 169 ++++++++++++++ 12 files changed, 1495 insertions(+), 70 deletions(-) create mode 100644 aprs_cot_fallback.json create mode 100644 cot/symbol_mapper.py diff --git a/aprs_cot_fallback.json b/aprs_cot_fallback.json new file mode 100644 index 0000000..727b7bf --- /dev/null +++ b/aprs_cot_fallback.json @@ -0,0 +1,113 @@ +{ + "default": { + "person": "a-f-G-U-C", + "vehicle": "a-f-G-E-V-C", + "aircraft": "a-f-A", + "helicopter": "a-f-A-C-H", + "marine": "a-f-S", + "fixed": "a-.-G-I", + "unknown": "a-u-G-U-C" + }, + "symbols": { + "/>": { + "aprs": "car", + "cotType": "a-f-G-E-V-C", + "category": "vehicle" + }, + "\\>": { + "aprs": "car-alt", + "cotType": "a-f-G-E-V-C", + "category": "vehicle" + }, + "/<": { + "aprs": "motorcycle", + "cotType": "a-f-G-E-V-C", + "category": "vehicle" + }, + "/j": { + "aprs": "jeep", + "cotType": "a-f-G-E-V-C", + "category": "vehicle" + }, + "/v": { + "aprs": "van", + "cotType": "a-f-G-E-V-C", + "category": "vehicle" + }, + "/k": { + "aprs": "truck", + "cotType": "a-f-G-E-V-C", + "category": "vehicle" + }, + "/a": { + "aprs": "ambulance", + "cotType": "a-f-G-E-V-C", + "category": "vehicle" + }, + "/b": { + "aprs": "bicycle-bike", + "cotType": "a-f-G-U-C", + "category": "person" + }, + "/[": { + "aprs": "human-person", + "cotType": "a-f-G-U-C-I", + "category": "person" + }, + "/h": { + "aprs": "hiker", + "cotType": "a-f-G-U-C-I", + "category": "person" + }, + "/-": { + "aprs": "house-home", + "cotType": "a-.-G-I", + "category": "fixed" + }, + "\\#": { + "aprs": "digipeater", + "cotType": "a-.-G-I", + "category": "fixed" + }, + "\\&": { + "aprs": "igate-gateway", + "cotType": "a-.-G-I", + "category": "fixed" + }, + "/_": { + "aprs": "weather-station", + "cotType": "a-.-G-I", + "category": "fixed" + }, + "/r": { + "aprs": "repeater-antenna", + "cotType": "a-.-G-I", + "category": "fixed" + }, + "/^": { + "aprs": "aircraft", + "cotType": "a-f-A-C-F", + "category": "aircraft" + }, + "\\^": { + "aprs": "aircraft-alt", + "cotType": "a-f-A", + "category": "aircraft" + }, + "/X": { + "aprs": "helicopter", + "cotType": "a-f-A-C-H", + "category": "helicopter" + }, + "/Y": { + "aprs": "yacht-sailboat", + "cotType": "a-f-S", + "category": "marine" + }, + "/s": { + "aprs": "ship-boat", + "cotType": "a-f-S", + "category": "marine" + } + } +} \ No newline at end of file diff --git a/config.py b/config.py index 16121f6..cb27c1e 100644 --- a/config.py +++ b/config.py @@ -55,6 +55,9 @@ COT = {'enabled': True, 'cot_url': 'udp://127.0.0.1:4242', + 'rx_enabled': True, + 'listen_url': 'udp://0.0.0.0:4243', + 'chat_listen_url': 'udp://224.10.10.1:17012', 'pytak_no_hello': True, 'tak_proto': 0, 'stale_seconds': 300} @@ -211,6 +214,9 @@ def load_config() -> GatewayConfig: cot=CoTConfig( enabled=bool(COT["enabled"]), cot_url=str(COT.get("cot_url", "udp://127.0.0.1:4242")), + rx_enabled=bool(COT.get("rx_enabled", True)), + listen_url=str(COT.get("listen_url", "udp://0.0.0.0:4243")), + chat_listen_url=str(COT.get("chat_listen_url", "udp://224.10.10.1:17012")), pytak_no_hello=bool(COT.get("pytak_no_hello", True)), tak_proto=int(COT.get("tak_proto", 0)), stale_seconds=int(COT.get("stale_seconds", 300)), @@ -302,6 +308,9 @@ def persist_cot_config(config: CoTConfig) -> None: values = { "enabled": config.enabled, "cot_url": config.cot_url, + "rx_enabled": config.rx_enabled, + "listen_url": config.listen_url, + "chat_listen_url": config.chat_listen_url, "pytak_no_hello": config.pytak_no_hello, "tak_proto": config.tak_proto, "stale_seconds": config.stale_seconds, diff --git a/core/router.py b/core/router.py index c5e7430..cf87619 100644 --- a/core/router.py +++ b/core/router.py @@ -65,6 +65,13 @@ async def publish(self, event: PositionEvent | MessageEvent) -> None: tasks: list[asyncio.Task[object]] = [] for adapter in self.adapters: if adapter.source_name == event.source: + self.log.debug( + "ROUTE | skip-source-match | adapter=%s event_source=%s kind=%s id=%s", + self._adapter_label(adapter), + event.source.value, + event.kind, + event.id, + ) continue if not self._is_adapter_enabled(adapter): adapter_label = self._adapter_label(adapter) @@ -81,12 +88,25 @@ async def publish(self, event: PositionEvent | MessageEvent) -> None: async def _dispatch(self, adapter: Adapter, event: PositionEvent | MessageEvent) -> None: adapter_label = self._adapter_label(adapter) try: + self.log.debug( + "ROUTE | dispatch-start | adapter=%s kind=%s id=%s source=%s", + adapter_label, + event.kind, + event.id, + event.source.value, + ) if isinstance(event, PositionEvent) and hasattr(adapter, "send_position"): await getattr(adapter, "send_position")(event) elif isinstance(event, MessageEvent) and hasattr(adapter, "send_message"): await getattr(adapter, "send_message")(event) else: await adapter.send(event) + self.log.debug( + "ROUTE | dispatch-done | adapter=%s kind=%s id=%s", + adapter_label, + event.kind, + event.id, + ) if self.state_store: self.state_store.record_event( diff --git a/cot/__init__.py b/cot/__init__.py index 60b4459..05b8c92 100644 --- a/cot/__init__.py +++ b/cot/__init__.py @@ -1,15 +1,20 @@ from cot.adapter import CoTAdapter, CoTConfig, build_cot_event from cot.encoder import encode_event_to_cot_xml, position_event_to_canonical from cot.parser import CoTParseError, canonical_to_cot_event, decode_cot_xml, parse_cot_canonical, parse_cot_xml +from cot.symbol_mapper import APRSCotSymbol, categorize_cot_type, map_aprs_symbol_to_cot, map_cot_type_to_aprs_symbol __all__ = [ + "APRSCotSymbol", "CoTAdapter", "CoTConfig", "CoTParseError", "build_cot_event", "canonical_to_cot_event", + "categorize_cot_type", "decode_cot_xml", "encode_event_to_cot_xml", + "map_aprs_symbol_to_cot", + "map_cot_type_to_aprs_symbol", "parse_cot_canonical", "parse_cot_xml", "position_event_to_canonical", diff --git a/cot/cot.py b/cot/cot.py index 7647908..347422c 100644 --- a/cot/cot.py +++ b/cot/cot.py @@ -3,12 +3,16 @@ import asyncio import configparser import logging +import socket +import struct from dataclasses import dataclass from urllib.parse import urlparse from core.model import MessageEvent, PositionEvent, Source +from core.router import EventRouter from core.state import StateStore -from cot.encoder import encode_event_to_cot_xml, position_event_to_canonical +from cot.encoder import encode_event_to_cot_xml, encode_message_to_cot_xml, position_event_to_canonical +from cot.parser import CoTParseError, parse_cot_xml try: import pytak @@ -20,6 +24,9 @@ class CoTConfig: enabled: bool = True cot_url: str = "udp://127.0.0.1:4242" + rx_enabled: bool = True + listen_url: str = "udp://0.0.0.0:4243" + chat_listen_url: str = "udp://224.10.10.1:17012" pytak_no_hello: bool = True tak_proto: int = 0 stale_seconds: int = 300 @@ -35,11 +42,18 @@ def __init__(self, config: CoTConfig | None = None) -> None: self._tool: object | None = None self._tx_queue: asyncio.Queue[bytes] | None = None self._run_task: asyncio.Task[None] | None = None + self._rx_servers: list[tuple[str, asyncio.base_events.Server]] = [] + self._rx_transports: list[tuple[str, asyncio.DatagramTransport]] = [] + self._router: EventRouter | None = None def attach_state_store(self, state_store: StateStore) -> None: self.state_store = state_store self.state_store.update_status("cot", "offline", "not connected") + def attach_router(self, router: EventRouter) -> None: + self._router = router + self.log.debug("STATE | router-attached | adapter=cot") + async def connect(self) -> None: if not self.config.enabled: self.log.info("STATE | disabled | adapter=cot") @@ -54,9 +68,20 @@ async def connect(self) -> None: return await self._close_transport() + await self._close_rx_listener() try: effective_cot_url = _effective_cot_url(self.config.cot_url) + self.log.info( + "STATE | connect-config | adapter=cot tx_url=%s effective_tx_url=%s rx_enabled=%s listen_url=%s chat_listen_url=%s tak_proto=%s no_hello=%s", + self.config.cot_url, + effective_cot_url, + self.config.rx_enabled, + self.config.listen_url, + self.config.chat_listen_url, + self.config.tak_proto, + self.config.pytak_no_hello, + ) if self.state_store: self.state_store.update_status("cot", "connecting", effective_cot_url) tool = pytak.CLITool(_build_pytak_config(self.config)) @@ -71,6 +96,7 @@ async def connect(self) -> None: self.config.tak_proto, self.config.pytak_no_hello, ) + await self._start_rx_listener() if self.state_store: self.state_store.update_status("cot", "online", effective_cot_url) except Exception as exc: @@ -78,56 +104,89 @@ async def connect(self) -> None: if self.state_store: self.state_store.update_status("cot", "error", str(exc)) await self._close_transport() + await self._close_rx_listener() async def send(self, event: PositionEvent | MessageEvent) -> None: if not self.config.enabled: self.log.debug("TX | skip | adapter=cot reason=disabled") return - if not isinstance(event, PositionEvent): - self.log.debug("TX | skip | adapter=cot reason=unsupported-event event_type=%s", type(event).__name__) - return - if not event.is_position(): - self.log.debug("TX | skip | adapter=cot reason=no-position event_id=%s", event.id) - return if not self._tx_queue: self.log.warning("TX | skip | adapter=cot reason=not-connected event_id=%s", event.id) return - if self.config.stale_seconds > event.ttl: - event = PositionEvent( - id=event.id, - timestamp=event.timestamp, - source=event.source, - ttl=self.config.stale_seconds, - entity_kind=event.entity_kind, - type=event.type, - lat=event.lat, - lon=event.lon, - alt=event.alt, - speed=event.speed, - heading=event.heading, - tactical_name=event.tactical_name, - message=event.message, - symbol_table=event.symbol_table, - symbol_code=event.symbol_code, - object_name=event.object_name, - owner_id=event.owner_id, + if isinstance(event, PositionEvent): + if not event.is_position(): + self.log.debug("TX | skip | adapter=cot reason=no-position event_id=%s", event.id) + return + if self.config.stale_seconds > event.ttl: + event = PositionEvent( + id=event.id, + timestamp=event.timestamp, + source=event.source, + ttl=self.config.stale_seconds, + entity_kind=event.entity_kind, + type=event.type, + lat=event.lat, + lon=event.lon, + alt=event.alt, + speed=event.speed, + heading=event.heading, + tactical_name=event.tactical_name, + message=event.message, + symbol_table=event.symbol_table, + symbol_code=event.symbol_code, + object_name=event.object_name, + owner_id=event.owner_id, + ) + xml = build_cot_event(event) + self.log.info( + "TX | cot | adapter=cot entity=%s id=%s cot_url=%s stale_seconds=%s", + event.entity_kind, + event.id, + self.config.cot_url, + event.ttl, ) - xml = build_cot_event(event) - self.log.info( - "TX | cot | adapter=cot entity=%s id=%s cot_url=%s stale_seconds=%s", - event.entity_kind, - event.id, - self.config.cot_url, - event.ttl, - ) - self.log.debug("TX | cot-xml | adapter=cot payload=%s", xml) - await self._tx_queue.put(xml.encode("utf-8")) + self.log.debug( + "TX | cot-summary | adapter=cot id=%s entity=%s tactical=%r object=%s owner=%s lat=%s lon=%s symbol=%s%s", + event.id, + event.entity_kind, + event.tactical_name, + event.object_name, + event.owner_id, + event.lat, + event.lon, + event.symbol_table or "", + event.symbol_code or "", + ) + self.log.debug("TX | cot-xml | adapter=cot payload=%s", xml) + else: + xml = encode_message_to_cot_xml(event) + self.log.info( + "TX | cot-chat | adapter=cot source=%s target=%s cot_url=%s", + event.id, + event.target or "All Chat Rooms", + self.config.cot_url, + ) + self.log.debug( + "TX | cot-chat-summary | adapter=cot source=%s target=%s message_id=%r text=%r raw_keys=%s", + event.id, + event.target, + event.message_id, + event.message, + sorted(event.raw.keys()) if isinstance(event.raw, dict) else [], + ) + self.log.debug("TX | cot-chat-xml | adapter=cot payload=%s", xml) + xml_bytes = xml.encode("utf-8") + await self._tx_queue.put(xml_bytes) + self.log.debug("TX | queued | adapter=cot bytes=%s", len(xml_bytes)) def get_runtime_config(self) -> dict[str, object]: return { "enabled": self.config.enabled, "cot_url": self.config.cot_url, + "rx_enabled": self.config.rx_enabled, + "listen_url": self.config.listen_url, + "chat_listen_url": self.config.chat_listen_url, "pytak_no_hello": self.config.pytak_no_hello, "tak_proto": self.config.tak_proto, "stale_seconds": self.config.stale_seconds, @@ -138,6 +197,12 @@ async def apply_runtime_config(self, updates: dict[str, object]) -> dict[str, ob self.config.enabled = bool(updates["enabled"]) if "cot_url" in updates and updates["cot_url"] is not None: self.config.cot_url = str(updates["cot_url"]) + if "rx_enabled" in updates: + self.config.rx_enabled = bool(updates["rx_enabled"]) + if "listen_url" in updates and updates["listen_url"] is not None: + self.config.listen_url = str(updates["listen_url"]) + if "chat_listen_url" in updates and updates["chat_listen_url"] is not None: + self.config.chat_listen_url = str(updates["chat_listen_url"]) if "pytak_no_hello" in updates: self.config.pytak_no_hello = bool(updates["pytak_no_hello"]) if "tak_proto" in updates and updates["tak_proto"] is not None: @@ -146,6 +211,7 @@ async def apply_runtime_config(self, updates: dict[str, object]) -> dict[str, ob self.config.stale_seconds = int(updates["stale_seconds"]) await self._close_transport() + await self._close_rx_listener() if self.state_store: state = "disabled" if not self.config.enabled else "reconnecting" detail = "runtime config updated" if self.config.enabled else "adapter disabled" @@ -156,6 +222,7 @@ async def apply_runtime_config(self, updates: dict[str, object]) -> dict[str, ob async def reconnect_now(self) -> None: await self._close_transport() + await self._close_rx_listener() if self.state_store: state = "disabled" if not self.config.enabled else "reconnecting" detail = "manual reconnect requested" if self.config.enabled else "adapter disabled" @@ -197,6 +264,188 @@ async def _close_transport(self) -> None: if self.state_store and self.config.enabled: self.state_store.update_status("cot", "offline", "transport stopped") + async def _start_rx_listener(self) -> None: + if not self.config.rx_enabled: + self.log.info("STATE | rx-disabled | adapter=cot") + return + if self._router is None: + self.log.warning("STATE | rx-skipped | adapter=cot reason=router-not-attached") + return + + started = 0 + for listen_url in _iter_listen_urls(self.config): + try: + await self._start_single_rx_listener(listen_url) + started += 1 + except Exception as exc: + self.log.warning( + "STATE | rx-listener-error | adapter=cot listen_url=%s error=%s", + listen_url, + exc, + ) + + if started == 0: + self.log.warning("STATE | rx-unavailable | adapter=cot reason=no-listeners-started") + else: + self.log.info("STATE | rx-ready | adapter=cot listeners=%s", started) + + async def _close_rx_listener(self) -> None: + for listen_url, transport in self._rx_transports: + self.log.debug("STATE | rx-transport-closing | adapter=cot listen_url=%s", listen_url) + transport.close() + self._rx_transports.clear() + + for listen_url, server in self._rx_servers: + self.log.debug("STATE | rx-server-closing | adapter=cot listen_url=%s", listen_url) + server.close() + await server.wait_closed() + self._rx_servers.clear() + + async def _start_single_rx_listener(self, listen_url: str) -> None: + parsed = urlparse(listen_url) + scheme = parsed.scheme.lower() + host = parsed.hostname or "0.0.0.0" + port = parsed.port + if port is None: + raise ValueError(f"listen_url must include port: {listen_url}") + + if scheme == "udp": + loop = asyncio.get_running_loop() + if _is_multicast_host(host): + sock = _build_multicast_socket(host, port) + transport, _ = await loop.create_datagram_endpoint( + lambda: _CoTRxDatagramProtocol(self, listen_url), + sock=sock, + ) + self._rx_transports.append((listen_url, transport)) + self.log.info( + "STATE | rx-listening | adapter=cot listen_url=%s scheme=udp mode=multicast group=%s port=%s", + listen_url, + host, + port, + ) + return + + transport, _ = await loop.create_datagram_endpoint( + lambda: _CoTRxDatagramProtocol(self, listen_url), + local_addr=(host, port), + ) + self._rx_transports.append((listen_url, transport)) + self.log.info( + "STATE | rx-listening | adapter=cot listen_url=%s scheme=udp mode=unicast host=%s port=%s", + listen_url, + host, + port, + ) + return + + if scheme == "tcp": + server = await asyncio.start_server( + lambda reader, writer: self._handle_tcp_client(reader, writer, listen_url), + host, + port, + ) + self._rx_servers.append((listen_url, server)) + self.log.info( + "STATE | rx-listening | adapter=cot listen_url=%s scheme=tcp host=%s port=%s", + listen_url, + host, + port, + ) + return + + raise ValueError(f"unsupported CoT listen scheme: {scheme}") + + async def _handle_incoming_xml(self, xml_text: str, peer: object, listen_url: str) -> None: + if self._router is None: + self.log.warning("RX | dropped | adapter=cot listen_url=%s peer=%s reason=no-router", listen_url, peer) + return + self.log.debug( + "RX | raw-cot | adapter=cot listen_url=%s peer=%s bytes=%s payload=%s", + listen_url, + peer, + len(xml_text.encode("utf-8")), + xml_text, + ) + try: + event = parse_cot_xml(xml_text) + except CoTParseError as exc: + self.log.warning("RX | invalid | adapter=cot listen_url=%s peer=%s error=%s", listen_url, peer, exc) + return + + if isinstance(event.raw, dict): + event.raw["ingress_adapter"] = "cot" + event.raw["peer"] = str(peer) + event.raw["listen_url"] = listen_url + if isinstance(event, PositionEvent): + self.log.info( + "RX | accepted | adapter=cot listen_url=%s peer=%s kind=position id=%s lat=%s lon=%s", + listen_url, + peer, + event.id, + event.lat, + event.lon, + ) + self.log.debug( + "RX | mapped-position | adapter=cot listen_url=%s peer=%s id=%s entity=%s tactical=%r object=%s owner=%s symbol=%s%s", + listen_url, + peer, + event.id, + event.entity_kind, + event.tactical_name, + event.object_name, + event.owner_id, + event.symbol_table or "", + event.symbol_code or "", + ) + else: + self.log.info( + "RX | accepted | adapter=cot listen_url=%s peer=%s kind=message id=%s target=%s", + listen_url, + peer, + event.id, + event.target, + ) + self.log.debug( + "RX | mapped-message | adapter=cot listen_url=%s peer=%s source=%s target=%s message_id=%r text=%r", + listen_url, + peer, + event.id, + event.target, + event.message_id, + event.message, + ) + self.log.debug("RX | publish | adapter=cot listen_url=%s peer=%s kind=%s id=%s", listen_url, peer, event.kind, event.id) + await self._router.publish(event) + + async def _handle_tcp_client(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter, listen_url: str) -> None: + peer = writer.get_extra_info("peername") + self.log.info("STATE | rx-client-connected | adapter=cot listen_url=%s peer=%s", listen_url, peer) + buffer = "" + try: + while not reader.at_eof(): + chunk = await reader.read(4096) + if not chunk: + self.log.debug("RX | tcp-eof | adapter=cot listen_url=%s peer=%s", listen_url, peer) + break + self.log.debug("RX | tcp-chunk | adapter=cot listen_url=%s peer=%s bytes=%s", listen_url, peer, len(chunk)) + buffer += chunk.decode("utf-8", errors="replace") + while "" in buffer: + xml_text, buffer = buffer.split("", 1) + self.log.debug( + "RX | tcp-event-boundary | adapter=cot listen_url=%s peer=%s buffered_remaining=%s", + listen_url, + peer, + len(buffer), + ) + await self._handle_incoming_xml(f"{xml_text}", peer, listen_url) + except Exception as exc: + self.log.warning("STATE | rx-client-error | adapter=cot listen_url=%s peer=%s error=%s", listen_url, peer, exc) + finally: + writer.close() + await writer.wait_closed() + self.log.info("STATE | rx-client-disconnected | adapter=cot listen_url=%s peer=%s", listen_url, peer) + def build_cot_event(event: PositionEvent) -> str: canonical = position_event_to_canonical(event) @@ -227,3 +476,59 @@ def _effective_cot_url(cot_url: str) -> str: effective_scheme = f"{scheme}+wo" return parsed._replace(scheme=effective_scheme).geturl() return cot_url + + +def _iter_listen_urls(config: CoTConfig) -> list[str]: + seen: set[str] = set() + values: list[str] = [] + for candidate in (config.listen_url, config.chat_listen_url): + normalized = str(candidate or "").strip() + if not normalized or normalized in seen: + continue + values.append(normalized) + seen.add(normalized) + return values + + +def _is_multicast_host(host: str) -> bool: + try: + first_octet = int(host.split(".", 1)[0]) + except (ValueError, IndexError): + return False + return 224 <= first_octet <= 239 + + +def _build_multicast_socket(host: str, port: int) -> socket.socket: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if hasattr(socket, "SO_REUSEPORT"): + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + except OSError: + pass + try: + sock.bind(("", port)) + except OSError: + sock.close() + raise + + membership = struct.pack("=4s4s", socket.inet_aton(host), socket.inet_aton("0.0.0.0")) + sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, membership) + sock.setblocking(False) + return sock + + +class _CoTRxDatagramProtocol(asyncio.DatagramProtocol): + def __init__(self, adapter: CoTAdapter, listen_url: str) -> None: + self.adapter = adapter + self.listen_url = listen_url + + def datagram_received(self, data: bytes, addr) -> None: + self.adapter.log.debug( + "RX | udp-datagram | adapter=cot listen_url=%s peer=%s bytes=%s", + self.listen_url, + addr, + len(data), + ) + xml_text = data.decode("utf-8", errors="replace") + asyncio.create_task(self.adapter._handle_incoming_xml(xml_text, addr, self.listen_url)) diff --git a/cot/encoder.py b/cot/encoder.py index 76f89fe..11dca28 100644 --- a/cot/encoder.py +++ b/cot/encoder.py @@ -1,12 +1,13 @@ from __future__ import annotations -"""CoT XML encoder for runtime position events.""" +"""CoT XML encoder for runtime position and message events.""" import time import xml.etree.ElementTree as ET from core.canonical import empty_canonical_event -from core.model import PositionEvent +from core.model import MessageEvent, PositionEvent +from cot.symbol_mapper import map_aprs_symbol_to_cot def encode_event_to_cot_xml(event: PositionEvent) -> str: @@ -59,6 +60,21 @@ def encode_event_to_cot_xml(event: PositionEvent) -> str: "role": str(tak["tak_group_role"] or _default_group_role(event)), }, ) + if event.symbol_table or event.symbol_code: + symbol_mapping = map_aprs_symbol_to_cot( + event.symbol_table, + event.symbol_code, + entity_kind=event.entity_kind, + prefer_marker_for_objects=False, + ) + symbol_attrs = {"category": symbol_mapping.category} + if event.symbol_table: + symbol_attrs["symbol_table"] = str(event.symbol_table) + if event.symbol_code: + symbol_attrs["symbol_code"] = str(event.symbol_code) + if symbol_mapping.cot_type: + symbol_attrs["cot_type"] = str(symbol_mapping.cot_type) + ET.SubElement(detail, "ham_router_aprs", symbol_attrs) if event.is_object(): object_attrs = { "name": str(event.object_name or event.id), @@ -77,7 +93,77 @@ def encode_event_to_cot_xml(event: PositionEvent) -> str: return ET.tostring(root, encoding="unicode") +def encode_message_to_cot_xml(event: MessageEvent) -> str: + now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp)) + stale = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp + event.ttl)) + chatroom, destination = _chat_route(event) + message_uid = f"GeoChat.{event.id}.{chatroom}.{event.message_id or event.timestamp}" + + root = ET.Element( + "event", + { + "version": "2.0", + "uid": message_uid, + "type": "b-t-f", + "time": now, + "start": now, + "stale": stale, + "how": "h-g-i-g-o", + }, + ) + ET.SubElement( + root, + "point", + { + "lat": "0", + "lon": "0", + "hae": "9999999", + "ce": "9999999", + "le": "9999999", + }, + ) + detail = ET.SubElement(root, "detail") + chat = ET.SubElement( + detail, + "__chat", + { + "chatroom": chatroom, + "groupOwner": "false", + "id": str(event.message_id or event.timestamp), + "senderCallsign": str(event.id), + }, + ) + ET.SubElement( + chat, + "chatgrp", + { + "id": chatroom, + "uid0": str(event.id), + "uid1": destination, + }, + ) + marti = ET.SubElement(detail, "marti") + ET.SubElement(marti, "dest", {"callsign": destination}) + remarks = ET.SubElement( + detail, + "remarks", + { + "source": str(event.id), + "to": destination, + "time": now, + }, + ) + remarks.text = str(event.message) + return ET.tostring(root, encoding="unicode") + + def position_event_to_canonical(event: PositionEvent) -> dict[str, object]: + symbol_mapping = map_aprs_symbol_to_cot( + event.symbol_table, + event.symbol_code, + entity_kind=event.entity_kind, + prefer_marker_for_objects=event.is_object(), + ) canonical = empty_canonical_event() canonical["meta"]["source_format"] = "cot_xml" canonical["meta"]["source_subtype"] = "position" @@ -85,7 +171,7 @@ def position_event_to_canonical(event: PositionEvent) -> dict[str, object]: canonical["identity"]["uid"] = event.object_name or event.id canonical["identity"]["callsign"] = event.owner_id or event.id canonical["identity"]["short_name"] = event.tactical_name or event.object_name or event.id - canonical["identity"]["object_type"] = "unknown" if event.is_object() else "person" + canonical["identity"]["object_type"] = symbol_mapping.category canonical["position"]["lat"] = event.lat canonical["position"]["lon"] = event.lon canonical["position"]["alt_m"] = event.alt @@ -95,7 +181,7 @@ def position_event_to_canonical(event: PositionEvent) -> dict[str, object]: canonical["text"]["comment"] = event.message canonical["text"]["remarks"] = event.message canonical["tak"]["cot_version"] = "2.0" - canonical["tak"]["cot_type"] = "b-m-p-w" if event.is_object() else "a-f-G-U-C" + canonical["tak"]["cot_type"] = symbol_mapping.cot_type or ("b-m-p-w" if event.is_object() else "a-f-G-U-C") canonical["tak"]["how"] = "m-g" canonical["tak"]["start_utc"] = canonical["position"]["timestamp_utc"] canonical["tak"]["stale_utc"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp + event.ttl)) @@ -116,7 +202,41 @@ def _default_group_role(event: PositionEvent) -> str: return "Waypoint" return "Team Member" + +def _chat_route(event: MessageEvent) -> tuple[str, str]: + target = (event.target or "").strip() + subtype = _message_subtype(event) + + if subtype in {"bulletin", "group_message"}: + source_chat = (event.id or "").strip() or "UNKNOWN" + return source_chat, source_chat + + if not target: + source_chat = (event.id or "").strip() or "UNKNOWN" + return source_chat, source_chat + + return target, target + + +def _message_subtype(event: MessageEvent) -> str: + raw = event.raw if isinstance(event.raw, dict) else {} + canonical = raw.get("canonical") + if isinstance(canonical, dict): + meta = canonical.get("meta") + if isinstance(meta, dict): + subtype = str(meta.get("source_subtype") or "").strip() + if subtype: + return subtype + + target = (event.target or "").strip().upper() + if target.startswith("BLN"): + return "bulletin" + if target.startswith("TEAM") or target in {"GRUPA", "GROUP", "ALL"}: + return "group_message" + return "private_message" + __all__ = [ + "encode_message_to_cot_xml", "encode_event_to_cot_xml", "position_event_to_canonical", ] diff --git a/cot/parser.py b/cot/parser.py index ca79492..9dd1c39 100644 --- a/cot/parser.py +++ b/cot/parser.py @@ -1,11 +1,13 @@ from __future__ import annotations -"""CoT XML parser producing runtime position events plus canonical metadata.""" +"""CoT XML parser producing runtime position/message events plus canonical metadata.""" +import logging import xml.etree.ElementTree as ET from core.canonical import empty_canonical_event -from core.model import PositionEvent, Source, build_raw_payload +from core.model import MessageEvent, PositionEvent, Source, build_raw_payload +from cot.symbol_mapper import categorize_cot_type, map_cot_type_to_aprs_symbol class CoTParseError(ValueError): @@ -13,14 +15,34 @@ class CoTParseError(ValueError): OBJECT_COT_TYPES = {"b-m-p-s-p-loc", "b-m-p-w"} +_log = logging.getLogger("cot.parser") -def parse_cot_xml(xml_text: str) -> PositionEvent: +def parse_cot_xml(xml_text: str) -> PositionEvent | MessageEvent: canonical = parse_cot_canonical(xml_text) - return canonical_to_cot_event(canonical, raw={"cot_xml": xml_text}) + event = canonical_to_cot_event(canonical, raw={"cot_xml": xml_text}) + if isinstance(event, PositionEvent): + _log.debug( + "CoT xml -> position id=%s entity=%s lat=%s lon=%s symbol=%s%s", + event.id, + event.entity_kind, + event.lat, + event.lon, + event.symbol_table or "", + event.symbol_code or "", + ) + else: + _log.debug( + "CoT xml -> message source=%s target=%s message_id=%r text=%r", + event.id, + event.target, + event.message_id, + event.message, + ) + return event -def decode_cot_xml(xml_text: str) -> PositionEvent: +def decode_cot_xml(xml_text: str) -> PositionEvent | MessageEvent: """Operator-friendly alias for CoT RX decoding.""" return parse_cot_xml(xml_text) @@ -34,42 +56,140 @@ def parse_cot_canonical(xml_text: str) -> dict[str, object]: if root.tag != "event": raise CoTParseError(f"unsupported root tag: {root.tag}") - point = root.find("point") - if point is None: - raise CoTParseError("missing point element") - uid = root.attrib.get("uid") if not uid: raise CoTParseError("missing uid") - try: - lat = float(point.attrib["lat"]) - lon = float(point.attrib["lon"]) - except (KeyError, ValueError) as exc: - raise CoTParseError("invalid point coordinates") from exc - detail = root.find("detail") remarks = None + remarks_source = None + remarks_source_id = None + remarks_target = None object_meta: dict[str, str] = {} + aprs_meta: dict[str, str] = {} + chat_meta: dict[str, str] = {} + marti_dest: str | None = None + contact_callsign: str | None = None + droid_callsign: str | None = None if detail is not None: remarks_elem = detail.find("remarks") - if remarks_elem is not None and remarks_elem.text: - remarks = remarks_elem.text.strip() or None + if remarks_elem is not None: + if remarks_elem.text: + remarks = remarks_elem.text.strip() or None + remarks_source = remarks_elem.attrib.get("source") + remarks_source_id = remarks_elem.attrib.get("sourceID") + remarks_target = remarks_elem.attrib.get("to") + contact_elem = detail.find("contact") + if contact_elem is not None and contact_elem.attrib.get("callsign"): + contact_callsign = str(contact_elem.attrib.get("callsign")).strip() or None + droid_elem = detail.find("uid") + if droid_elem is not None and droid_elem.attrib.get("Droid"): + droid_callsign = str(droid_elem.attrib.get("Droid")).strip() or None object_elem = detail.find("ham_router_object") if object_elem is not None: object_meta = {str(key): str(value) for key, value in object_elem.attrib.items()} + aprs_elem = detail.find("ham_router_aprs") + if aprs_elem is not None: + aprs_meta = {str(key): str(value) for key, value in aprs_elem.attrib.items()} + chat_elem = detail.find("__chat") + if chat_elem is not None: + chat_meta = {str(key): str(value) for key, value in chat_elem.attrib.items()} + marti_dest_elem = detail.find("marti/dest") + if marti_dest_elem is not None and marti_dest_elem.attrib.get("callsign"): + marti_dest = str(marti_dest_elem.attrib.get("callsign")) cot_type = root.attrib.get("type", "") + if _is_chat_event(cot_type, chat_meta): + preferred_sender = ( + _clean_callsign_like(chat_meta.get("senderCallsign")) + or _clean_callsign_like(contact_callsign) + or _clean_callsign_like(droid_callsign) + or _clean_callsign_like(remarks_source_id) + or _clean_callsign_like(remarks_source) + or uid + ) + preferred_target = ( + _clean_callsign_like(remarks_target) + or _clean_callsign_like(marti_dest) + or _clean_callsign_like(chat_meta.get("chatroom")) + ) + preferred_message_id = ( + str(chat_meta.get("messageId") or "").strip() + or str(chat_meta.get("id") or "").strip() + or None + ) + _log.debug( + "Parsed CoT chat uid=%s cot_type=%s sender=%s target=%s chatroom=%s message_id=%r", + uid, + cot_type, + preferred_sender, + preferred_target, + chat_meta.get("chatroom"), + preferred_message_id, + ) + canonical = empty_canonical_event() + canonical["meta"]["source_format"] = "cot_xml" + canonical["meta"]["source_subtype"] = "message" + canonical["meta"]["parser_version"] = 1 + canonical["meta"]["raw_message"] = xml_text + canonical["identity"]["uid"] = uid + canonical["identity"]["callsign"] = preferred_sender + canonical["identity"]["short_name"] = canonical["identity"]["callsign"] + canonical["text"]["message"] = remarks + canonical["text"]["comment"] = remarks + canonical["tak"]["cot_version"] = root.attrib.get("version") + canonical["tak"]["cot_type"] = cot_type or None + canonical["tak"]["how"] = root.attrib.get("how") + canonical["tak"]["start_utc"] = root.attrib.get("start") + canonical["tak"]["stale_utc"] = root.attrib.get("stale") + canonical["tak"]["chat"] = { + "chatroom": chat_meta.get("chatroom"), + "message_id": preferred_message_id, + "sender_callsign": chat_meta.get("senderCallsign"), + "target": preferred_target, + } + return canonical + + point = root.find("point") + if point is None: + raise CoTParseError("missing point element") + + try: + lat = float(point.attrib["lat"]) + lon = float(point.attrib["lon"]) + except (KeyError, ValueError) as exc: + raise CoTParseError("invalid point coordinates") from exc + entity_kind = "object" if cot_type in OBJECT_COT_TYPES else "station" + aprs_symbol = map_cot_type_to_aprs_symbol(cot_type) + _log.debug( + "Parsed CoT position uid=%s cot_type=%s entity=%s lat=%s lon=%s mapped_symbol=%s%s remarks=%r", + uid, + cot_type, + entity_kind, + lat, + lon, + aprs_meta.get("symbol_table") or aprs_symbol.symbol_table or "", + aprs_meta.get("symbol_code") or aprs_symbol.symbol_code or "", + remarks, + ) canonical = empty_canonical_event() canonical["meta"]["source_format"] = "cot_xml" canonical["meta"]["source_subtype"] = "object" if entity_kind == "object" else "position" canonical["meta"]["parser_version"] = 1 canonical["meta"]["raw_message"] = xml_text canonical["identity"]["uid"] = uid - canonical["identity"]["callsign"] = uid - canonical["identity"]["short_name"] = uid - canonical["identity"]["object_type"] = "unknown" if entity_kind == "object" else "person" + canonical["identity"]["callsign"] = ( + _clean_callsign_like(contact_callsign) + or _clean_callsign_like(droid_callsign) + or uid + ) + canonical["identity"]["short_name"] = ( + _clean_callsign_like(contact_callsign) + or _clean_callsign_like(droid_callsign) + or uid + ) + canonical["identity"]["object_type"] = categorize_cot_type(cot_type) canonical["position"]["lat"] = lat canonical["position"]["lon"] = lon canonical["position"]["alt_m"] = _optional_float(point.attrib.get("hae")) @@ -83,6 +203,8 @@ def parse_cot_canonical(xml_text: str) -> dict[str, object]: canonical["tak"]["how"] = root.attrib.get("how") canonical["tak"]["start_utc"] = root.attrib.get("start") canonical["tak"]["stale_utc"] = root.attrib.get("stale") + canonical["comms"]["symbol_table"] = aprs_meta.get("symbol_table") or aprs_symbol.symbol_table + canonical["comms"]["symbol_code"] = aprs_meta.get("symbol_code") or aprs_symbol.symbol_code canonical["sartrack"]["custom"] = object_meta or None return canonical @@ -91,11 +213,40 @@ def canonical_to_cot_event( canonical: dict[str, object], *, raw: dict[str, object] | None = None, -) -> PositionEvent: +) -> PositionEvent | MessageEvent: identity = canonical["identity"] - position = canonical["position"] text = canonical["text"] tak = canonical["tak"] + meta = canonical["meta"] + + if meta.get("source_subtype") == "message": + source_id = str(identity["callsign"] or identity["uid"] or "") + if not source_id: + raise CoTParseError("canonical CoT message missing source id") + chat = tak.get("chat") if isinstance(tak.get("chat"), dict) else {} + target = None + message_id = None + if isinstance(chat, dict): + target = str(chat.get("target") or "").strip() or None + message_id = str(chat.get("message_id") or "").strip() or None + message = str(text.get("message") or text.get("comment") or text.get("remarks") or "") + event = MessageEvent( + id=source_id, + message=message, + target=target, + message_id=message_id, + source=Source.COT, + raw=build_raw_payload(canonical=canonical, raw_context=raw), + ) + _log.debug( + "CoT canonical -> message source=%s target=%s message_id=%r", + event.id, + event.target, + event.message_id, + ) + return event + + position = canonical["position"] uid = identity["uid"] if not uid: @@ -111,19 +262,21 @@ def canonical_to_cot_event( owner_id = None message = remarks tactical_name = None - symbol_table = None - symbol_code = None + symbol_mapping = map_cot_type_to_aprs_symbol(cot_type) + symbol_table = canonical.get("comms", {}).get("symbol_table") or symbol_mapping.symbol_table + symbol_code = canonical.get("comms", {}).get("symbol_code") or symbol_mapping.symbol_code custom = canonical.get("sartrack", {}).get("custom") if entity_kind == "object" and isinstance(custom, dict): object_name = str(custom.get("name") or "").strip() or None owner_id = str(custom.get("owner") or "").strip() or None tactical_name = str(custom.get("tactical_name") or "").strip() or None - symbol_table = str(custom.get("symbol_table") or "").strip() or None - symbol_code = str(custom.get("symbol_code") or "").strip() or None + symbol_table = str(custom.get("symbol_table") or symbol_table or "").strip() or None + symbol_code = str(custom.get("symbol_code") or symbol_code or "").strip() or None - return PositionEvent( - id=str(object_name or uid), + display_id = str(object_name or identity.get("callsign") or uid) + event = PositionEvent( + id=display_id, entity_kind=entity_kind, type=position_type, lat=float(position["lat"]), @@ -138,6 +291,15 @@ def canonical_to_cot_event( source=Source.COT, raw=build_raw_payload(canonical=canonical, raw_context=raw), ) + _log.debug( + "CoT canonical -> position id=%s entity=%s object=%s owner=%s tactical=%r", + event.id, + event.entity_kind, + event.object_name, + event.owner_id, + event.tactical_name, + ) + return event def _optional_float(value: object) -> float | None: @@ -149,6 +311,19 @@ def _optional_float(value: object) -> float | None: return None +def _clean_callsign_like(value: object) -> str | None: + if value is None: + return None + text = str(value).strip() + if not text: + return None + return text + + +def _is_chat_event(cot_type: str, chat_meta: dict[str, str]) -> bool: + return bool(chat_meta) or cot_type.startswith("b-t-f") + + __all__ = [ "CoTParseError", "canonical_to_cot_event", diff --git a/cot/symbol_mapper.py b/cot/symbol_mapper.py new file mode 100644 index 0000000..d383cae --- /dev/null +++ b/cot/symbol_mapper.py @@ -0,0 +1,229 @@ +from __future__ import annotations + +"""APRS <-> CoT symbol/category mapping backed by aprs_cot_fallback.json.""" + +import json +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path +from typing import Any + + +FALLBACK_PATH = Path(__file__).resolve().parent.parent / "aprs_cot_fallback.json" + + +@dataclass(frozen=True, slots=True) +class APRSCotSymbol: + symbol_table: str | None + symbol_code: str | None + category: str + cot_type: str | None = None + aprs_name: str | None = None + + +@lru_cache(maxsize=1) +def _mapping_data() -> dict[str, Any]: + with FALLBACK_PATH.open("r", encoding="utf-8") as handle: + return json.load(handle) + + +@lru_cache(maxsize=1) +def _reverse_symbol_defaults() -> dict[str, APRSCotSymbol]: + data = _mapping_data() + preferred: dict[str, str] = { + "person": "/[", + "vehicle": "/>", + "aircraft": "/^", + "helicopter": "/X", + "marine": "/s", + "fixed": "/-", + } + result: dict[str, APRSCotSymbol] = {} + for key, entry in data.get("symbols", {}).items(): + category = str(entry.get("category") or "unknown") + if category in preferred and preferred[category] != key: + continue + result[category] = APRSCotSymbol( + symbol_table=key[0] if len(key) >= 1 else None, + symbol_code=key[1] if len(key) >= 2 else None, + category=category, + cot_type=entry.get("cotType"), + aprs_name=entry.get("aprs"), + ) + return result + + +def load_symbol_mapping_document() -> dict[str, Any]: + """Return a copy of the editable APRS<->CoT mapping document.""" + + data = _mapping_data() + return { + "default": dict(data.get("default", {})), + "symbols": { + str(key): dict(value) if isinstance(value, dict) else value + for key, value in data.get("symbols", {}).items() + }, + } + + +def save_symbol_mapping_document(document: dict[str, Any]) -> dict[str, Any]: + """Persist the mapping file and clear runtime caches.""" + + normalized = { + "default": dict(document.get("default", {})), + "symbols": dict(sorted(document.get("symbols", {}).items())), + } + with FALLBACK_PATH.open("w", encoding="utf-8") as handle: + json.dump(normalized, handle, indent=2, ensure_ascii=True) + handle.write("\n") + _mapping_data.cache_clear() + _reverse_symbol_defaults.cache_clear() + return load_symbol_mapping_document() + + +def update_default_mapping(category: str, cot_type: str) -> dict[str, Any]: + """Update one default category fallback mapping.""" + + document = load_symbol_mapping_document() + document.setdefault("default", {})[str(category)] = str(cot_type) + return save_symbol_mapping_document(document) + + +def upsert_symbol_mapping( + key: str, + *, + aprs: str, + cot_type: str, + category: str, +) -> dict[str, Any]: + """Create or update one exact APRS symbol mapping.""" + + document = load_symbol_mapping_document() + document.setdefault("symbols", {})[str(key)] = { + "aprs": str(aprs), + "cotType": str(cot_type), + "category": str(category), + } + return save_symbol_mapping_document(document) + + +def delete_symbol_mapping(key: str) -> dict[str, Any]: + """Delete one exact APRS symbol mapping if present.""" + + document = load_symbol_mapping_document() + document.setdefault("symbols", {}).pop(str(key), None) + return save_symbol_mapping_document(document) + + +def map_aprs_symbol_to_cot( + symbol_table: str | None, + symbol_code: str | None, + *, + entity_kind: str | None = None, + default_category: str | None = None, + prefer_marker_for_objects: bool = False, +) -> APRSCotSymbol: + data = _mapping_data() + key = f"{symbol_table or ''}{symbol_code or ''}" + symbol_entry = data.get("symbols", {}).get(key) + if isinstance(symbol_entry, dict): + return APRSCotSymbol( + symbol_table=symbol_table, + symbol_code=symbol_code, + category=str(symbol_entry.get("category") or "unknown"), + cot_type=str(symbol_entry.get("cotType")) if symbol_entry.get("cotType") else None, + aprs_name=str(symbol_entry.get("aprs")) if symbol_entry.get("aprs") else None, + ) + + category = _infer_aprs_category( + symbol_table=symbol_table, + symbol_code=symbol_code, + entity_kind=entity_kind, + default_category=default_category, + ) + if prefer_marker_for_objects and entity_kind == "object": + return APRSCotSymbol(symbol_table, symbol_code, category=category, cot_type="b-m-p-w") + + cot_type = data.get("default", {}).get(category) or data.get("default", {}).get("unknown") + return APRSCotSymbol( + symbol_table=symbol_table, + symbol_code=symbol_code, + category=category, + cot_type=str(cot_type) if cot_type else None, + ) + + +def map_cot_type_to_aprs_symbol(cot_type: str | None) -> APRSCotSymbol: + category = categorize_cot_type(cot_type) + defaults = _reverse_symbol_defaults() + symbol = defaults.get(category) + if symbol is not None: + return symbol + return APRSCotSymbol(symbol_table=None, symbol_code=None, category=category, cot_type=cot_type) + + +def categorize_cot_type(cot_type: str | None) -> str: + value = (cot_type or "").strip() + if not value: + return "unknown" + if value == "b-m-p-w": + return "fixed" + if value.startswith("a-f-A-C-H"): + return "helicopter" + if value.startswith("a-f-A"): + return "aircraft" + if value.startswith("a-f-S"): + return "marine" + if value.startswith("a-f-G-E-V") or "-V-" in value: + return "vehicle" + if value.startswith("a-.-G-I"): + return "fixed" + if value.startswith("a-u-"): + return "unknown" + if value.startswith("a-f-G-U-C"): + return "person" + return "unknown" + + +def _infer_aprs_category( + *, + symbol_table: str | None, + symbol_code: str | None, + entity_kind: str | None = None, + default_category: str | None = None, +) -> str: + code = (symbol_code or "").strip() + table = (symbol_table or "").strip() + + if code == "X" and table == "/": + return "helicopter" + if code == "^" and table in {"/", "\\"}: + return "aircraft" + if code in {"Y", "s"}: + return "marine" + if code in {">", "<", "j", "v", "k", "a"}: + return "vehicle" + if code in {"[", "h", "b"}: + return "person" + if code in {"-", "#", "&", "_", "r"}: + return "fixed" + if default_category: + return default_category + if entity_kind == "object": + return "fixed" + if entity_kind == "station": + return "person" + return "unknown" + + +__all__ = [ + "APRSCotSymbol", + "categorize_cot_type", + "delete_symbol_mapping", + "load_symbol_mapping_document", + "map_aprs_symbol_to_cot", + "map_cot_type_to_aprs_symbol", + "save_symbol_mapping_document", + "update_default_mapping", + "upsert_symbol_mapping", +] diff --git a/gui/server.py b/gui/server.py index 3a8a038..ca5ee18 100644 --- a/gui/server.py +++ b/gui/server.py @@ -8,6 +8,12 @@ from dataclasses import dataclass from typing import Awaitable, Callable +from cot.symbol_mapper import ( + delete_symbol_mapping, + load_symbol_mapping_document, + update_default_mapping, + upsert_symbol_mapping, +) from core.model import MessageEvent, PositionEvent, Source from core.state import StateStore @@ -272,6 +278,77 @@ async def get_adapters() -> dict[str, object]: self.log.exception("GUI adapters endpoint failed") raise + @app.get("/api/symbol-map") + async def get_symbol_map() -> dict[str, object]: + try: + document = load_symbol_mapping_document() + self.log.debug( + "GUI symbol map returned %s exact mappings", + len(document.get("symbols", {})), + ) + return document + except Exception: + self.log.exception("GUI symbol map endpoint failed") + raise + + @app.post("/api/symbol-map/defaults/{category}") + async def update_symbol_default(category: str, payload: dict[str, object]) -> dict[str, object]: + try: + cot_type = str(payload.get("cot_type") or "").strip() + if not cot_type: + raise HTTPException(status_code=400, detail="cot_type is required") + self.log.info("GUI default symbol mapping update category=%s cot_type=%s", category, cot_type) + document = update_default_mapping(category, cot_type) + return {"ok": True, "document": document} + except HTTPException: + raise + except Exception: + self.log.exception("GUI default symbol mapping update failed for category=%s", category) + raise + + @app.post("/api/symbol-map/symbols/{symbol_key}") + async def update_symbol_mapping(symbol_key: str, payload: dict[str, object]) -> dict[str, object]: + try: + aprs = str(payload.get("aprs") or "").strip() + cot_type = str(payload.get("cot_type") or "").strip() + category = str(payload.get("category") or "").strip() + if len(symbol_key) != 2: + raise HTTPException(status_code=400, detail="symbol key must be 2 characters") + if not aprs or not cot_type or not category: + raise HTTPException(status_code=400, detail="aprs, cot_type and category are required") + self.log.info( + "GUI exact symbol mapping update key=%s category=%s cot_type=%s", + symbol_key, + category, + cot_type, + ) + document = upsert_symbol_mapping( + symbol_key, + aprs=aprs, + cot_type=cot_type, + category=category, + ) + return {"ok": True, "document": document} + except HTTPException: + raise + except Exception: + self.log.exception("GUI exact symbol mapping update failed for key=%s", symbol_key) + raise + + @app.delete("/api/symbol-map/symbols/{symbol_key}") + async def remove_symbol_mapping(symbol_key: str) -> dict[str, object]: + try: + if len(symbol_key) != 2: + raise HTTPException(status_code=400, detail="symbol key must be 2 characters") + self.log.info("GUI exact symbol mapping delete key=%s", symbol_key) + document = delete_symbol_mapping(symbol_key) + return {"ok": True, "document": document} + except HTTPException: + raise + except Exception: + self.log.exception("GUI exact symbol mapping delete failed for key=%s", symbol_key) + raise + @app.post("/api/adapters/{name}/config") async def update_adapter_config(name: str, payload: dict[str, object]) -> dict[str, object]: try: @@ -568,6 +645,40 @@ def _build_index_html(title: str) -> str: 
Waiting for action.
+
+

APRS <-> CoT Symbol Map

+
+ +
+
+
+
Default Categories
+
+
+
+
Add / Update Exact Symbol
+
+ + + + + + +
+
+
+
+

Logs

@@ -592,6 +703,7 @@ def _build_index_html(title: str) -> str: ...{initial}, expandedAdapters: {{}}, refreshPauseUntil: 0, + symbolMap: null, }}; function pauseRefresh(ms = 5000) {{ @@ -651,6 +763,9 @@ def _build_index_html(title: str) -> str: ${{renderField('tx_callsign', 'TX Callsign')}} ${{renderField('tx_destination', 'TX Destination')}} ${{renderField('tx_path', 'TX Path')}} + ${{editable.has('rx_enabled') ? `` : ''}} + ${{renderField('listen_url', 'Listen URL')}} + ${{renderField('chat_listen_url', 'Chat Listen URL')}} ${{editable.has('pytak_no_hello') ? `` : ''}} ${{renderField('cot_url', 'CoT URL')}} ${{renderField('tak_proto', 'TAK Proto', 'number')}} @@ -785,6 +900,45 @@ def _build_index_html(title: str) -> str: }} }} + function renderSymbolMap(mappingDocument) {{ + state.symbolMap = mappingDocument; + + const defaultsRoot = window.document.getElementById('symbol-defaults'); + defaultsRoot.innerHTML = ''; + const defaults = mappingDocument.default || {{}}; + for (const [category, cotType] of Object.entries(defaults)) {{ + const form = document.createElement('form'); + form.className = 'adapter-form'; + form.dataset.symbolDefault = category; + form.innerHTML = ` + +
+ +
+ `; + defaultsRoot.appendChild(form); + }} + + const listRoot = window.document.getElementById('symbol-list'); + listRoot.innerHTML = ''; + const symbols = Object.entries(mappingDocument.symbols || {{}}).sort(([left], [right]) => left.localeCompare(right)); + for (const [key, value] of symbols) {{ + const row = document.createElement('div'); + row.className = 'item'; + row.innerHTML = ` +
${{key}} ${{value.aprs || '-'}}
+
category=${{value.category || '-'}} | cotType=${{value.cotType || '-'}}
+
+ + +
+ `; + listRoot.appendChild(row); + }} + }} + async function updateAdapterConfig(name, payload) {{ const result = await fetchJson(`/api/adapters/${{name}}/config`, {{ method: 'POST', @@ -801,6 +955,44 @@ def _build_index_html(title: str) -> str: document.getElementById('tx-result').textContent = JSON.stringify(result, null, 2); }} + async function refreshSymbolMap() {{ + const mappingDocument = await fetchJson('/api/symbol-map'); + renderSymbolMap(mappingDocument); + bindSymbolMapForms(); + }} + + async function saveSymbolDefault(category, cotType) {{ + const result = await fetchJson(`/api/symbol-map/defaults/${{encodeURIComponent(category)}}`, {{ + method: 'POST', + headers: {{'Content-Type': 'application/json'}}, + body: JSON.stringify({{ cot_type: cotType }}), + }}); + document.getElementById('tx-result').textContent = JSON.stringify(result, null, 2); + renderSymbolMap(result.document); + bindSymbolMapForms(); + }} + + async function saveExactSymbolMapping(payload) {{ + const symbolKey = `${{payload.symbol_table || ''}}${{payload.symbol_code || ''}}`; + const result = await fetchJson(`/api/symbol-map/symbols/${{encodeURIComponent(symbolKey)}}`, {{ + method: 'POST', + headers: {{'Content-Type': 'application/json'}}, + body: JSON.stringify(payload), + }}); + document.getElementById('tx-result').textContent = JSON.stringify(result, null, 2); + renderSymbolMap(result.document); + bindSymbolMapForms(); + }} + + async function deleteExactSymbolMapping(symbolKey) {{ + const result = await fetchJson(`/api/symbol-map/symbols/${{encodeURIComponent(symbolKey)}}`, {{ + method: 'DELETE', + }}); + document.getElementById('tx-result').textContent = JSON.stringify(result, null, 2); + renderSymbolMap(result.document); + bindSymbolMapForms(); + }} + async function sendPositionPayload(payload) {{ const result = await fetchJson('/api/test/send-position', {{ method: 'POST', @@ -900,6 +1092,57 @@ def _build_index_html(title: str) -> str: }}); }} + function bindSymbolMapForms() {{ + document.querySelectorAll('[data-symbol-default]').forEach((form) => {{ + if (form.dataset.bound === '1') return; + form.dataset.bound = '1'; + form.addEventListener('submit', async (event) => {{ + event.preventDefault(); + pauseRefresh(8000); + const category = form.dataset.symbolDefault; + const cotType = form.querySelector('[name="cot_type"]').value; + await saveSymbolDefault(category, cotType); + }}); + }}); + + const symbolForm = document.getElementById('symbol-form'); + if (symbolForm.dataset.bound !== '1') {{ + symbolForm.dataset.bound = '1'; + symbolForm.addEventListener('submit', async (event) => {{ + event.preventDefault(); + pauseRefresh(8000); + const form = new FormData(symbolForm); + const payload = Object.fromEntries(form.entries()); + await saveExactSymbolMapping(payload); + }}); + }} + + document.querySelectorAll('[data-symbol-edit]').forEach((button) => {{ + if (button.dataset.bound === '1') return; + button.dataset.bound = '1'; + button.addEventListener('click', () => {{ + const key = button.dataset.symbolEdit; + const symbol = (state.symbolMap && state.symbolMap.symbols && state.symbolMap.symbols[key]) || {{}}; + const form = document.getElementById('symbol-form'); + form.querySelector('[name="symbol_table"]').value = key.slice(0, 1); + form.querySelector('[name="symbol_code"]').value = key.slice(1, 2); + form.querySelector('[name="aprs"]').value = symbol.aprs || ''; + form.querySelector('[name="category"]').value = symbol.category || ''; + form.querySelector('[name="cot_type"]').value = symbol.cotType || ''; + pauseRefresh(5000); + }}); + }}); + + document.querySelectorAll('[data-symbol-delete]').forEach((button) => {{ + if (button.dataset.bound === '1') return; + button.dataset.bound = '1'; + button.addEventListener('click', async () => {{ + pauseRefresh(8000); + await deleteExactSymbolMapping(button.dataset.symbolDelete); + }}); + }}); + }} + async function refresh() {{ if (isRefreshPaused()) {{ return; @@ -941,6 +1184,9 @@ def _build_index_html(title: str) -> str: renderLogs(logs.items || []); bindAdapterForms(); bindPresetButtons(); + if (!state.symbolMap) {{ + await refreshSymbolMap(); + }} }} catch (error) {{ document.getElementById('tx-result').textContent = `Refresh failed: ${{error.message}}`; }} @@ -967,6 +1213,11 @@ def _build_index_html(title: str) -> str: await refresh(); }}); + document.getElementById('symbol-map-refresh').addEventListener('click', async () => {{ + pauseRefresh(3000); + await refreshSymbolMap(); + }}); + refresh(); setInterval(refresh, 3000); diff --git a/main.py b/main.py index d55abc8..bc78fd2 100644 --- a/main.py +++ b/main.py @@ -180,7 +180,7 @@ async def update_cot(payload: dict[str, object]) -> dict[str, object]: get_config=cot.get_runtime_config, update_config=update_cot, reconnect=cot.reconnect_now, - editable_fields=("enabled", "cot_url", "pytak_no_hello", "tak_proto", "stale_seconds"), + editable_fields=("enabled", "cot_url", "rx_enabled", "listen_url", "chat_listen_url", "pytak_no_hello", "tak_proto", "stale_seconds"), ), } for mesh_adapter in mesh_adapters: @@ -221,6 +221,7 @@ async def update_mesh(payload: dict[str, object], adapter: MeshtasticAdapter = m agwpe_server.attach_state_store(state_store) aprs_is_server.attach_state_store(state_store) cot.attach_state_store(state_store) + cot.attach_router(router) sartrack.attach_state_store(state_store) sartrack_aprs.attach_state_store(state_store) for mesh in mesh_adapters: diff --git a/tests/test_cot_adapter.py b/tests/test_cot_adapter.py index be9a288..131a03a 100644 --- a/tests/test_cot_adapter.py +++ b/tests/test_cot_adapter.py @@ -42,7 +42,35 @@ async def test_send_ignores_message_events(self) -> None: await adapter.send(event) - self.assertTrue(adapter._tx_queue.empty()) + payload = await adapter._tx_queue.get() + self.assertIsInstance(payload, bytes) + self.assertIn(b" None: + adapter = CoTAdapter(CoTConfig(enabled=True)) + adapter._tx_queue = asyncio.Queue() + event = MessageEvent( + id="N0CALL-10", + target="TEAM1", + message="group hello", + source=Source.SARTRACK, + raw={ + "canonical": { + "meta": { + "source_subtype": "group_message", + } + } + }, + ) + + await adapter.send(event) + + payload = await adapter._tx_queue.get() + xml = payload.decode("utf-8") + self.assertIn('chatroom="N0CALL-10"', xml) + self.assertIn('callsign="N0CALL-10"', xml) def test_build_pytak_config(self) -> None: config = CoTConfig( diff --git a/tests/test_cot_mesh_adapters.py b/tests/test_cot_mesh_adapters.py index 3a2f5da..060c967 100644 --- a/tests/test_cot_mesh_adapters.py +++ b/tests/test_cot_mesh_adapters.py @@ -2,8 +2,10 @@ import unittest +from aprs.encoder import build_object_packet, build_position_packet from cot.encoder import encode_event_to_cot_xml from cot.parser import canonical_to_cot_event, parse_cot_canonical, parse_cot_xml +from cot.symbol_mapper import categorize_cot_type, map_aprs_symbol_to_cot, map_cot_type_to_aprs_symbol from core.model import MessageEvent, PositionEvent, Source from mesh.parser import decode_mesh_packet, decode_mesh_text_payload from mesh.encoder import encode_position_payload, encode_text_payload @@ -125,6 +127,20 @@ def test_object_refresh_is_filtered(self) -> None: class CoTParserTests(unittest.TestCase): + def test_symbol_mapper_uses_exact_aprs_match(self) -> None: + mapping = map_aprs_symbol_to_cot("/", ">", entity_kind="station") + + self.assertEqual(mapping.category, "vehicle") + self.assertEqual(mapping.cot_type, "a-f-G-E-V-C") + + def test_symbol_mapper_uses_safe_reverse_fallback(self) -> None: + mapping = map_cot_type_to_aprs_symbol("a-f-S") + + self.assertEqual(mapping.category, "marine") + self.assertEqual(mapping.symbol_table, "/") + self.assertEqual(mapping.symbol_code, "s") + self.assertEqual(categorize_cot_type("a-f-S"), "marine") + def test_station_cot_contains_contact_and_group(self) -> None: event = PositionEvent( id="N0CALL-10", @@ -134,6 +150,8 @@ def test_station_cot_contains_contact_and_group(self) -> None: lon=19.0, tactical_name="ALPHA", message="ready", + symbol_table="/", + symbol_code=">", source=Source.SARTRACK, ) @@ -141,6 +159,8 @@ def test_station_cot_contains_contact_and_group(self) -> None: self.assertIn('ready", xml) def test_cot_canonical_roundtrip_object(self) -> None: @@ -200,6 +220,155 @@ def test_roundtrip_object_cot(self) -> None: self.assertEqual(decoded.message, "marker") self.assertEqual(decoded.tactical_name, "pin-1") + def test_reverse_maps_station_symbol_from_external_cot_type(self) -> None: + xml = ( + '' + '' + 'vehicle' + "" + ) + + decoded = parse_cot_xml(xml) + + self.assertEqual(decoded.entity_kind, "station") + self.assertEqual(decoded.symbol_table, "/") + self.assertEqual(decoded.symbol_code, ">") + + def test_external_cot_station_can_be_encoded_back_to_aprs(self) -> None: + xml = ( + '' + '' + 'vehicle' + "" + ) + + event = parse_cot_xml(xml) + frame = build_position_packet(event, destination="APSAR", path="WIDE2-2") + + self.assertEqual(frame, "CAR-1>APSAR,WIDE2-2:!5006.00N/01906.00E>vehicle") + + def test_external_cot_object_can_be_encoded_back_to_aprs(self) -> None: + xml = ( + '' + '' + '' + '' + 'marker' + '' + "" + ) + + event = parse_cot_xml(xml) + frame = build_object_packet(event, source=event.owner_id or "N0CALL-10", destination="APSAR", path="WIDE2-2") + + self.assertIn("N0CALL-10>APSAR,WIDE2-2:;OBJ1", frame) + self.assertIn("5006.00N\\01906.00Ezmarker[:pin-1", frame) + + def test_chat_cot_is_decoded_to_message_event(self) -> None: + xml = ( + '' + '' + '' + '<__chat chatroom="N2CALL-10" groupOwner="false" id="01" senderCallsign="N0CALL-10">' + '' + '' + '' + 'test' + '' + "" + ) + + decoded = parse_cot_xml(xml) + + self.assertIsInstance(decoded, MessageEvent) + assert isinstance(decoded, MessageEvent) + self.assertEqual(decoded.id, "N0CALL-10") + self.assertEqual(decoded.target, "N2CALL-10") + self.assertEqual(decoded.message, "test") + self.assertEqual(decoded.message_id, "01") + + def test_wintak_chat_prefers_sender_callsign_and_message_id(self) -> None: + xml = ( + '' + '' + '' + '' + '<__chat id="All Chat Rooms" chatroom="All Chat Rooms" senderCallsign="SP9PET" groupOwner="false" ' + 'messageId="fa914ef0-0829-44d2-b2b5-3ae18eb4ae60">' + '' + '' + '' + 'at breach' + '' + '' + ) + + decoded = parse_cot_xml(xml) + + self.assertIsInstance(decoded, MessageEvent) + assert isinstance(decoded, MessageEvent) + self.assertEqual(decoded.id, "SP9PET") + self.assertEqual(decoded.target, "All Chat Rooms") + self.assertEqual(decoded.message, "at breach") + self.assertEqual(decoded.message_id, "fa914ef0-0829-44d2-b2b5-3ae18eb4ae60") + + def test_wintak_position_prefers_contact_callsign_for_station_id(self) -> None: + xml = ( + '' + '' + '' + '' + '' + '' + '' + '<__group name="Cyan" role="Team Member"/>' + '' + '' + '' + '' + ) + + decoded = parse_cot_xml(xml) + + self.assertIsInstance(decoded, PositionEvent) + assert isinstance(decoded, PositionEvent) + self.assertEqual(decoded.id, "SP9PET") + self.assertEqual(decoded.entity_kind, "station") + + def test_bulletin_message_uses_sender_chatroom_for_cot(self) -> None: + from cot.encoder import encode_message_to_cot_xml + + event = MessageEvent( + id="N0CALL-10", + target="BLN13", + message="bulletin", + source=Source.SARTRACK, + raw={ + "canonical": { + "meta": { + "source_subtype": "bulletin", + } + } + }, + ) + + xml = encode_message_to_cot_xml(event) + + self.assertIn('chatroom="N0CALL-10"', xml) + self.assertIn(' Date: Thu, 16 Apr 2026 05:40:52 +0200 Subject: [PATCH 4/6] drobna poprawka encodera czatu. --- cot/encoder.py | 6 +++++- tests/test_cot_mesh_adapters.py | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/cot/encoder.py b/cot/encoder.py index 11dca28..c15dfb0 100644 --- a/cot/encoder.py +++ b/cot/encoder.py @@ -207,7 +207,11 @@ def _chat_route(event: MessageEvent) -> tuple[str, str]: target = (event.target or "").strip() subtype = _message_subtype(event) - if subtype in {"bulletin", "group_message"}: + if subtype == "bulletin": + bulletin_room = target or "BULLETIN" + return bulletin_room, bulletin_room + + if subtype == "group_message": source_chat = (event.id or "").strip() or "UNKNOWN" return source_chat, source_chat diff --git a/tests/test_cot_mesh_adapters.py b/tests/test_cot_mesh_adapters.py index 060c967..eff4b0d 100644 --- a/tests/test_cot_mesh_adapters.py +++ b/tests/test_cot_mesh_adapters.py @@ -347,7 +347,7 @@ def test_wintak_position_prefers_contact_callsign_for_station_id(self) -> None: self.assertEqual(decoded.id, "SP9PET") self.assertEqual(decoded.entity_kind, "station") - def test_bulletin_message_uses_sender_chatroom_for_cot(self) -> None: + def test_bulletin_message_uses_target_chatroom_for_cot(self) -> None: from cot.encoder import encode_message_to_cot_xml event = MessageEvent( @@ -366,8 +366,8 @@ def test_bulletin_message_uses_sender_chatroom_for_cot(self) -> None: xml = encode_message_to_cot_xml(event) - self.assertIn('chatroom="N0CALL-10"', xml) - self.assertIn(' Date: Thu, 16 Apr 2026 18:09:42 +0200 Subject: [PATCH 5/6] koljena drobna poprawka cot tak encodera. --- cot/encoder.py | 6 ++++-- tests/test_cot_mesh_adapters.py | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cot/encoder.py b/cot/encoder.py index c15dfb0..bd4f94a 100644 --- a/cot/encoder.py +++ b/cot/encoder.py @@ -97,7 +97,8 @@ def encode_message_to_cot_xml(event: MessageEvent) -> str: now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp)) stale = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp + event.ttl)) chatroom, destination = _chat_route(event) - message_uid = f"GeoChat.{event.id}.{chatroom}.{event.message_id or event.timestamp}" + message_token = str(event.message_id or event.timestamp) + message_uid = f"GeoChat.{event.id}.{chatroom}.{message_token}" root = ET.Element( "event", @@ -129,7 +130,8 @@ def encode_message_to_cot_xml(event: MessageEvent) -> str: { "chatroom": chatroom, "groupOwner": "false", - "id": str(event.message_id or event.timestamp), + "id": chatroom, + "messageId": message_token, "senderCallsign": str(event.id), }, ) diff --git a/tests/test_cot_mesh_adapters.py b/tests/test_cot_mesh_adapters.py index eff4b0d..23f9f0f 100644 --- a/tests/test_cot_mesh_adapters.py +++ b/tests/test_cot_mesh_adapters.py @@ -367,6 +367,7 @@ def test_bulletin_message_uses_target_chatroom_for_cot(self) -> None: xml = encode_message_to_cot_xml(event) self.assertIn('chatroom="BLN13"', xml) + self.assertIn('__chat chatroom="BLN13" groupOwner="false" id="BLN13" messageId=', xml) self.assertIn(' Date: Fri, 17 Apr 2026 04:25:17 +0200 Subject: [PATCH 6/6] Add Meshtastic adapter using MeshTAK / ATAK_PLUGIN transport - add primary Meshtastic adapter flow based on TAK protobuf packets - support TCP connection lifecycle, reconnect backoff and heartbeat handling - encode core PositionEvent / MessageEvent into MeshTAK payloads - decode inbound Meshtastic packets back into core events - add meshtasticd simulator compatibility by unwrapping SIMULATOR_APP into embedded ATAK_PLUGIN - add echo suppression and duplicate protection for bridge-originated packets - improve mesh logging and reduce library log spam while keeping bridge-level diagnostics - expose mesh adapter runtime config through config.py and GUI controls --- .gitignore | 5 + config.py | 46 ++++- cot/__init__.py | 4 + cot/encoder.py | 84 +++----- cot/parser.py | 36 +++- cot/tak_mapper.py | 276 +++++++++++++++++++++++++ gui/server.py | 6 + main.py | 7 + mesh/__init__.py | 5 +- mesh/adapter.py | 188 ++++++++++++++--- mesh/encoder.py | 87 +++++++- mesh/parser.py | 351 +++++++++++++++++++++++++++++--- tests/test_cot_mesh_adapters.py | 264 +++++++++++++++++++++++- utils/logger.py | 14 ++ 14 files changed, 1262 insertions(+), 111 deletions(-) create mode 100644 cot/tak_mapper.py diff --git a/.gitignore b/.gitignore index eba1354..a820c31 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,8 @@ build/ .DS_Store manual aprs decode.md atak cot and mesh adapter proposition.md +run_meshtasticd_native.bat +run_meshtasticd_node1_4405.bat +Perplexity_meshtak.md +run_meshtasticd_node0_4404.bat +run_meshtasticd_node2_4406.bat diff --git a/config.py b/config.py index cb27c1e..337bb1c 100644 --- a/config.py +++ b/config.py @@ -80,10 +80,16 @@ "agwpe.server": False, "aprs.parser": True, "sartrack.tcp": False, - "meshtastic": False, + "meshtastic": True, + "meshtastic.mesh_interface": False, + "meshtastic.stream_interface": False, + "meshtastic.tcp_interface": False, "cot": True, "kiss": False, }, + "disabled_exact_loggers": [ + "meshtastic", + ], } DEMO = { @@ -100,13 +106,19 @@ MESH_CALLSIGN_ALIASES = {} MESH_ADAPTERS = [{'name': 'mesh4405', - 'enabled': False, - 'host': '192.168.0.117', - 'port': 4405, + 'enabled': True, + 'host': '127.0.0.1', + 'port': 4404, 'destination_id': '^all', 'hop_limit': None, 'aprs_channel_index': 0, - 'position_transport_mode': 'waypoint', + 'transport_mode': 'tak_packet', + 'position_transport_mode': 'tak_packet', + 'tak_rx_enabled': True, + 'tak_tx_enabled': True, + 'tak_detail_enabled': True, + 'tak_detail_compress': True, + 'tak_detail_compress_threshold': 180, 'waypoint_expire_seconds': 1800, 'waypoint_icon': 0, 'outbound_echo_ttl_seconds': 30, @@ -142,6 +154,7 @@ class GatewayConfig: mesh_callsign_aliases: dict[str, str] log_level: int disabled_logger_prefixes: list[str] + disabled_logger_names: list[str] def load_config() -> GatewayConfig: @@ -154,7 +167,13 @@ def load_config() -> GatewayConfig: destination_id=str(item.get("destination_id", "^all")), hop_limit=item.get("hop_limit"), aprs_channel_index=int(item.get("aprs_channel_index", 0)), + transport_mode=str(item.get("transport_mode", item.get("position_transport_mode", "tak_packet"))), position_transport_mode=str(item.get("position_transport_mode", "waypoint")), + tak_rx_enabled=bool(item.get("tak_rx_enabled", True)), + tak_tx_enabled=bool(item.get("tak_tx_enabled", True)), + tak_detail_enabled=bool(item.get("tak_detail_enabled", True)), + tak_detail_compress=bool(item.get("tak_detail_compress", True)), + tak_detail_compress_threshold=int(item.get("tak_detail_compress_threshold", 180)), waypoint_expire_seconds=int(item.get("waypoint_expire_seconds", 1800)), waypoint_icon=int(item.get("waypoint_icon", 0)), outbound_echo_ttl_seconds=int(item.get("outbound_echo_ttl_seconds", 30)), @@ -241,6 +260,7 @@ def load_config() -> GatewayConfig: mesh_callsign_aliases=dict(MESH_CALLSIGN_ALIASES), log_level=_parse_log_level(str(LOGGING.get("level", "DEBUG"))), disabled_logger_prefixes=_disabled_logger_prefixes(LOGGING.get("enabled_loggers", {})), + disabled_logger_names=_disabled_logger_names(LOGGING.get("disabled_exact_loggers", [])), ) @@ -341,6 +361,10 @@ def _disabled_logger_prefixes(enabled_loggers: dict[str, object]) -> list[str]: return disabled +def _disabled_logger_names(names: list[object]) -> list[str]: + return [str(name) for name in names if str(name).strip()] + + def persist_mesh_adapter_config(config: MeshtasticConfig) -> None: updated = [] found = False @@ -355,7 +379,13 @@ def persist_mesh_adapter_config(config: MeshtasticConfig) -> None: "destination_id": config.destination_id, "hop_limit": config.hop_limit, "aprs_channel_index": config.aprs_channel_index, + "transport_mode": config.transport_mode, "position_transport_mode": config.position_transport_mode, + "tak_rx_enabled": config.tak_rx_enabled, + "tak_tx_enabled": config.tak_tx_enabled, + "tak_detail_enabled": config.tak_detail_enabled, + "tak_detail_compress": config.tak_detail_compress, + "tak_detail_compress_threshold": config.tak_detail_compress_threshold, "waypoint_expire_seconds": config.waypoint_expire_seconds, "waypoint_icon": config.waypoint_icon, "outbound_echo_ttl_seconds": config.outbound_echo_ttl_seconds, @@ -378,7 +408,13 @@ def persist_mesh_adapter_config(config: MeshtasticConfig) -> None: "destination_id": config.destination_id, "hop_limit": config.hop_limit, "aprs_channel_index": config.aprs_channel_index, + "transport_mode": config.transport_mode, "position_transport_mode": config.position_transport_mode, + "tak_rx_enabled": config.tak_rx_enabled, + "tak_tx_enabled": config.tak_tx_enabled, + "tak_detail_enabled": config.tak_detail_enabled, + "tak_detail_compress": config.tak_detail_compress, + "tak_detail_compress_threshold": config.tak_detail_compress_threshold, "waypoint_expire_seconds": config.waypoint_expire_seconds, "waypoint_icon": config.waypoint_icon, "outbound_echo_ttl_seconds": config.outbound_echo_ttl_seconds, diff --git a/cot/__init__.py b/cot/__init__.py index 05b8c92..48cf567 100644 --- a/cot/__init__.py +++ b/cot/__init__.py @@ -2,6 +2,7 @@ from cot.encoder import encode_event_to_cot_xml, position_event_to_canonical from cot.parser import CoTParseError, canonical_to_cot_event, decode_cot_xml, parse_cot_canonical, parse_cot_xml from cot.symbol_mapper import APRSCotSymbol, categorize_cot_type, map_aprs_symbol_to_cot, map_cot_type_to_aprs_symbol +from cot.tak_mapper import classify_message_subtype, message_event_chat_route, message_event_subtype __all__ = [ "APRSCotSymbol", @@ -15,6 +16,9 @@ "encode_event_to_cot_xml", "map_aprs_symbol_to_cot", "map_cot_type_to_aprs_symbol", + "classify_message_subtype", + "message_event_chat_route", + "message_event_subtype", "parse_cot_canonical", "parse_cot_xml", "position_event_to_canonical", diff --git a/cot/encoder.py b/cot/encoder.py index bd4f94a..9b60c44 100644 --- a/cot/encoder.py +++ b/cot/encoder.py @@ -8,6 +8,12 @@ from core.canonical import empty_canonical_event from core.model import MessageEvent, PositionEvent from cot.symbol_mapper import map_aprs_symbol_to_cot +from cot.tak_mapper import ( + apply_tak_metadata_to_canonical, + message_event_chat_route, + message_event_to_tak_common, + position_event_to_tak_common, +) def encode_event_to_cot_xml(event: PositionEvent) -> str: @@ -16,6 +22,7 @@ def encode_event_to_cot_xml(event: PositionEvent) -> str: tak = canonical["tak"] identity = canonical["identity"] text = canonical["text"] + common = position_event_to_tak_common(event) now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp)) stale = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp + event.ttl)) @@ -49,17 +56,19 @@ def encode_event_to_cot_xml(event: PositionEvent) -> str: detail, "contact", { - "callsign": str(tak["contact_callsign"] or identity["short_name"] or identity["uid"] or event.id), + "callsign": str(common.contact_callsign or tak["contact_callsign"] or identity["short_name"] or identity["uid"] or event.id), }, ) ET.SubElement( detail, "__group", { - "name": str(tak["tak_group_name"] or "Ham Router"), - "role": str(tak["tak_group_role"] or _default_group_role(event)), + "name": str(common.cot_group_name or tak["tak_group_name"] or "Ham Router"), + "role": str(common.cot_group_role or tak["tak_group_role"] or _default_group_role(event)), }, ) + if common.battery is not None: + ET.SubElement(detail, "status", {"battery": str(common.battery)}) if event.symbol_table or event.symbol_code: symbol_mapping = map_aprs_symbol_to_cot( event.symbol_table, @@ -96,9 +105,10 @@ def encode_event_to_cot_xml(event: PositionEvent) -> str: def encode_message_to_cot_xml(event: MessageEvent) -> str: now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp)) stale = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp + event.ttl)) - chatroom, destination = _chat_route(event) + chatroom, destination = message_event_chat_route(event) message_token = str(event.message_id or event.timestamp) message_uid = f"GeoChat.{event.id}.{chatroom}.{message_token}" + common = message_event_to_tak_common(event) root = ET.Element( "event", @@ -124,6 +134,17 @@ def encode_message_to_cot_xml(event: MessageEvent) -> str: }, ) detail = ET.SubElement(root, "detail") + ET.SubElement(detail, "contact", {"callsign": common.contact_callsign}) + ET.SubElement( + detail, + "__group", + { + "name": common.cot_group_name, + "role": common.cot_group_role, + }, + ) + if common.battery is not None: + ET.SubElement(detail, "status", {"battery": str(common.battery)}) chat = ET.SubElement( detail, "__chat", @@ -166,6 +187,7 @@ def position_event_to_canonical(event: PositionEvent) -> dict[str, object]: entity_kind=event.entity_kind, prefer_marker_for_objects=event.is_object(), ) + common = position_event_to_tak_common(event) canonical = empty_canonical_event() canonical["meta"]["source_format"] = "cot_xml" canonical["meta"]["source_subtype"] = "position" @@ -187,16 +209,13 @@ def position_event_to_canonical(event: PositionEvent) -> dict[str, object]: canonical["tak"]["how"] = "m-g" canonical["tak"]["start_utc"] = canonical["position"]["timestamp_utc"] canonical["tak"]["stale_utc"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(event.timestamp + event.ttl)) - canonical["tak"]["contact_callsign"] = _contact_callsign(event) - canonical["tak"]["tak_group_name"] = "Ham Router" - canonical["tak"]["tak_group_role"] = _default_group_role(event) - return canonical - - -def _contact_callsign(event: PositionEvent) -> str: - if event.is_object(): - return event.tactical_name or event.object_name or event.id - return event.tactical_name or event.id + return apply_tak_metadata_to_canonical( + canonical, + contact_callsign=common.contact_callsign, + cot_group_name=common.cot_group_name, + cot_group_role=common.cot_group_role, + battery=common.battery, + ) def _default_group_role(event: PositionEvent) -> str: @@ -204,43 +223,6 @@ def _default_group_role(event: PositionEvent) -> str: return "Waypoint" return "Team Member" - -def _chat_route(event: MessageEvent) -> tuple[str, str]: - target = (event.target or "").strip() - subtype = _message_subtype(event) - - if subtype == "bulletin": - bulletin_room = target or "BULLETIN" - return bulletin_room, bulletin_room - - if subtype == "group_message": - source_chat = (event.id or "").strip() or "UNKNOWN" - return source_chat, source_chat - - if not target: - source_chat = (event.id or "").strip() or "UNKNOWN" - return source_chat, source_chat - - return target, target - - -def _message_subtype(event: MessageEvent) -> str: - raw = event.raw if isinstance(event.raw, dict) else {} - canonical = raw.get("canonical") - if isinstance(canonical, dict): - meta = canonical.get("meta") - if isinstance(meta, dict): - subtype = str(meta.get("source_subtype") or "").strip() - if subtype: - return subtype - - target = (event.target or "").strip().upper() - if target.startswith("BLN"): - return "bulletin" - if target.startswith("TEAM") or target in {"GRUPA", "GROUP", "ALL"}: - return "group_message" - return "private_message" - __all__ = [ "encode_message_to_cot_xml", "encode_event_to_cot_xml", diff --git a/cot/parser.py b/cot/parser.py index 9dd1c39..807201c 100644 --- a/cot/parser.py +++ b/cot/parser.py @@ -8,6 +8,7 @@ from core.canonical import empty_canonical_event from core.model import MessageEvent, PositionEvent, Source, build_raw_payload from cot.symbol_mapper import categorize_cot_type, map_cot_type_to_aprs_symbol +from cot.tak_mapper import apply_tak_metadata_to_canonical, classify_message_subtype, merge_tak_common_into_position_event class CoTParseError(ValueError): @@ -71,6 +72,9 @@ def parse_cot_canonical(xml_text: str) -> dict[str, object]: marti_dest: str | None = None contact_callsign: str | None = None droid_callsign: str | None = None + group_name: str | None = None + group_role: str | None = None + battery_pct: int | None = None if detail is not None: remarks_elem = detail.find("remarks") if remarks_elem is not None: @@ -85,6 +89,16 @@ def parse_cot_canonical(xml_text: str) -> dict[str, object]: droid_elem = detail.find("uid") if droid_elem is not None and droid_elem.attrib.get("Droid"): droid_callsign = str(droid_elem.attrib.get("Droid")).strip() or None + group_elem = detail.find("__group") + if group_elem is not None: + group_name = str(group_elem.attrib.get("name") or "").strip() or None + group_role = str(group_elem.attrib.get("role") or "").strip() or None + status_elem = detail.find("status") + if status_elem is not None and status_elem.attrib.get("battery") not in (None, ""): + try: + battery_pct = int(float(status_elem.attrib.get("battery"))) + except (TypeError, ValueError): + battery_pct = None object_elem = detail.find("ham_router_object") if object_elem is not None: object_meta = {str(key): str(value) for key, value in object_elem.attrib.items()} @@ -148,7 +162,13 @@ def parse_cot_canonical(xml_text: str) -> dict[str, object]: "sender_callsign": chat_meta.get("senderCallsign"), "target": preferred_target, } - return canonical + return apply_tak_metadata_to_canonical( + canonical, + contact_callsign=contact_callsign or droid_callsign, + cot_group_name=group_name, + cot_group_role=group_role, + battery=battery_pct, + ) point = root.find("point") if point is None: @@ -206,7 +226,13 @@ def parse_cot_canonical(xml_text: str) -> dict[str, object]: canonical["comms"]["symbol_table"] = aprs_meta.get("symbol_table") or aprs_symbol.symbol_table canonical["comms"]["symbol_code"] = aprs_meta.get("symbol_code") or aprs_symbol.symbol_code canonical["sartrack"]["custom"] = object_meta or None - return canonical + return apply_tak_metadata_to_canonical( + canonical, + contact_callsign=contact_callsign or droid_callsign, + cot_group_name=group_name, + cot_group_role=group_role, + battery=battery_pct, + ) def canonical_to_cot_event( @@ -238,6 +264,7 @@ def canonical_to_cot_event( source=Source.COT, raw=build_raw_payload(canonical=canonical, raw_context=raw), ) + canonical["meta"]["source_subtype"] = classify_message_subtype(event.target, event.message) _log.debug( "CoT canonical -> message source=%s target=%s message_id=%r", event.id, @@ -291,6 +318,11 @@ def canonical_to_cot_event( source=Source.COT, raw=build_raw_payload(canonical=canonical, raw_context=raw), ) + event = merge_tak_common_into_position_event( + event, + contact_callsign=str(tak.get("contact_callsign") or "").strip() or None, + device_callsign=str(identity.get("callsign") or "").strip() or None, + ) _log.debug( "CoT canonical -> position id=%s entity=%s object=%s owner=%s tactical=%r", event.id, diff --git a/cot/tak_mapper.py b/cot/tak_mapper.py new file mode 100644 index 0000000..cbd224e --- /dev/null +++ b/cot/tak_mapper.py @@ -0,0 +1,276 @@ +from __future__ import annotations + +"""Small TAK/CoT mapping helpers local to the CoT family. + +These helpers are intentionally not a second runtime model. They only provide +reusable field bundles for transports such as XML CoT and MeshTAK protobuf. +""" + +from dataclasses import dataclass +from typing import Any + +from core.model import MessageEvent, PositionEvent + + +@dataclass(slots=True) +class TAKCommonFields: + contact_callsign: str + device_callsign: str + cot_group_name: str + cot_group_role: str + mesh_team: str + mesh_role: str + battery: int | None + + +def position_event_to_tak_common(event: PositionEvent) -> TAKCommonFields: + return TAKCommonFields( + contact_callsign=_position_contact_callsign(event), + device_callsign=_position_device_callsign(event), + cot_group_name=_cot_group_name(event), + cot_group_role=_cot_group_role(event), + mesh_team=_mesh_team_name(event), + mesh_role=_mesh_role_name(event), + battery=_battery_pct(event), + ) + + +def message_event_to_tak_common(event: MessageEvent) -> TAKCommonFields: + return TAKCommonFields( + contact_callsign=(event.id or "UNKNOWN").strip() or "UNKNOWN", + device_callsign=(event.id or "UNKNOWN").strip() or "UNKNOWN", + cot_group_name=_cot_group_name(event), + cot_group_role=_cot_group_role(event), + mesh_team=_mesh_team_name(event), + mesh_role=_mesh_role_name(event), + battery=_battery_pct(event), + ) + + +def message_event_chat_route(event: MessageEvent) -> tuple[str, str]: + target = (event.target or "").strip() + subtype = message_event_subtype(event) + + if subtype == "bulletin": + bulletin_room = target or "BULLETIN" + return bulletin_room, bulletin_room + + if subtype == "group_message": + source_chat = (event.id or "").strip() or "UNKNOWN" + return source_chat, source_chat + + if not target: + source_chat = (event.id or "").strip() or "UNKNOWN" + return source_chat, source_chat + + return target, target + + +def message_event_subtype(event: MessageEvent) -> str: + raw = event.raw if isinstance(event.raw, dict) else {} + canonical = raw.get("canonical") + if isinstance(canonical, dict): + meta = canonical.get("meta") + if isinstance(meta, dict): + subtype = str(meta.get("source_subtype") or "").strip() + if subtype: + return subtype + + return classify_message_subtype(event.target, event.message) + + +def classify_message_subtype(target: str | None, message: str | None) -> str: + target_upper = (target or "").strip().upper() + message_upper = (message or "").strip().upper() + message_text = (message or "").strip() + + if target_upper.startswith("BLN"): + return "bulletin" + if target_upper.startswith("TEAM") or target_upper in {"GRUPA", "GROUP", "ALL", "ALL CHAT ROOMS"}: + return "group_message" + if message_upper.startswith("PING"): + return "ping" + if message_text.startswith("?"): + return "query" + return "private_message" + + +def merge_tak_common_into_position_event( + event: PositionEvent, + *, + contact_callsign: str | None, + device_callsign: str | None, +) -> PositionEvent: + updated_id = event.id + updated_tactical = event.tactical_name + updated_owner = event.owner_id + + if event.is_object(): + if contact_callsign and not updated_tactical: + updated_tactical = contact_callsign + if device_callsign and not updated_owner: + updated_owner = device_callsign + else: + if contact_callsign: + updated_id = contact_callsign + if contact_callsign != event.id: + updated_tactical = updated_tactical or contact_callsign + + return PositionEvent( + id=updated_id, + timestamp=event.timestamp, + source=event.source, + ttl=event.ttl, + raw=event.raw, + entity_kind=event.entity_kind, + type=event.type, + lat=event.lat, + lon=event.lon, + alt=event.alt, + speed=event.speed, + heading=event.heading, + tactical_name=updated_tactical, + message=event.message, + symbol_table=event.symbol_table, + symbol_code=event.symbol_code, + object_name=event.object_name, + owner_id=updated_owner, + ) + + +def apply_tak_metadata_to_canonical( + canonical: dict[str, Any], + *, + contact_callsign: str | None = None, + cot_group_name: str | None = None, + cot_group_role: str | None = None, + battery: int | None = None, +) -> dict[str, Any]: + if contact_callsign: + canonical["tak"]["contact_callsign"] = contact_callsign + if cot_group_name: + canonical["tak"]["tak_group_name"] = cot_group_name + if cot_group_role: + canonical["tak"]["tak_group_role"] = cot_group_role + if battery is not None: + canonical["telemetry"]["battery_pct"] = battery + return canonical + + +def _position_contact_callsign(event: PositionEvent) -> str: + if event.is_object(): + return event.tactical_name or event.object_name or event.id + return event.tactical_name or event.id + + +def _position_device_callsign(event: PositionEvent) -> str: + if event.is_object(): + return event.owner_id or event.id + return event.id + + +def _cot_group_name(event: PositionEvent | MessageEvent) -> str: + canonical = _raw_canonical(event) + tak = canonical.get("tak") if isinstance(canonical, dict) else None + if isinstance(tak, dict): + value = str(tak.get("tak_group_name") or "").strip() + if value: + return value + return "Ham Router" + + +def _cot_group_role(event: PositionEvent | MessageEvent) -> str: + canonical = _raw_canonical(event) + tak = canonical.get("tak") if isinstance(canonical, dict) else None + if isinstance(tak, dict): + value = str(tak.get("tak_group_role") or "").strip() + if value: + return value + if isinstance(event, PositionEvent) and event.is_object(): + return "Waypoint" + return "Team Member" + + +def _mesh_team_name(event: PositionEvent | MessageEvent) -> str: + canonical = _raw_canonical(event) + tak = canonical.get("tak") if isinstance(canonical, dict) else None + candidate = None + if isinstance(tak, dict): + candidate = str(tak.get("tak_group_name") or "").strip() + if candidate in { + "White", + "Yellow", + "Orange", + "Magenta", + "Red", + "Maroon", + "Purple", + "Dark_Blue", + "Blue", + "Cyan", + "Teal", + "Green", + "Dark_Green", + "Brown", + }: + return candidate + return "Cyan" + + +def _mesh_role_name(event: PositionEvent | MessageEvent) -> str: + canonical = _raw_canonical(event) + tak = canonical.get("tak") if isinstance(canonical, dict) else None + candidate = None + if isinstance(tak, dict): + candidate = str(tak.get("tak_group_role") or "").strip() + role_map = { + "Team Member": "TeamMember", + "Waypoint": "Unspecifed", + "Unknown Team Member": "TeamMember", + "TeamMember": "TeamMember", + "TeamLead": "TeamLead", + "HQ": "HQ", + "Sniper": "Sniper", + "Medic": "Medic", + "ForwardObserver": "ForwardObserver", + "RTO": "RTO", + "K9": "K9", + "Unspecifed": "Unspecifed", + } + if candidate and candidate in role_map: + return role_map[candidate] + if isinstance(event, PositionEvent) and event.is_object(): + return "Unspecifed" + return "TeamMember" + + +def _battery_pct(event: PositionEvent | MessageEvent) -> int | None: + canonical = _raw_canonical(event) + telemetry = canonical.get("telemetry") if isinstance(canonical, dict) else None + if isinstance(telemetry, dict): + value = telemetry.get("battery_pct") + try: + if value is None: + return None + return max(0, min(100, int(round(float(value))))) + except (TypeError, ValueError): + return None + return None + + +def _raw_canonical(event: PositionEvent | MessageEvent) -> dict[str, Any]: + raw = event.raw if isinstance(event.raw, dict) else {} + canonical = raw.get("canonical") + return canonical if isinstance(canonical, dict) else {} + + +__all__ = [ + "TAKCommonFields", + "apply_tak_metadata_to_canonical", + "classify_message_subtype", + "merge_tak_common_into_position_event", + "message_event_chat_route", + "message_event_subtype", + "message_event_to_tak_common", + "position_event_to_tak_common", +] diff --git a/gui/server.py b/gui/server.py index ca5ee18..7a2a79b 100644 --- a/gui/server.py +++ b/gui/server.py @@ -772,7 +772,13 @@ def _build_index_html(title: str) -> str: ${{renderField('stale_seconds', 'Stale Seconds', 'number')}} ${{renderField('destination_id', 'Destination ID')}} ${{renderField('aprs_channel_index', 'Mesh Channel Index', 'number')}} + ${{renderField('transport_mode', 'Mesh Transport Mode')}} ${{renderField('position_transport_mode', 'Position Mode')}} + ${{editable.has('tak_rx_enabled') ? `` : ''}} + ${{editable.has('tak_tx_enabled') ? `` : ''}} + ${{editable.has('tak_detail_enabled') ? `` : ''}} + ${{editable.has('tak_detail_compress') ? `` : ''}} + ${{renderField('tak_detail_compress_threshold', 'Mesh TAK Compress Threshold', 'number')}} ${{renderField('server_name', 'Server Name')}} ${{renderField('title', 'Title')}} ${{clientInfoHtml}} diff --git a/main.py b/main.py index bc78fd2..537d215 100644 --- a/main.py +++ b/main.py @@ -72,6 +72,7 @@ async def main() -> None: level=app_config.log_level, state_store=state_store, disabled_logger_prefixes=app_config.disabled_logger_prefixes, + disabled_logger_names=app_config.disabled_logger_names, ) log = logging.getLogger("main") aprs = AGWPEAdapter(app_config.agwpe) @@ -201,7 +202,13 @@ async def update_mesh(payload: dict[str, object], adapter: MeshtasticAdapter = m "port", "destination_id", "aprs_channel_index", + "transport_mode", "position_transport_mode", + "tak_rx_enabled", + "tak_tx_enabled", + "tak_detail_enabled", + "tak_detail_compress", + "tak_detail_compress_threshold", ), ) gui = GuiServer( diff --git a/mesh/__init__.py b/mesh/__init__.py index a385bc1..bd8dcd7 100644 --- a/mesh/__init__.py +++ b/mesh/__init__.py @@ -1,11 +1,12 @@ from mesh.adapter import MeshtasticAdapter, MeshtasticConfig -from mesh.encoder import build_waypoint_fields, build_waypoint_id, encode_position_payload, encode_text_payload +from mesh.encoder import build_waypoint_fields, build_waypoint_id, encode_mesh_tak_packet, encode_position_payload, encode_text_payload from mesh.filter import MeshFilter, MeshFilterDecision from mesh.parser import ( MeshtasticParseError, canonical_to_mesh_event, decode_mesh_packet, decode_mesh_position_packet, + decode_mesh_tak_packet_payload, decode_mesh_text_packet, decode_mesh_text_payload, parse_meshtastic_packet, @@ -22,8 +23,10 @@ "canonical_to_mesh_event", "decode_mesh_packet", "decode_mesh_position_packet", + "decode_mesh_tak_packet_payload", "decode_mesh_text_packet", "decode_mesh_text_payload", + "encode_mesh_tak_packet", "encode_position_payload", "encode_text_payload", "parse_meshtastic_packet", diff --git a/mesh/adapter.py b/mesh/adapter.py index eb92fbb..29c988b 100644 --- a/mesh/adapter.py +++ b/mesh/adapter.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import base64 import logging import re import socket @@ -10,12 +11,19 @@ import meshtastic.tcp_interface import meshtastic.stream_interface +from meshtastic.protobuf import portnums_pb2 from pubsub import pub from core.model import MessageEvent, PositionEvent, Source from core.router import EventRouter from core.state import StateStore -from mesh.encoder import build_waypoint_fields, build_waypoint_id, encode_position_payload, encode_text_payload +from mesh.encoder import ( + build_waypoint_fields, + build_waypoint_id, + encode_mesh_tak_packet, + encode_position_payload, + encode_text_payload, +) from mesh.mesh_filter import MeshFilter from mesh.parser import decode_mesh_packet, parse_meshtastic_packet @@ -26,19 +34,25 @@ @dataclass(slots=True) class MeshtasticConfig: name: str = "mesh" - host: str = "192.168.0.117" - port: int = 4406 + host: str = "127.0.0.1" + port: int = 4405 enabled: bool = True destination_id: str = "^all" hop_limit: int | None = None aprs_channel_index: int = 0 - position_transport_mode: str = "waypoint" + transport_mode: str = "tak_packet" + position_transport_mode: str = "tak_packet" waypoint_expire_seconds: int = 1800 waypoint_icon: int = 0 outbound_echo_ttl_seconds: int = 30 connect_quiet_seconds: float = 2.5 read_timeout_seconds: float = 2.0 reconnect_backoff_seconds: tuple[float, ...] = (2.0, 5.0, 10.0) + tak_rx_enabled: bool = True + tak_tx_enabled: bool = True + tak_detail_enabled: bool = True + tak_detail_compress: bool = True + tak_detail_compress_threshold: int = 180 callsign_aliases: dict[str, str] = field(default_factory=dict) @@ -57,6 +71,7 @@ def __init__( self._read_timeout_seconds = read_timeout_seconds self._socket_logger = socket_logger self._socket_endpoint = f"{hostname}:{port_number}" + self._last_read_timeout_log = 0.0 super().__init__( hostname=hostname, portNumber=port_number, @@ -107,11 +122,14 @@ def _readBytes(self, length) -> bytes | None: try: data = self.socket.recv(length) except socket.timeout: - self._socket_logger.debug( - "Meshtastic TCP read timeout on %s after %.1fs", - self._socket_endpoint, - self._read_timeout_seconds, - ) + now = time.monotonic() + if now - self._last_read_timeout_log >= 30.0: + self._socket_logger.debug( + "Meshtastic TCP read timeout on %s after %.1fs", + self._socket_endpoint, + self._read_timeout_seconds, + ) + self._last_read_timeout_log = now return None except ConnectionResetError as exc: self._socket_logger.warning( @@ -219,7 +237,7 @@ async def send_position(self, event: PositionEvent) -> None: self.log.debug("Meshtastic position send skipped: adapter disabled") return self.log.info( - "Meshtastic APRS position outbound entity=%s id=%s name=%s lat=%s lon=%s", + "Meshtastic outbound position entity=%s id=%s name=%s lat=%s lon=%s", event.entity_kind, event.id, event.tactical_name, @@ -245,6 +263,26 @@ async def send_position(self, event: PositionEvent) -> None: return try: + if self.config.tak_tx_enabled and self.config.transport_mode == "tak_packet": + payload, summary = encode_mesh_tak_packet( + event, + detail_enabled=self.config.tak_detail_enabled, + detail_compress=self.config.tak_detail_compress, + detail_compress_threshold=self.config.tak_detail_compress_threshold, + ) + self.log.info( + "TX | mesh-tak | adapter=%s variant=%s compressed=%s source=%s lat=%s lon=%s", + self.config.name, + summary["variant"], + summary["compressed"], + summary["contact_callsign"], + event.lat, + event.lon, + ) + self._remember_outbound_payload_bytes(payload) + await asyncio.to_thread(self._send_tak_blocking, payload, self.config.destination_id) + return + if self.config.position_transport_mode == "waypoint": name, description = build_waypoint_fields(event) waypoint_id = build_waypoint_id(event.object_name or event.id) @@ -284,8 +322,8 @@ async def send_message(self, event: MessageEvent) -> None: self.log.debug("Meshtastic message send skipped: id=%s reason=%s", event.id, decision.reason) return payload = encode_text_payload(event) - self.log.info("Meshtastic message outbound id=%s target=%s", event.id, event.target) - self.log.debug("Meshtastic APRS message payload: %s", payload) + self.log.info("Meshtastic outbound message id=%s target=%s", event.id, event.target) + self.log.debug("Meshtastic legacy text payload: %s", payload) if not self.interface: self.log.warning("Meshtastic message send skipped: no active interface") @@ -304,6 +342,25 @@ async def send_message(self, event: MessageEvent) -> None: return try: + if self.config.tak_tx_enabled and self.config.transport_mode == "tak_packet": + binary_payload, summary = encode_mesh_tak_packet( + event, + detail_enabled=self.config.tak_detail_enabled, + detail_compress=self.config.tak_detail_compress, + detail_compress_threshold=self.config.tak_detail_compress_threshold, + ) + self.log.info( + "TX | mesh-tak | adapter=%s variant=%s compressed=%s source=%s target=%s", + self.config.name, + summary["variant"], + summary["compressed"], + summary["contact_callsign"], + summary["target"], + ) + self._remember_outbound_payload_bytes(binary_payload) + await asyncio.to_thread(self._send_tak_blocking, binary_payload, destination_id) + return + self._remember_outbound_payload(payload) await asyncio.to_thread(self._send_text_blocking, payload, destination_id) except Exception as exc: @@ -332,6 +389,17 @@ def _send_text_blocking(self, text: str, destination_id: str) -> Any: hopLimit=self.config.hop_limit, ) + def _send_tak_blocking(self, payload: bytes, destination_id: str) -> Any: + assert self.interface is not None + return self.interface.sendData( + payload, + destinationId=destination_id, + wantAck=False, + channelIndex=self.config.aprs_channel_index, + hopLimit=self.config.hop_limit, + portNum=portnums_pb2.PortNum.ATAK_PLUGIN, + ) + def _send_waypoint_blocking( self, event: PositionEvent, @@ -561,17 +629,61 @@ def _on_any_receive(self, packet: dict[str, Any], interface: Any = None, topic: to_id = packet.get("toId") packet_id = packet.get("id") - self.log.debug( - "Meshtastic RX topic=%s packet_id=%s from=%s to=%s portnum=%s payload_keys=%s canonical=%s packet=%s", - topic_name, - packet_id, - from_id, - to_id, - portnum, - sorted(decoded.keys()), - canonical, - packet, - ) + if portnum == "ATAK_PLUGIN": + self.log.debug( + "Meshtastic RX topic=%s packet_id=%s from=%s to=%s portnum=%s payload_keys=%s canonical=%s", + topic_name, + packet_id, + from_id, + to_id, + portnum, + sorted(decoded.keys()), + canonical, + ) + else: + self.log.debug( + "Meshtastic RX topic=%s packet_id=%s from=%s to=%s portnum=%s payload_keys=%s", + topic_name, + packet_id, + from_id, + to_id, + portnum, + sorted(decoded.keys()), + ) + payload_b64 = canonical.get("meta", {}).get("raw_bytes_b64") + if ( + self.config.tak_rx_enabled + and portnum == "ATAK_PLUGIN" + and isinstance(payload_b64, str) + and payload_b64 + and self._is_recent_outbound_payload_bytes(payload_b64) + ): + self.log.debug("Meshtastic RX | mesh-tak | ignored bridge echo packet_id=%s", packet_id) + return + + if self.config.tak_rx_enabled and portnum == "ATAK_PLUGIN": + source_id = self._resolve_callsign_from_packet(packet) + if not source_id: + source_id = str(packet.get("fromId") or packet.get("from") or "MESH-UNKNOWN") + event = decode_mesh_packet( + packet, + source_id=source_id, + target_id=self._resolve_target_from_packet(packet), + ) + if event is None: + self.log.warning("Meshtastic RX | mesh-tak | dropped | reason=decode-none packet_id=%s", packet_id) + return + self.log.info( + "RX | mesh-tak | adapter=%s variant=%s compressed=%s kind=%s id=%s target=%s", + self.config.name, + (event.raw.get("mesh_tak_variant") if isinstance(event.raw, dict) else None) or "unknown", + bool(event.raw.get("mesh_tak_compressed")) if isinstance(event.raw, dict) else False, + event.kind, + event.id, + getattr(event, "target", None), + ) + self._publish_from_callback(event) + return def _publish_from_callback(self, event: PositionEvent | MessageEvent) -> None: if not self.router or not self.loop: @@ -601,7 +713,13 @@ def get_runtime_config(self) -> dict[str, object]: "port": self.config.port, "destination_id": self.config.destination_id, "aprs_channel_index": self.config.aprs_channel_index, + "transport_mode": self.config.transport_mode, "position_transport_mode": self.config.position_transport_mode, + "tak_rx_enabled": self.config.tak_rx_enabled, + "tak_tx_enabled": self.config.tak_tx_enabled, + "tak_detail_enabled": self.config.tak_detail_enabled, + "tak_detail_compress": self.config.tak_detail_compress, + "tak_detail_compress_threshold": self.config.tak_detail_compress_threshold, } async def apply_runtime_config(self, updates: dict[str, object]) -> dict[str, object]: @@ -615,8 +733,20 @@ async def apply_runtime_config(self, updates: dict[str, object]) -> dict[str, ob self.config.destination_id = str(updates["destination_id"]) if "aprs_channel_index" in updates and updates["aprs_channel_index"] is not None: self.config.aprs_channel_index = int(updates["aprs_channel_index"]) + if "transport_mode" in updates and updates["transport_mode"] is not None: + self.config.transport_mode = str(updates["transport_mode"]) if "position_transport_mode" in updates and updates["position_transport_mode"] is not None: self.config.position_transport_mode = str(updates["position_transport_mode"]) + if "tak_rx_enabled" in updates: + self.config.tak_rx_enabled = bool(updates["tak_rx_enabled"]) + if "tak_tx_enabled" in updates: + self.config.tak_tx_enabled = bool(updates["tak_tx_enabled"]) + if "tak_detail_enabled" in updates: + self.config.tak_detail_enabled = bool(updates["tak_detail_enabled"]) + if "tak_detail_compress" in updates: + self.config.tak_detail_compress = bool(updates["tak_detail_compress"]) + if "tak_detail_compress_threshold" in updates and updates["tak_detail_compress_threshold"] is not None: + self.config.tak_detail_compress_threshold = int(updates["tak_detail_compress_threshold"]) self._close_interface() if self.state_store: @@ -637,10 +767,22 @@ def _remember_outbound_payload(self, payload: str) -> None: self._prune_recent_outbound_payloads() self._recent_outbound_payloads[payload] = time.time() + def _remember_outbound_payload_bytes(self, payload: bytes) -> None: + self._prune_recent_outbound_payloads() + self._recent_outbound_payloads[payload.hex()] = time.time() + def _is_recent_outbound_payload(self, payload: str) -> bool: self._prune_recent_outbound_payloads() return payload in self._recent_outbound_payloads + def _is_recent_outbound_payload_bytes(self, payload_b64: str) -> bool: + self._prune_recent_outbound_payloads() + try: + decoded = base64.b64decode(payload_b64) + return decoded.hex() in self._recent_outbound_payloads + except Exception: + return False + def _prune_recent_outbound_payloads(self) -> None: now = time.time() expired = [ diff --git a/mesh/encoder.py b/mesh/encoder.py index 2c451e0..001ab59 100644 --- a/mesh/encoder.py +++ b/mesh/encoder.py @@ -1,8 +1,76 @@ from __future__ import annotations -"""Meshtastic payload encoder for compact APRS-derived mesh transport.""" +"""Meshtastic payload encoder with MeshTAK protobuf as the primary path.""" + +import zlib +from typing import Any + +from meshtastic.protobuf import atak_pb2 from core.model import MessageEvent, PositionEvent +from cot.encoder import encode_event_to_cot_xml +from cot.tak_mapper import message_event_chat_route, message_event_to_tak_common, position_event_to_tak_common + + +def encode_mesh_tak_packet( + event: PositionEvent | MessageEvent, + *, + detail_enabled: bool = True, + detail_compress: bool = True, + detail_compress_threshold: int = 180, +) -> tuple[bytes, dict[str, Any]]: + packet = atak_pb2.TAKPacket() + summary: dict[str, Any] = { + "variant": None, + "compressed": False, + "contact_callsign": None, + "device_callsign": None, + "target": None, + } + + if isinstance(event, PositionEvent): + common = position_event_to_tak_common(event) + _apply_common_fields(packet, common) + summary["contact_callsign"] = common.contact_callsign + summary["device_callsign"] = common.device_callsign + + if event.lat is not None and event.lon is not None: + packet.pli.latitude_i = int(round(float(event.lat) * 10_000_000)) + packet.pli.longitude_i = int(round(float(event.lon) * 10_000_000)) + if event.alt is not None: + packet.pli.altitude = int(round(float(event.alt))) + if event.speed is not None: + packet.pli.speed = max(0, int(round(float(event.speed)))) + if event.heading is not None: + packet.pli.course = max(0, int(round(float(event.heading)))) + summary["variant"] = "pli" + + if detail_enabled and event.is_object(): + detail_bytes = encode_event_to_cot_xml(event).encode("utf-8") + compressed, payload = _encode_detail_payload( + detail_bytes, + compress=detail_compress, + threshold=detail_compress_threshold, + ) + packet.detail = payload + packet.is_compressed = compressed + summary["compressed"] = compressed + summary["variant"] = "detail" if summary["variant"] is None else "pli+detail" + + return packet.SerializeToString(), summary + + common = message_event_to_tak_common(event) + _apply_common_fields(packet, common) + chatroom, destination = message_event_chat_route(event) + packet.chat.message = event.message + packet.chat.to = destination + if event.target: + packet.chat.to_callsign = event.target + summary["variant"] = "chat" + summary["target"] = destination + summary["contact_callsign"] = common.contact_callsign + summary["device_callsign"] = common.device_callsign + return packet.SerializeToString(), summary def encode_position_payload(event: PositionEvent) -> str: @@ -59,9 +127,26 @@ def build_waypoint_id(source_id: str) -> int: value = ((value * 33) + ord(char)) % 1_000_000_000 return value or 1 + +def _apply_common_fields(packet: atak_pb2.TAKPacket, common) -> None: + packet.contact.callsign = common.contact_callsign + packet.contact.device_callsign = common.device_callsign + packet.group.team = atak_pb2.Team.Value(common.mesh_team) + packet.group.role = atak_pb2.MemberRole.Value(common.mesh_role) + if common.battery is not None: + packet.status.battery = int(common.battery) + + +def _encode_detail_payload(data: bytes, *, compress: bool, threshold: int) -> tuple[bool, bytes]: + if compress and len(data) >= threshold: + return True, zlib.compress(data) + return False, data + + __all__ = [ "build_waypoint_fields", "build_waypoint_id", + "encode_mesh_tak_packet", "encode_position_payload", "encode_text_payload", ] diff --git a/mesh/parser.py b/mesh/parser.py index 2284426..3d09221 100644 --- a/mesh/parser.py +++ b/mesh/parser.py @@ -5,10 +5,19 @@ import base64 import logging import re +import zlib from typing import Any +from meshtastic.protobuf import atak_pb2 + from core.canonical import empty_canonical_event, iso_utc_now, to_iso_utc from core.model import MessageEvent, PositionEvent, Source, build_raw_payload +from cot.parser import CoTParseError, parse_cot_xml +from cot.tak_mapper import ( + apply_tak_metadata_to_canonical, + classify_message_subtype, + merge_tak_common_into_position_event, +) log = logging.getLogger("mesh.parser") @@ -33,8 +42,10 @@ def parse_meshtastic_packet(packet: dict[str, Any]) -> dict[str, Any]: if not isinstance(decoded, dict): decoded = {} + effective_portnum = _effective_portnum(decoded) + canonical["meta"]["source_format"] = "meshtastic_protobuf" - canonical["meta"]["source_subtype"] = decoded.get("portnum") + canonical["meta"]["source_subtype"] = effective_portnum canonical["meta"]["parser_version"] = 1 canonical["meta"]["raw_message"] = repr(packet) canonical["meta"]["received_at_utc"] = iso_utc_now() @@ -46,7 +57,7 @@ def parse_meshtastic_packet(packet: dict[str, Any]) -> dict[str, Any]: canonical["identity"]["node_id"] = packet.get("fromId") canonical["comms"]["channel_index"] = packet.get("channel") - canonical["comms"]["portnum"] = decoded.get("portnum") + canonical["comms"]["portnum"] = effective_portnum canonical["comms"]["hop_limit"] = packet.get("hopLimit") or packet.get("hop_limit") canonical["comms"]["hop_start"] = packet.get("hopStart") or packet.get("hop_start") canonical["comms"]["want_ack"] = packet.get("wantAck") or packet.get("want_ack") @@ -150,6 +161,24 @@ def decode_mesh_packet( decoded = packet.get("decoded") or {} portnum = canonical.get("comms", {}).get("portnum") + if portnum == "ATAK_PLUGIN": + payload_b64 = canonical.get("meta", {}).get("raw_bytes_b64") + if not payload_b64: + log.warning("Meshtastic RX | mesh-tak | dropped | reason=missing-payload packet=%s", packet) + return None + try: + payload = base64.b64decode(payload_b64) + except (TypeError, ValueError) as exc: + log.warning("Meshtastic RX | mesh-tak | dropped | reason=payload-b64 error=%s", exc) + return None + return decode_mesh_tak_packet_payload( + payload, + source_id=source_id, + target_id=target_id, + packet=packet, + canonical=canonical, + ) + if "position" in decoded or portnum == "POSITION_APP": native_position = decode_mesh_position_packet(packet, source_id=source_id, canonical=canonical) if native_position is not None: @@ -171,6 +200,201 @@ def decode_mesh_packet( ) +def decode_mesh_tak_packet_payload( + payload: bytes, + *, + source_id: str, + target_id: str | None = None, + packet: dict[str, Any] | None = None, + canonical: dict[str, Any] | None = None, +) -> PositionEvent | MessageEvent | None: + tak_packet = atak_pb2.TAKPacket() + try: + tak_packet.ParseFromString(payload) + except Exception as exc: + log.warning("Meshtastic RX | mesh-tak | dropped | reason=protobuf error=%s", exc) + return None + + common = _tak_common_fields(tak_packet, source_id) + variant = _tak_variant(tak_packet) + log.debug( + "Meshtastic RX | mesh-tak | decoded | variant=%s compressed=%s source=%s target=%s", + variant, + tak_packet.is_compressed, + common["contact_callsign"], + target_id, + ) + + if tak_packet.HasField("chat"): + event = MessageEvent( + id=common["contact_callsign"], + target=(tak_packet.chat.to_callsign or tak_packet.chat.to or target_id or "").strip() or None, + message=tak_packet.chat.message, + message_id=str(packet.get("id")) if isinstance(packet, dict) and packet.get("id") is not None else None, + source=Source.MESHTASTIC, + raw=build_raw_payload( + canonical=_mesh_tak_canonical_for_message( + tak_packet, + source_id=source_id, + target_id=target_id, + common=common, + packet_id=packet.get("id") if isinstance(packet, dict) else None, + ), + raw_context={ + "mesh_packet": packet, + "mesh_tak_variant": variant, + "mesh_tak_compressed": tak_packet.is_compressed, + }, + ), + ) + log.debug( + "Meshtastic RX | mesh-tak | event | variant=chat source=%s target=%s subtype=%s", + event.id, + event.target, + classify_message_subtype(event.target, event.message), + ) + return event + + detail_event = None + if tak_packet.detail: + detail_payload = bytes(tak_packet.detail) + if tak_packet.is_compressed: + try: + detail_payload = zlib.decompress(detail_payload) + except zlib.error as exc: + log.warning("Meshtastic RX | mesh-tak | dropped | reason=detail-decompress error=%s", exc) + return None + try: + detail_text = detail_payload.decode("utf-8") + except UnicodeDecodeError as exc: + log.warning("Meshtastic RX | mesh-tak | detail-ignored | reason=utf8 error=%s", exc) + detail_text = None + if detail_text: + try: + detail_event = parse_cot_xml(detail_text) + except CoTParseError as exc: + log.warning("Meshtastic RX | mesh-tak | detail-ignored | reason=cot-parse error=%s", exc) + + if detail_event is not None: + if isinstance(detail_event, PositionEvent): + detail_event = merge_tak_common_into_position_event( + detail_event, + contact_callsign=common["contact_callsign"], + device_callsign=common["device_callsign"], + ) + detail_event = PositionEvent( + id=detail_event.id, + timestamp=detail_event.timestamp, + source=Source.MESHTASTIC, + ttl=detail_event.ttl, + raw=detail_event.raw, + entity_kind=detail_event.entity_kind, + type=detail_event.type, + lat=detail_event.lat, + lon=detail_event.lon, + alt=detail_event.alt, + speed=detail_event.speed, + heading=detail_event.heading, + tactical_name=detail_event.tactical_name, + message=detail_event.message, + symbol_table=detail_event.symbol_table, + symbol_code=detail_event.symbol_code, + object_name=detail_event.object_name, + owner_id=detail_event.owner_id, + ) + if tak_packet.HasField("pli") and (detail_event.lat is None or detail_event.lon is None): + detail_event = PositionEvent( + id=detail_event.id, + timestamp=detail_event.timestamp, + source=Source.MESHTASTIC, + ttl=detail_event.ttl, + raw=detail_event.raw, + entity_kind=detail_event.entity_kind, + type=detail_event.type, + lat=float(tak_packet.pli.latitude_i) / 10_000_000.0, + lon=float(tak_packet.pli.longitude_i) / 10_000_000.0, + alt=float(tak_packet.pli.altitude) if tak_packet.pli.altitude else detail_event.alt, + speed=detail_event.speed, + heading=detail_event.heading, + tactical_name=detail_event.tactical_name, + message=detail_event.message, + symbol_table=detail_event.symbol_table, + symbol_code=detail_event.symbol_code, + object_name=detail_event.object_name, + owner_id=detail_event.owner_id, + ) + if isinstance(detail_event.raw, dict): + detail_event.raw["mesh_packet"] = packet + detail_event.raw["mesh_tak_variant"] = variant + detail_event.raw["mesh_tak_compressed"] = tak_packet.is_compressed + return detail_event + if isinstance(detail_event, MessageEvent): + detail_event = MessageEvent( + id=detail_event.id, + timestamp=detail_event.timestamp, + source=Source.MESHTASTIC, + ttl=detail_event.ttl, + raw=detail_event.raw, + message=detail_event.message, + target=detail_event.target, + message_id=detail_event.message_id, + ) + if isinstance(detail_event.raw, dict): + detail_event.raw["mesh_packet"] = packet + detail_event.raw["mesh_tak_variant"] = variant + detail_event.raw["mesh_tak_compressed"] = tak_packet.is_compressed + return detail_event + + if tak_packet.HasField("pli"): + canonical_payload = canonical or empty_canonical_event() + canonical_payload["meta"]["source_format"] = "meshtastic_tak" + canonical_payload["meta"]["source_subtype"] = "position" + canonical_payload["identity"]["callsign"] = common["contact_callsign"] + canonical_payload["identity"]["uid"] = common["device_callsign"] + canonical_payload["position"]["lat"] = float(tak_packet.pli.latitude_i) / 10_000_000.0 + canonical_payload["position"]["lon"] = float(tak_packet.pli.longitude_i) / 10_000_000.0 + canonical_payload["position"]["alt_m"] = float(tak_packet.pli.altitude) if tak_packet.pli.altitude else None + canonical_payload["motion"]["speed_mps"] = float(tak_packet.pli.speed) if tak_packet.pli.speed else None + canonical_payload["motion"]["course_deg"] = float(tak_packet.pli.course) if tak_packet.pli.course else None + apply_tak_metadata_to_canonical( + canonical_payload, + contact_callsign=common["contact_callsign"], + cot_group_name=common["group_name"], + cot_group_role=common["group_role"], + battery=common["battery"], + ) + event = PositionEvent( + id=common["contact_callsign"], + entity_kind="station", + type="position", + lat=float(tak_packet.pli.latitude_i) / 10_000_000.0, + lon=float(tak_packet.pli.longitude_i) / 10_000_000.0, + alt=float(tak_packet.pli.altitude) if tak_packet.pli.altitude else None, + speed=float(tak_packet.pli.speed) if tak_packet.pli.speed else None, + heading=float(tak_packet.pli.course) if tak_packet.pli.course else None, + tactical_name=common["contact_callsign"], + source=Source.MESHTASTIC, + raw=build_raw_payload( + canonical=canonical_payload, + raw_context={ + "mesh_packet": packet, + "mesh_tak_variant": variant, + "mesh_tak_compressed": tak_packet.is_compressed, + }, + ), + ) + log.debug( + "Meshtastic RX | mesh-tak | event | variant=pli source=%s lat=%s lon=%s", + event.id, + event.lat, + event.lon, + ) + return event + + log.warning("Meshtastic RX | mesh-tak | dropped | reason=unsupported-variant") + return None + + def decode_mesh_text_packet( packet: dict[str, Any], *, @@ -327,34 +551,19 @@ def _apply_telemetry_fields(canonical: dict[str, Any], telemetry: Any) -> None: dest["air_util_tx_pct"] = device_metrics.get("airUtilTx") dest["uptime_s"] = device_metrics.get("uptimeSeconds") - environment_metrics = telemetry.get("environmentMetrics") - if isinstance(environment_metrics, dict): - dest["temperature_c"] = environment_metrics.get("temperature") - dest["humidity_pct"] = environment_metrics.get("relativeHumidity") - dest["pressure_hpa"] = environment_metrics.get("barometricPressure") - dest["gas_resistance_mohm"] = environment_metrics.get("gasResistance") - - power_metrics = telemetry.get("powerMetrics") - if isinstance(power_metrics, dict): - dest["voltage_v"] = power_metrics.get("voltage") or dest["voltage_v"] - dest["current_a"] = power_metrics.get("current") - dest["power_w"] = power_metrics.get("power") - - local_stats = telemetry.get("localStats") - if isinstance(local_stats, dict): - dest["uptime_s"] = local_stats.get("uptimeSeconds") or dest["uptime_s"] - dest["channel_utilization_pct"] = ( - local_stats.get("channelUtilization") or dest["channel_utilization_pct"] - ) - dest["air_util_tx_pct"] = local_stats.get("airUtilTx") or dest["air_util_tx_pct"] - def _apply_simulator_fields(canonical: dict[str, Any], simulator: Any) -> None: if not isinstance(simulator, dict): return portnum = simulator.get("portnum") - if portnum and not canonical["meta"]["source_subtype"]: + # meshtasticd wraps simulated packets as SIMULATOR_APP but preserves the + # inner application in decoded.simulator.portnum. We only promote that + # embedded port when it exists; real hardware never uses this wrapper. + if portnum and canonical["comms"]["portnum"] == "SIMULATOR_APP": + canonical["meta"]["source_subtype"] = portnum + canonical["comms"]["portnum"] = portnum + elif portnum and not canonical["meta"]["source_subtype"]: canonical["meta"]["source_subtype"] = portnum if portnum and not canonical["comms"]["portnum"]: canonical["comms"]["portnum"] = portnum @@ -365,8 +574,8 @@ def _apply_simulator_fields(canonical: dict[str, Any], simulator: Any) -> None: def _apply_tak_fields(canonical: dict[str, Any], decoded: dict[str, Any]) -> None: - portnum = decoded.get("portnum") - if portnum != "TAK_APP": + portnum = _effective_portnum(decoded) + if portnum != "ATAK_PLUGIN": return canonical["tak"]["detail_xml"] = canonical["meta"]["raw_bytes_b64"] @@ -375,6 +584,12 @@ def _apply_tak_fields(canonical: dict[str, Any], decoded: dict[str, Any]) -> Non def _extract_payload_b64(decoded: dict[str, Any]) -> str | None: + simulator = decoded.get("simulator") + if decoded.get("portnum") == "SIMULATOR_APP" and isinstance(simulator, dict): + data_b64 = simulator.get("data") + if isinstance(data_b64, str) and data_b64: + return data_b64 + payload = decoded.get("payload") if isinstance(payload, bytes): return base64.b64encode(payload).decode("ascii") @@ -385,7 +600,6 @@ def _extract_payload_b64(decoded: dict[str, Any]) -> str | None: if isinstance(payload, bytes): return base64.b64encode(payload).decode("ascii") - simulator = decoded.get("simulator") if isinstance(simulator, dict): data_b64 = simulator.get("data") if isinstance(data_b64, str) and data_b64: @@ -394,6 +608,15 @@ def _extract_payload_b64(decoded: dict[str, Any]) -> str | None: return None +def _effective_portnum(decoded: dict[str, Any]) -> Any: + portnum = decoded.get("portnum") + if portnum == "SIMULATOR_APP": + simulator = decoded.get("simulator") + if isinstance(simulator, dict) and simulator.get("portnum"): + return simulator.get("portnum") + return portnum + + def _parse_pipe_payload(text: str) -> dict[str, str]: values: dict[str, str] = {} for part in text.split("|")[1:]: @@ -451,11 +674,85 @@ def _safe_str(value: Any) -> str | None: return str(value) +def _tak_variant(packet: atak_pb2.TAKPacket) -> str: + if packet.HasField("chat"): + return "chat" + if packet.HasField("pli") and packet.detail: + return "pli+detail" + if packet.HasField("pli"): + return "pli" + if packet.detail: + return "detail" + return "unknown" + + +def _tak_common_fields(packet: atak_pb2.TAKPacket, source_id: str) -> dict[str, Any]: + contact_callsign = packet.contact.callsign.strip() if packet.HasField("contact") and packet.contact.callsign else source_id + device_callsign = ( + packet.contact.device_callsign.strip() + if packet.HasField("contact") and packet.contact.device_callsign + else source_id + ) + group_name = atak_pb2.Team.Name(packet.group.team) if packet.HasField("group") else "Cyan" + group_role = atak_pb2.MemberRole.Name(packet.group.role) if packet.HasField("group") else "TeamMember" + battery = int(packet.status.battery) if packet.HasField("status") else None + role_display = { + "TeamMember": "Team Member", + "TeamLead": "TeamLead", + "HQ": "HQ", + "Sniper": "Sniper", + "Medic": "Medic", + "ForwardObserver": "ForwardObserver", + "RTO": "RTO", + "K9": "K9", + "Unspecifed": "Unspecifed", + }.get(group_role, group_role) + return { + "contact_callsign": contact_callsign, + "device_callsign": device_callsign, + "group_name": group_name, + "group_role": role_display, + "battery": battery, + } + + +def _mesh_tak_canonical_for_message( + packet: atak_pb2.TAKPacket, + *, + source_id: str, + target_id: str | None, + common: dict[str, Any], + packet_id: Any, +) -> dict[str, Any]: + canonical = empty_canonical_event() + canonical["meta"]["source_format"] = "meshtastic_tak" + canonical["meta"]["source_subtype"] = classify_message_subtype( + packet.chat.to_callsign or packet.chat.to or target_id, + packet.chat.message, + ) + canonical["meta"]["parser_version"] = 1 + canonical["meta"]["message_id"] = str(packet_id) if packet_id is not None else None + canonical["identity"]["uid"] = common["device_callsign"] or source_id + canonical["identity"]["callsign"] = common["contact_callsign"] or source_id + canonical["identity"]["short_name"] = common["contact_callsign"] or source_id + canonical["text"]["text"] = packet.chat.message + canonical["sartrack"]["receiver"] = packet.chat.to_callsign or packet.chat.to or target_id + apply_tak_metadata_to_canonical( + canonical, + contact_callsign=common["contact_callsign"], + cot_group_name=common["group_name"], + cot_group_role=common["group_role"], + battery=common["battery"], + ) + return canonical + + __all__ = [ "MeshtasticParseError", "canonical_to_mesh_event", "decode_mesh_packet", "decode_mesh_position_packet", + "decode_mesh_tak_packet_payload", "decode_mesh_text_packet", "decode_mesh_text_payload", "parse_meshtastic_packet", diff --git a/tests/test_cot_mesh_adapters.py b/tests/test_cot_mesh_adapters.py index 23f9f0f..c5d39e0 100644 --- a/tests/test_cot_mesh_adapters.py +++ b/tests/test_cot_mesh_adapters.py @@ -8,7 +8,7 @@ from cot.symbol_mapper import categorize_cot_type, map_aprs_symbol_to_cot, map_cot_type_to_aprs_symbol from core.model import MessageEvent, PositionEvent, Source from mesh.parser import decode_mesh_packet, decode_mesh_text_payload -from mesh.encoder import encode_position_payload, encode_text_payload +from mesh.encoder import encode_mesh_tak_packet, encode_position_payload, encode_text_payload from mesh.mesh_filter import MeshFilter @@ -81,6 +81,268 @@ def test_roundtrip_aprs_text_message_payload(self) -> None: self.assertEqual(decoded.message, "test") self.assertEqual(decoded.message_id, "01") + def test_roundtrip_mesh_tak_station_position(self) -> None: + event = PositionEvent( + id="N0CALL-10", + entity_kind="station", + type="position", + lat=50.0647, + lon=19.9450, + alt=200, + source=Source.SARTRACK, + ) + + payload, summary = encode_mesh_tak_packet(event) + packet = { + "id": 101, + "fromId": "!abcd1234", + "toId": "^all", + "decoded": { + "portnum": "ATAK_PLUGIN", + "payload": payload, + }, + } + + decoded = decode_mesh_packet(packet, source_id="N0CALL-10") + + self.assertEqual(summary["variant"], "pli") + self.assertIsInstance(decoded, PositionEvent) + assert isinstance(decoded, PositionEvent) + self.assertEqual(decoded.id, "N0CALL-10") + self.assertAlmostEqual(decoded.lat or 0.0, 50.0647, places=4) + self.assertAlmostEqual(decoded.lon or 0.0, 19.9450, places=4) + self.assertEqual(int(decoded.alt or 0), 200) + + def test_roundtrip_mesh_tak_object_position(self) -> None: + event = PositionEvent( + id="OBJ1", + entity_kind="object", + type="object", + lat=50.1, + lon=19.1, + tactical_name="pin-1", + message="marker", + object_name="OBJ1", + owner_id="N0CALL-10", + source=Source.SARTRACK, + ) + + payload, summary = encode_mesh_tak_packet(event, detail_enabled=True) + packet = { + "id": 102, + "fromId": "!abcd1234", + "toId": "^all", + "decoded": { + "portnum": "ATAK_PLUGIN", + "payload": payload, + }, + } + + decoded = decode_mesh_packet(packet, source_id="N0CALL-10") + + self.assertEqual(summary["variant"], "pli+detail") + self.assertIsInstance(decoded, PositionEvent) + assert isinstance(decoded, PositionEvent) + self.assertEqual(decoded.entity_kind, "object") + self.assertEqual(decoded.object_name, "OBJ1") + self.assertEqual(decoded.owner_id, "N0CALL-10") + self.assertEqual(decoded.tactical_name, "pin-1") + + def test_roundtrip_mesh_tak_private_message(self) -> None: + event = MessageEvent( + id="N0CALL-10", + target="SP9PET-10", + message="test", + source=Source.SARTRACK, + ) + + payload, summary = encode_mesh_tak_packet(event) + packet = { + "id": 103, + "fromId": "!abcd1234", + "toId": "^all", + "decoded": { + "portnum": "ATAK_PLUGIN", + "payload": payload, + }, + } + + decoded = decode_mesh_packet(packet, source_id="N0CALL-10") + + self.assertEqual(summary["variant"], "chat") + self.assertIsInstance(decoded, MessageEvent) + assert isinstance(decoded, MessageEvent) + self.assertEqual(decoded.id, "N0CALL-10") + self.assertEqual(decoded.target, "SP9PET-10") + self.assertEqual(decoded.message, "test") + + def test_roundtrip_mesh_tak_group_message(self) -> None: + event = MessageEvent( + id="N0CALL-10", + target="TEAM1", + message="group hello", + source=Source.SARTRACK, + raw={"canonical": {"meta": {"source_subtype": "group_message"}}}, + ) + + payload, _ = encode_mesh_tak_packet(event) + packet = { + "id": 104, + "fromId": "!abcd1234", + "toId": "^all", + "decoded": { + "portnum": "ATAK_PLUGIN", + "payload": payload, + }, + } + + decoded = decode_mesh_packet(packet, source_id="N0CALL-10") + + self.assertIsInstance(decoded, MessageEvent) + assert isinstance(decoded, MessageEvent) + self.assertEqual(decoded.target, "TEAM1") + self.assertEqual(decoded.message, "group hello") + self.assertEqual(decoded.raw["canonical"]["meta"]["source_subtype"], "group_message") + + def test_roundtrip_mesh_tak_bulletin_message(self) -> None: + event = MessageEvent( + id="N0CALL-10", + target="BLN13", + message="bulletin", + source=Source.SARTRACK, + raw={"canonical": {"meta": {"source_subtype": "bulletin"}}}, + ) + + payload, _ = encode_mesh_tak_packet(event) + packet = { + "id": 105, + "fromId": "!abcd1234", + "toId": "^all", + "decoded": { + "portnum": "ATAK_PLUGIN", + "payload": payload, + }, + } + + decoded = decode_mesh_packet(packet, source_id="N0CALL-10") + + self.assertIsInstance(decoded, MessageEvent) + assert isinstance(decoded, MessageEvent) + self.assertEqual(decoded.target, "BLN13") + self.assertEqual(decoded.raw["canonical"]["meta"]["source_subtype"], "bulletin") + + def test_mesh_tak_preserves_group_and_status_metadata(self) -> None: + event = PositionEvent( + id="N0CALL-10", + entity_kind="station", + type="position", + lat=50.0, + lon=19.0, + source=Source.SARTRACK, + raw={ + "canonical": { + "tak": { + "tak_group_name": "Cyan", + "tak_group_role": "Team Member", + }, + "telemetry": { + "battery_pct": 78, + }, + } + }, + ) + + payload, _ = encode_mesh_tak_packet(event) + packet = { + "id": 106, + "fromId": "!abcd1234", + "toId": "^all", + "decoded": { + "portnum": "ATAK_PLUGIN", + "payload": payload, + }, + } + + decoded = decode_mesh_packet(packet, source_id="N0CALL-10") + + self.assertIsInstance(decoded, PositionEvent) + assert isinstance(decoded, PositionEvent) + canonical = decoded.raw["canonical"] + self.assertEqual(canonical["tak"]["tak_group_name"], "Cyan") + self.assertEqual(canonical["telemetry"]["battery_pct"], 78) + + def test_mesh_tak_compressed_detail_rx_succeeds(self) -> None: + event = PositionEvent( + id="OBJ2", + entity_kind="object", + type="object", + lat=50.1, + lon=19.1, + tactical_name="pin-2", + message="marker", + object_name="OBJ2", + owner_id="N0CALL-10", + source=Source.SARTRACK, + ) + + payload, summary = encode_mesh_tak_packet( + event, + detail_enabled=True, + detail_compress=True, + detail_compress_threshold=1, + ) + packet = { + "id": 107, + "fromId": "!abcd1234", + "toId": "^all", + "decoded": { + "portnum": "ATAK_PLUGIN", + "payload": payload, + }, + } + + decoded = decode_mesh_packet(packet, source_id="N0CALL-10") + + self.assertTrue(summary["compressed"]) + self.assertIsInstance(decoded, PositionEvent) + assert isinstance(decoded, PositionEvent) + self.assertEqual(decoded.object_name, "OBJ2") + + def test_mesh_tak_malformed_compressed_detail_fails_safely(self) -> None: + from meshtastic.protobuf import atak_pb2 + + tak = atak_pb2.TAKPacket() + tak.is_compressed = True + tak.detail = b"not-valid-zlib" + packet = { + "id": 108, + "fromId": "!abcd1234", + "toId": "^all", + "decoded": { + "portnum": "ATAK_PLUGIN", + "payload": tak.SerializeToString(), + }, + } + + decoded = decode_mesh_packet(packet, source_id="N0CALL-10") + + self.assertIsNone(decoded) + + def test_mesh_tak_malformed_protobuf_fails_safely(self) -> None: + packet = { + "id": 109, + "fromId": "!abcd1234", + "toId": "^all", + "decoded": { + "portnum": "ATAK_PLUGIN", + "payload": b"not-a-real-protobuf", + }, + } + + decoded = decode_mesh_packet(packet, source_id="N0CALL-10") + + self.assertIsNone(decoded) + class MeshFilterTests(unittest.TestCase): def test_station_position_is_throttled_when_unchanged(self) -> None: diff --git a/utils/logger.py b/utils/logger.py index 5e82917..bfcc708 100644 --- a/utils/logger.py +++ b/utils/logger.py @@ -23,12 +23,22 @@ def filter(self, record: logging.LogRecord) -> bool: return True +class ExactExcludeFilter(logging.Filter): + def __init__(self, names: list[str] | None = None) -> None: + super().__init__() + self.names = {item for item in (names or []) if item} + + def filter(self, record: logging.LogRecord) -> bool: + return record.name not in self.names + + def setup_logging( level: int = logging.DEBUG, log_path: Path | None = None, *, state_store: StateStore | None = None, disabled_logger_prefixes: list[str] | None = None, + disabled_logger_names: list[str] | None = None, ) -> None: path = log_path or DEFAULT_LOG_PATH path.parent.mkdir(parents=True, exist_ok=True) @@ -39,16 +49,19 @@ def setup_logging( formatter = logging.Formatter(LOG_FORMAT) prefix_filter = PrefixExcludeFilter(disabled_logger_prefixes) + exact_filter = ExactExcludeFilter(disabled_logger_names) console_handler = logging.StreamHandler() console_handler.setLevel(level) console_handler.setFormatter(formatter) console_handler.addFilter(prefix_filter) + console_handler.addFilter(exact_filter) file_handler = logging.FileHandler(path, encoding="utf-8") file_handler.setLevel(level) file_handler.setFormatter(formatter) file_handler.addFilter(prefix_filter) + file_handler.addFilter(exact_filter) root.addHandler(console_handler) root.addHandler(file_handler) @@ -58,4 +71,5 @@ def setup_logging( state_handler.setLevel(level) state_handler.setFormatter(formatter) state_handler.addFilter(prefix_filter) + state_handler.addFilter(exact_filter) root.addHandler(state_handler)