diff --git a/Meshflow/meshcore_packets/services/path_resolution.py b/Meshflow/meshcore_packets/services/path_resolution.py index 0eff291..21f68f0 100644 --- a/Meshflow/meshcore_packets/services/path_resolution.py +++ b/Meshflow/meshcore_packets/services/path_resolution.py @@ -4,42 +4,79 @@ from typing import Any, Iterable +from django.db.models import Q + +from common.protocol import Protocol from meshcore_packet_path.models import MeshCorePathSegmentResolution, SegmentStatus +from nodes.models import ObservedNode from text_messages.map_helpers import observed_node_map_position HOP_STATUS_UNKNOWN = "unknown" HOP_STATUS_AMBIGUOUS = "ambiguous" HOP_STATUS_RESOLVED = "resolved" +SegmentIdentity = tuple[int | None, int | None, str] + def _normalize_segment(segment: str) -> str: return str(segment).strip().lower().replace("0x", "") +def segment_identity_key( + segment: str, + hash_mode: int | None = None, + hash_size: int | None = None, +) -> SegmentIdentity: + return (hash_mode, hash_size, _normalize_segment(segment)) + + +def _empty_hop(segment: str) -> dict[str, Any]: + normalized = _normalize_segment(segment) + return { + "hash": normalized, + "status": HOP_STATUS_UNKNOWN, + "node_id_str": None, + "internal_id": None, + "long_name": None, + "short_name": None, + "ambiguous": False, + "position": None, + "candidates": [], + } + + +def _serialize_candidate(node: ObservedNode) -> dict[str, Any]: + return { + "internal_id": str(node.internal_id), + "node_id_str": node.node_id_str, + "long_name": node.long_name, + "short_name": node.short_name, + "position": observed_node_map_position(node), + } + + +def _hop_from_node(segment: str, node: ObservedNode) -> dict[str, Any]: + normalized = _normalize_segment(segment) + return { + "hash": normalized, + "status": HOP_STATUS_RESOLVED, + "node_id_str": node.node_id_str, + "internal_id": str(node.internal_id), + "long_name": node.long_name, + "short_name": node.short_name, + "ambiguous": False, + "position": observed_node_map_position(node), + "candidates": [], + } + + def _hop_from_resolution(segment: str, resolution: MeshCorePathSegmentResolution | None) -> dict[str, Any]: normalized = _normalize_segment(segment) if resolution is None: - return { - "hash": normalized, - "status": HOP_STATUS_UNKNOWN, - "node_id_str": None, - "internal_id": None, - "long_name": None, - "ambiguous": False, - "position": None, - } + return _empty_hop(segment) if resolution.status == SegmentStatus.RESOLVED and resolution.observed_node_id: - node = resolution.observed_node - return { - "hash": normalized, - "status": HOP_STATUS_RESOLVED, - "node_id_str": node.node_id_str if node else None, - "internal_id": str(node.internal_id) if node else None, - "long_name": node.long_name if node else None, - "ambiguous": False, - "position": observed_node_map_position(node), - } + return _hop_from_node(segment, resolution.observed_node) if resolution.status == SegmentStatus.AMBIGUOUS: return { @@ -48,68 +85,166 @@ def _hop_from_resolution(segment: str, resolution: MeshCorePathSegmentResolution "node_id_str": None, "internal_id": None, "long_name": None, + "short_name": None, "ambiguous": True, "position": None, + "candidates": [], } + return _empty_hop(segment) + + +def _suffix_match_nodes(segment: str) -> list[ObservedNode]: + """Match ObservedNode rows whose mc_pubkey_prefix or mc_pubkey ends with segment hex.""" + normalized = _normalize_segment(segment) + if not normalized: + return [] + return list( + ObservedNode.objects.filter(protocol=Protocol.MESHCORE) + .filter(Q(mc_pubkey_prefix__iendswith=normalized) | Q(mc_pubkey__iendswith=normalized)) + .select_related("latest_status") + .distinct()[:20] + ) + + +def _apply_auto_matcher(hop: dict[str, Any]) -> dict[str, Any]: + if hop["status"] != HOP_STATUS_UNKNOWN: + return hop + nodes = _suffix_match_nodes(hop["hash"]) + if not nodes: + return hop + if len(nodes) == 1: + return _hop_from_node(hop["hash"], nodes[0]) return { - "hash": normalized, - "status": HOP_STATUS_UNKNOWN, + "hash": hop["hash"], + "status": HOP_STATUS_AMBIGUOUS, "node_id_str": None, "internal_id": None, "long_name": None, - "ambiguous": False, + "short_name": None, + "ambiguous": True, "position": None, + "candidates": [_serialize_candidate(node) for node in nodes], } -def format_path_hop(segment: str, *, resolution_cache: dict[str, dict[str, Any]] | None = None) -> dict[str, Any]: +def _coerce_segment_ref(item: str | dict[str, Any]) -> SegmentIdentity: + if isinstance(item, str): + return segment_identity_key(item) + return segment_identity_key( + item["segment"], + item.get("hash_mode"), + item.get("hash_size"), + ) + + +def _lookup_resolution_row( + segment: str, + hash_mode: int | None, + hash_size: int | None, + by_triple: dict[SegmentIdentity, MeshCorePathSegmentResolution], + by_hash: dict[str, list[MeshCorePathSegmentResolution]], +) -> MeshCorePathSegmentResolution | None: + key = segment_identity_key(segment, hash_mode, hash_size) + row = by_triple.get(key) + if row is not None: + return row normalized = _normalize_segment(segment) - if resolution_cache is not None and normalized in resolution_cache: - return resolution_cache[normalized] - return _hop_from_resolution(segment, None) + rows = by_hash.get(normalized) or [] + if len(rows) == 1: + return rows[0] + for candidate in rows: + if candidate.hash_mode == hash_mode and candidate.hash_size == hash_size: + return candidate + return None + + +def format_path_hop( + segment: str, + *, + hash_mode: int | None = None, + hash_size: int | None = None, + resolution_cache: dict[SegmentIdentity, dict[str, Any]] | None = None, +) -> dict[str, Any]: + key = segment_identity_key(segment, hash_mode, hash_size) + if resolution_cache is not None and key in resolution_cache: + return resolution_cache[key] + hop = _apply_auto_matcher(_empty_hop(segment)) + return hop def format_path_hops( segments: list[str] | None, *, - resolution_cache: dict[str, dict[str, Any]] | None = None, + hash_mode: int | None = None, + hash_size: int | None = None, + resolution_cache: dict[SegmentIdentity, dict[str, Any]] | None = None, ) -> list[dict[str, Any]]: if not segments: return [] - return [format_path_hop(segment, resolution_cache=resolution_cache) for segment in segments] - - -def bulk_format_path_hops(segments: Iterable[str]) -> dict[str, dict[str, Any]]: - """Dedupe segments and load MeshCorePathSegmentResolution rows when present.""" - normalized_list: list[str] = [] - for segment in segments: - normalized = _normalize_segment(segment) - if normalized and normalized not in normalized_list: - normalized_list.append(normalized) - - if not normalized_list: + return [ + format_path_hop(segment, hash_mode=hash_mode, hash_size=hash_size, resolution_cache=resolution_cache) + for segment in segments + ] + + +def bulk_format_path_hops( + segment_refs: Iterable[str | dict[str, Any]], +) -> dict[SegmentIdentity, dict[str, Any]]: + """Load segment resolutions and auto-matcher results for message heard[] bulk cache.""" + keys: list[SegmentIdentity] = [] + seen: set[SegmentIdentity] = set() + for item in segment_refs: + key = _coerce_segment_ref(item) + if key[2] and key not in seen: + seen.add(key) + keys.append(key) + + if not keys: return {} - resolutions = { - _normalize_segment(row.segment_hash): row - for row in MeshCorePathSegmentResolution.objects.filter( - segment_hash__in=normalized_list, + segment_hashes = {key[2] for key in keys} + rows = list( + MeshCorePathSegmentResolution.objects.filter( + segment_hash__in=segment_hashes, ).select_related("observed_node", "observed_node__latest_status") - } - - cache: dict[str, dict[str, Any]] = {} - for normalized in normalized_list: - cache[normalized] = _hop_from_resolution(normalized, resolutions.get(normalized)) + ) + by_triple: dict[SegmentIdentity, MeshCorePathSegmentResolution] = {} + by_hash: dict[str, list[MeshCorePathSegmentResolution]] = {} + for row in rows: + normalized = _normalize_segment(row.segment_hash) + triple = segment_identity_key(normalized, row.hash_mode, row.hash_size) + by_triple[triple] = row + by_hash.setdefault(normalized, []).append(row) + + cache: dict[SegmentIdentity, dict[str, Any]] = {} + for hash_mode, hash_size, normalized in keys: + key = (hash_mode, hash_size, normalized) + resolution = _lookup_resolution_row(normalized, hash_mode, hash_size, by_triple, by_hash) + hop = _hop_from_resolution(normalized, resolution) + if hop["status"] == HOP_STATUS_UNKNOWN: + hop = _apply_auto_matcher(hop) + elif hop["status"] == HOP_STATUS_AMBIGUOUS and not hop["candidates"]: + nodes = _suffix_match_nodes(normalized) + if nodes: + hop = {**hop, "candidates": [_serialize_candidate(node) for node in nodes]} + cache[key] = hop return cache def path_known_for_segments( segments: list[str] | None, *, - resolution_cache: dict[str, dict[str, Any]] | None = None, + hash_mode: int | None = None, + hash_size: int | None = None, + resolution_cache: dict[SegmentIdentity, dict[str, Any]] | None = None, ) -> bool: - hops = format_path_hops(segments, resolution_cache=resolution_cache) + hops = format_path_hops( + segments, + hash_mode=hash_mode, + hash_size=hash_size, + resolution_cache=resolution_cache, + ) if not hops: return False return all(hop.get("status") == HOP_STATUS_RESOLVED and hop.get("position") is not None for hop in hops) diff --git a/Meshflow/meshcore_packets/tests/test_path_resolution.py b/Meshflow/meshcore_packets/tests/test_path_resolution.py index 939c760..7c91f2c 100644 --- a/Meshflow/meshcore_packets/tests/test_path_resolution.py +++ b/Meshflow/meshcore_packets/tests/test_path_resolution.py @@ -8,25 +8,23 @@ bulk_format_path_hops, format_path_hops, path_known_for_segments, + segment_identity_key, ) -from nodes.models import ObservedNode +from nodes.models import NodeLatestStatus, ObservedNode +@pytest.mark.django_db def test_format_path_hops_unknown_status(): hops = format_path_hops(["f3bc", "f1"]) assert len(hops) == 2 - assert hops[0] == { - "hash": "f3bc", - "status": "unknown", - "node_id_str": None, - "internal_id": None, - "long_name": None, - "ambiguous": False, - "position": None, - } + assert hops[0]["hash"] == "f3bc" + assert hops[0]["status"] == "unknown" + assert hops[0]["short_name"] is None + assert hops[0]["candidates"] == [] assert hops[1]["hash"] == "f1" +@pytest.mark.django_db def test_format_path_hops_normalizes_hex(): hops = format_path_hops(["0xF3BC"]) assert hops[0]["hash"] == "f3bc" @@ -35,8 +33,9 @@ def test_format_path_hops_normalizes_hex(): @pytest.mark.django_db def test_bulk_format_path_hops_dedupes(): cache = bulk_format_path_hops(["aa", "aa", "bb"]) - assert set(cache.keys()) == {"aa", "bb"} - assert cache["aa"]["status"] == "unknown" + assert segment_identity_key("aa") in cache + assert segment_identity_key("bb") in cache + assert cache[segment_identity_key("aa")]["status"] == "unknown" @pytest.mark.django_db @@ -53,6 +52,7 @@ def test_bulk_format_path_hops_uses_segment_resolution_table(): mc_pubkey="a" * 64, mc_pubkey_prefix="a" * 12, long_name="Resolved Hop", + short_name="RH", ) MeshCorePathSegmentResolution.objects.create( segment_hash="f3bc", @@ -60,8 +60,111 @@ def test_bulk_format_path_hops_uses_segment_resolution_table(): status=SegmentStatus.RESOLVED, observed_node=node, ) - cache = bulk_format_path_hops(["f3bc"]) - hop = cache["f3bc"] + cache = bulk_format_path_hops([{"segment": "f3bc", "hash_mode": None, "hash_size": 2}]) + hop = cache[segment_identity_key("f3bc", hash_size=2)] assert hop["status"] == "resolved" assert hop["node_id_str"] == node.node_id_str assert hop["long_name"] == "Resolved Hop" + assert hop["short_name"] == "RH" + + +@pytest.mark.django_db +def test_bulk_format_path_hops_respects_hash_mode_size_identity(): + node_a = ObservedNode.objects.create( + protocol=Protocol.MESHCORE, + mc_pubkey="a" * 64, + mc_pubkey_prefix="a" * 12, + long_name="Size 2", + ) + node_b = ObservedNode.objects.create( + protocol=Protocol.MESHCORE, + mc_pubkey="b" * 64, + mc_pubkey_prefix="b" * 12, + long_name="Size 3", + ) + MeshCorePathSegmentResolution.objects.create( + segment_hash="ab", + hash_size=2, + status=SegmentStatus.RESOLVED, + observed_node=node_a, + ) + MeshCorePathSegmentResolution.objects.create( + segment_hash="ab", + hash_size=3, + status=SegmentStatus.RESOLVED, + observed_node=node_b, + ) + cache = bulk_format_path_hops( + [ + {"segment": "ab", "hash_mode": None, "hash_size": 2}, + {"segment": "ab", "hash_mode": None, "hash_size": 3}, + ] + ) + assert cache[segment_identity_key("ab", hash_size=2)]["long_name"] == "Size 2" + assert cache[segment_identity_key("ab", hash_size=3)]["long_name"] == "Size 3" + + +@pytest.mark.django_db +def test_auto_matcher_unique_suffix_resolves(): + prefix = "00000000beef" + node = ObservedNode.objects.create( + protocol=Protocol.MESHCORE, + mc_pubkey=prefix + ("c" * 52), + mc_pubkey_prefix=prefix, + long_name="Unique Suffix", + short_name="US", + ) + NodeLatestStatus.objects.create(node=node, latitude=55.0, longitude=-4.0) + cache = bulk_format_path_hops([{"segment": "beef", "hash_mode": None, "hash_size": 2}]) + hop = cache[segment_identity_key("beef", hash_size=2)] + assert hop["status"] == "resolved" + assert hop["node_id_str"] == node.node_id_str + assert hop["short_name"] == "US" + + +@pytest.mark.django_db +def test_auto_matcher_multiple_suffix_matches_ambiguous(): + ObservedNode.objects.create( + protocol=Protocol.MESHCORE, + mc_pubkey="a" * 64, + mc_pubkey_prefix="aaaaaaaacafe", + long_name="Node A", + ) + ObservedNode.objects.create( + protocol=Protocol.MESHCORE, + mc_pubkey="b" * 64, + mc_pubkey_prefix="bbbbbbbbcafe", + long_name="Node B", + ) + cache = bulk_format_path_hops([{"segment": "cafe", "hash_mode": None, "hash_size": 2}]) + hop = cache[segment_identity_key("cafe", hash_size=2)] + assert hop["status"] == "ambiguous" + assert hop["ambiguous"] is True + assert len(hop["candidates"]) == 2 + + +@pytest.mark.django_db +def test_manual_resolution_overrides_auto_matcher(): + prefix = "feedface0000" + auto_node = ObservedNode.objects.create( + protocol=Protocol.MESHCORE, + mc_pubkey=prefix + ("d" * 52), + mc_pubkey_prefix=prefix, + long_name="Auto", + ) + staff_node = ObservedNode.objects.create( + protocol=Protocol.MESHCORE, + mc_pubkey="e" * 64, + mc_pubkey_prefix="e" * 12, + long_name="Staff", + ) + MeshCorePathSegmentResolution.objects.create( + segment_hash="face", + hash_size=2, + status=SegmentStatus.RESOLVED, + observed_node=staff_node, + ) + cache = bulk_format_path_hops([{"segment": "face", "hash_mode": None, "hash_size": 2}]) + hop = cache[segment_identity_key("face", hash_size=2)] + assert hop["long_name"] == "Staff" + assert hop["node_id_str"] != auto_node.node_id_str diff --git a/Meshflow/text_messages/serializers.py b/Meshflow/text_messages/serializers.py index 71ca5c1..b826fe7 100644 --- a/Meshflow/text_messages/serializers.py +++ b/Meshflow/text_messages/serializers.py @@ -3,7 +3,11 @@ from common.protocol import Protocol from constellations.models import MessageChannel from meshcore_packets.models import MeshCorePacketObservation -from meshcore_packets.services.path_resolution import format_path_hops, path_known_for_segments +from meshcore_packets.services.path_resolution import ( + format_path_hops, + path_known_for_segments, + segment_identity_key, +) from nodes.models import ManagedNode, ObservedNode from packets.serializers import PrefetchedPacketObservationSerializer @@ -16,15 +20,15 @@ def _normalize_path_segment(segment) -> str: return str(segment).strip().lower().replace("0x", "") -def _resolved_path_from_cache(segments, cache): +def _resolved_path_from_cache(segments, cache, *, hash_mode=None, hash_size=None): if not segments: return [] hops = [] for segment in segments: - key = _normalize_path_segment(segment) + key = segment_identity_key(segment, hash_mode, hash_size) hop = cache.get(key) if cache else None if hop is None: - hop = format_path_hops([segment])[0] + hop = format_path_hops([segment], hash_mode=hash_mode, hash_size=hash_size)[0] hops.append(hop) return hops @@ -139,9 +143,21 @@ def get_heard(self, obj): "rx_time": obs.rx_time, "rx_rssi": obs.rx_rssi, "rx_snr": obs.rx_snr, + "path_hash_mode": obs.path_hash_mode, + "path_hash_size": obs.path_hash_size, "path_hashes": segments, - "resolved_path": _resolved_path_from_cache(segments, path_hop_cache), - "path_known": path_known_for_segments(segments, resolution_cache=path_hop_cache), + "resolved_path": _resolved_path_from_cache( + segments, + path_hop_cache, + hash_mode=obs.path_hash_mode, + hash_size=obs.path_hash_size, + ), + "path_known": path_known_for_segments( + segments, + hash_mode=obs.path_hash_mode, + hash_size=obs.path_hash_size, + resolution_cache=path_hop_cache, + ), } ) return heard diff --git a/Meshflow/text_messages/tests/test_heard_api.py b/Meshflow/text_messages/tests/test_heard_api.py index 083e6d0..41e870f 100644 --- a/Meshflow/text_messages/tests/test_heard_api.py +++ b/Meshflow/text_messages/tests/test_heard_api.py @@ -196,3 +196,45 @@ def test_mc_message_heard_resolved_path_from_segment_table(meshcore_feeder, inge assert hop["node_id_str"] == node.node_id_str assert hop["position"]["latitude"] == 55.95 assert row["heard"][0]["path_known"] is True + + +@pytest.mark.django_db +def test_mc_message_heard_ambiguous_hop_candidates(meshcore_feeder, ingest_client): + reconcile_mc_channels( + meshcore_feeder["node"], + [{"mc_channel_idx": 0, "name": "Public", "mc_channel_type": "PUBLIC"}], + ) + for name, letter in (("Node A", "a"), ("Node B", "b")): + node = ObservedNode.objects.create( + protocol=Protocol.MESHCORE, + mc_pubkey=letter * 64, + mc_pubkey_prefix=f"{letter * 8}cafe", + long_name=name, + ) + NodeLatestStatus.objects.create(node=node, latitude=55.0, longitude=-4.0) + now = timezone.now() + url = feeder_url("meshcore-feeder-packet-ingest", FEEDER_MC_PUBKEY_PREFIX) + ingest_client.post( + url, + { + "event_type": "channel_message", + "payload_type": "channel_text", + "channel_idx": 0, + "pkt_hash": 88012, + "rx_time": now.timestamp(), + "text": "ambiguous hop test", + "path_hashes": ["cafe"], + "path_hash_size": 2, + "raw": {}, + }, + format="json", + ) + tm = TextMessage.objects.get(message_text="ambiguous hop test") + client = APIClient() + response = client.get(reverse("textmessage-list"), {"channel_id": tm.channel_id}) + row = next(item for item in response.data["results"] if item["id"] == str(tm.id)) + heard = row["heard"][0] + assert heard["path_hash_size"] == 2 + hop = heard["resolved_path"][0] + assert hop["status"] == "ambiguous" + assert len(hop["candidates"]) == 2 diff --git a/Meshflow/text_messages/views.py b/Meshflow/text_messages/views.py index 4c1e17d..856e98b 100644 --- a/Meshflow/text_messages/views.py +++ b/Meshflow/text_messages/views.py @@ -88,8 +88,8 @@ def _mc_sender_labels_for_messages(self, messages): labels.add(label) return labels - def _path_segments_for_messages(self, messages): - segments = [] + def _path_segment_refs_for_messages(self, messages): + refs = [] for msg in messages: if msg.protocol != Protocol.MESHCORE or not msg.original_mc_packet_id: continue @@ -97,15 +97,22 @@ def _path_segments_for_messages(self, messages): observations = getattr(packet, "prefetched_mc_observations", None) or [] for obs in observations: if obs.path_hashes: - segments.extend(_normalize_path_segment(segment) for segment in obs.path_hashes) - return segments + for segment in obs.path_hashes: + refs.append( + { + "segment": _normalize_path_segment(segment), + "hash_mode": obs.path_hash_mode, + "hash_size": obs.path_hash_size, + } + ) + return refs def list(self, request, *args, **kwargs): queryset = self.filter_queryset(self.get_queryset()) page = self.paginate_queryset(queryset) messages = page if page is not None else list(queryset) context = self.get_serializer_context() - context["path_hop_cache"] = bulk_format_path_hops(self._path_segments_for_messages(messages)) + context["path_hop_cache"] = bulk_format_path_hops(self._path_segment_refs_for_messages(messages)) context["mc_sender_candidates_by_label"] = bulk_mc_sender_candidates_by_label( self._mc_sender_labels_for_messages(messages) ) diff --git a/docs/features/meshcore/packet-path-tracing/bug-no-path-info.md b/docs/features/meshcore/packet-path-tracing/bug-no-path-info.md index 741ffa8..8c2cf04 100644 --- a/docs/features/meshcore/packet-path-tracing/bug-no-path-info.md +++ b/docs/features/meshcore/packet-path-tracing/bug-no-path-info.md @@ -1,8 +1,8 @@ # Bug: MeshCore channel text often has no path in heard / DB -**Status:** Investigating (pre-prod verified 2026-06-03) +**Status:** Tier-1 mitigated on `main` ([#390](https://github.com/pskillen/meshflow-api/pull/390)); sparse capture still dominates empty paths. Heard resolution UI/API in open PRs ([#395](https://github.com/pskillen/meshflow-api/pull/395), [ui#322](https://github.com/pskillen/meshflow-ui/pull/322)). **Tracking:** [meshflow-api#385](https://github.com/pskillen/meshflow-api/issues/385) (Tier 1 twin), epic [#267](https://github.com/pskillen/meshflow-api/issues/267) -**Related:** [packet-path-tracing-outstanding.md](./packet-path-tracing-outstanding.md), [tier-1-message-path-twin.md](./tier-1-message-path-twin.md) +**Related:** [packet-path-tracing-outstanding.md](./packet-path-tracing-outstanding.md), [tier-1-message-path-twin.md](./tier-1-message-path-twin.md), [tier-2-heard-resolution.md](./tier-2-heard-resolution.md) ## Symptom @@ -16,10 +16,11 @@ Operators expect path hashes to ride with (or immediately follow) the same on-ai | --- | --- | | [packet-path-tracing-outstanding.md](./packet-path-tracing-outstanding.md) § Message path data chain | `channel_message` decode has `path_len` / mode but usually **no `path` hex**; path on wire is often on **`rx_log_data` PATH/TEXT_MSG**; bot historically uploaded **ADVERT only**. | | [tier-1-message-path-twin.md](./tier-1-message-path-twin.md) | **Shipped approach:** thin bot uploads TEXT_MSG/PATH `rx_log_data`; API **twin-merge** copies `path_hashes` onto `channel_text` within **120s** (`MESHCORE_DECODED_TWIN_WINDOW_SECONDS`). | -| PR [#390](https://github.com/pskillen/meshflow-api/pull/390) (`api-388`) | Implemented `path_twin.py`, `path_hashes` ingest, heard tests — merged to `main`. | -| Agent [meta workspace consolidation](c074c495-8f39-4d1d-b755-e81567cd29ad) | Confirmed tier-1 docs on `main`; branch `api-388/pskillen/mc-path-twin-and-resolution`. | +| PR [#390](https://github.com/pskillen/meshflow-api/pull/390) (`api-388`) | Implemented `path_twin.py`, `path_hashes` ingest, heard tests — **merged to `main`**. | +| PR [#395](https://github.com/pskillen/meshflow-api/pull/395) | Heard `resolved_path`: composite segment lookup, `path_hash_mode` / `path_hash_size` on `heard[]`, guarded pubkey-suffix auto-matcher + `candidates[]`; ADR addendum in [traceroute ADR-0001 § addendum](../../traceroute/adr/0001-mc-path-hash-resolution.md). | +| PR [ui#322](https://github.com/pskillen/meshflow-ui/pull/322) | `HeardPathGeoMap` path polylines, hop MapPin icons, ambiguity tooltips; shared layers with `HeardPathMap` ([#311](https://github.com/pskillen/meshflow-ui/issues/311)). Deploy after API #395. | -**Conclusion from this session:** Tier-1 **code path works when a PATH/TEXT_MSG twin exists in-window**; pre-prod failure rate is dominated by **missing or mis-timed rx_log twins**, not broken heard assembly. +**Conclusion:** Tier-1 **code path works when a PATH/TEXT_MSG twin exists in-window**; pre-prod failure rate is still dominated by **missing or mis-timed rx_log twins**, not broken heard assembly. Tier-2 improves **display** when `path_hashes` exist; it does not increase twin capture rate. ## Pre-prod sample messages (2026-06-03) @@ -31,8 +32,22 @@ Compared four `TextMessage` rows (`protocol = MeshCore`) on the single feeder: | `Rogue Two: Beans 🤦‍♂️` | 07:59:59 | **Yes** (same path) | **PATH** @ 07:58:48 (−71s) | | `Rogue Two: 😂` | 07:50:48 | **No** | **None** | | `MCEK 4: @[Rogue Two]: Lol` | 07:50:58 | **No** | **None** | +| `☘️GI7ULG☘️: Test` (2026-06-04 re-check) | (see DB) | **Yes** `['6edc9b','70929f','f3bcf1']` on PDYa obs | Twin merge succeeded for this row | -**7-day aggregate (pre-prod):** 22 / 533 MeshCore `TextMessage` rows with `path_hashes` length ≥ 2 → **4.1%** success rate (hourly buckets 0–33%). +**7-day aggregate (pre-prod, 2026-06-03):** 22 / 533 MeshCore `TextMessage` rows with `path_hashes` length ≥ 2 → **4.1%** success rate (hourly buckets 0–33%). Re-run after tier-1 deploy on feeders to refresh. + +### GI7ULG follow-up (2026-06-04) — path present, resolution / sender UX + +Message id `d2083011-6445-4a51-924b-a47b4685ac15` (`☘️GI7ULG☘️: Test`): + +| Check | Pre-prod result | +| --- | --- | +| `path_hashes` on text observation | **Yes** (3 segments) | +| Segment resolution (suffix auto-matcher + M1 table) | All **unknown** — no `ObservedNode` suffix match for `6edc9b`, `70929f`, `f3bcf1` | +| `mc_sender_candidates` | **1** (`mc:9cce73b9b3ee`, long name `☘️GI7ULG☘️`) | +| `NodeLatestStatus` for sender | **None** — no position in DB | + +**Not the same bug as empty `path_hashes`:** channel list and heard dialog previously disagreed when one candidate had no position — **fixed in ui#322** (`resolveHeardPathSender`: `identified` without requiring `position`; geo map unchanged). ### Same 20-minute window (07:45–08:05 UTC) @@ -93,5 +108,7 @@ Failures **are** because: | Date | Action | Result | | --- | --- | --- | -| 2026-06-03 | Pre-prod DB via Django + `Meshflow/ai-env` | Sample table + 4.1% / 7d rate; fail cases = no in-window PATH twin | +| 2026-06-03 | Pre-prod DB via Django + `meshflow-api/Meshflow/ai-env` | Sample table + 4.1% / 7d rate; fail cases = no in-window PATH twin | | 2026-06-03 | This doc created | Baseline for continued debugging | +| 2026-06-04 | Tier-1 merged ([#390](https://github.com/pskillen/meshflow-api/pull/390)); tier-2 PRs opened | API #395 + UI #322; GI7ULG sample has path but unresolved hops + sender/position mismatch | +| 2026-06-04 | Pre-prod Django shell (`GI7ULG` messages) | Confirmed twin path on `Test`; segments unknown; single sender candidate without `latest_status` | diff --git a/docs/features/meshcore/packet-path-tracing/packet-path-tracing-outstanding.md b/docs/features/meshcore/packet-path-tracing/packet-path-tracing-outstanding.md index 7589820..4cb1555 100644 --- a/docs/features/meshcore/packet-path-tracing/packet-path-tracing-outstanding.md +++ b/docs/features/meshcore/packet-path-tracing/packet-path-tracing-outstanding.md @@ -17,25 +17,30 @@ Items **skipped**, **incomplete**, or **discovered during planning** — not the ## Message heard map (UI — logical layout, not M7) -- [ ] **[meshflow-ui#311](https://github.com/pskillen/meshflow-ui/issues/311)** — HeardPathMap logical path per feeder: dashed schematic hop chain (one node per hash segment), **not** placed at map coordinates; keep sender/feeder at geo positions when known. Feeder list below graph shows **each observer’s distinct path** beside its row. Uses existing `heard[]` `path_hashes` / `resolved_path` from #360; no new API. +- [x] **[meshflow-ui#311](https://github.com/pskillen/meshflow-ui/issues/311)** — HeardPathMap logical path per feeder ([ui#322](https://github.com/pskillen/meshflow-ui/pull/322)): dashed schematic hop chain, per-feeder paths in heard dialog. -## Geographic path on maps (future milestone — plan explicitly) +## Geographic path on maps -The logical heard-map slice above is **not** a substitute for placing hops at real coordinates. A later plan/milestone must cover: - -- [ ] **Geographic hop placement** — when M2/M3 (or manual segment annotation) yields `ObservedNode` positions for path segments, message heard map and/or M7 topology UI should render hops at **lat/lng** (and set `path_known` only when all hops are resolved per ADR). -- [ ] **Wire message `heard[]` to segment resolution** — optional read path from `MeshCorePathSegmentResolution` (or resolver output) so the heard dialog benefits from staff annotations / proven matcher without duplicating rollup tables in the client. +- [x] **Geographic hop placement (message heard, partial)** — [ui#322](https://github.com/pskillen/meshflow-ui/pull/322): `HeardPathGeoMap` draws polylines when hop `position` values exist; partial segments when only some hops resolve. `path_known` still requires all hops resolved **with** positions (ADR). +- [x] **Wire message `heard[]` to segment resolution** — [api#395](https://github.com/pskillen/meshflow-api/pull/395): `bulk_format_path_hops` reads `MeshCorePathSegmentResolution` + guarded suffix auto-matcher; `path_hash_mode` / `path_hash_size` on `heard[]`. - [ ] **M7 realtime/history maps** ([meshflow-ui#309](https://github.com/pskillen/meshflow-ui/issues/309)) — edge-based geographic and logical topology; depends on API M5/M6. +- [ ] **Sync [tier-2-heard-resolution.md](./tier-2-heard-resolution.md)** with shipped auto-matcher and `candidates[]` (doc still says “no heuristics in v1”). -Until then, operators should assume heard-map paths are **list-order hash evidence**, not RF geography. +Until all hops resolve with positions, operators should treat geo lines as **best-effort**; hash chains remain list-order evidence when resolution is incomplete. --- ## Carried from prior passive slice -- [ ] **Proven hash → `ObservedNode` matcher** — still unproven; no production matcher until [traceroute ADR-0001 §A](../../traceroute/adr/0001-mc-path-hash-resolution.md) documents a safe rule. Gates M3. Tests must reject suffix/prefix/recency heuristics. +- [ ] **Proven hash → `ObservedNode` matcher (M3 / rollups)** — spike still **unproven** for authoritative identity ([traceroute ADR-0001 §A](../../traceroute/adr/0001-mc-path-hash-resolution.md)). **Display-only** guarded suffix matcher for message `heard[]` shipped in [api#395](https://github.com/pskillen/meshflow-api/pull/395) (addendum in same ADR); ambiguity → `candidates[]`, no map placement for ambiguous hops. - [ ] **`resolved_path` on `GET /api/meshcore/packets/`** — deferred from #360 (message API only). Optional; revisit alongside the edges API. -- [ ] **Upload `rx_log_data` PATH-only frames** — bot still skips non-ADVERT `rx_log_data`; needed for relays with `path_len > 0` and no business message (M1 capture / bot follow-up of [#119](https://github.com/pskillen/meshflow-bot/issues/119)). +- [ ] **Upload `rx_log_data` PATH-only frames** — bot still skips non-ADVERT `rx_log_data` except TEXT_MSG/PATH typenames added for tier-1 twin; PATH-only relays without a correlatable text row may still be missing (M1 capture / bot follow-up of [#119](https://github.com/pskillen/meshflow-bot/issues/119)). + +--- + +## Heard dialog UX (discovered 2026-06-04) + +- [x] **Sender “unknown” vs channel list** — [ui#322](https://github.com/pskillen/meshflow-ui/pull/322): `resolveHeardPathSender` sets `identified` from a single `mc_sender_candidate` without requiring position; `HopPositionIcon` and geo map use position only. --- @@ -49,9 +54,11 @@ Until then, operators should assume heard-map paths are **list-order hash eviden ## Message path data chain (confirmed — pre-prod Jun 2026) -**Symptom:** Message “Heard” UI and `GET` message `heard[]` show observers but **no hop chain** for MeshCore channel/contact text on pre-prod, even though M1 edge rollups exist and the heard-map UI (#311) is wired to `path_hashes`. +**Symptom (baseline):** Message “Heard” UI and `GET` message `heard[]` show observers but **no hop chain** for most MeshCore channel/contact text on pre-prod, even though M1 edge rollups exist. + +**Update (post tier-1 [#390](https://github.com/pskillen/meshflow-api/pull/390)):** Some rows now have `path_hashes` on the text observation when a PATH/TEXT_MSG twin merged in-window (e.g. `☘️GI7ULG☘️: Test` — 3 segments). Overall rate remains low (~4% on 2026-06-03 sample); see [bug-no-path-info.md](./bug-no-path-info.md). -**Not the cause:** Single feeder (one observation per `packet_id` is expected). API heard assembly and UI #311 behave correctly when `MeshCorePacketObservation.path_hashes` is set on the **same** packet as `TextMessage.original_mc_packet` (see `text_messages/tests/test_heard_api.py`). +**Not the cause:** Single feeder (one observation per `packet_id` is expected). API heard assembly and UI behave correctly when `MeshCorePacketObservation.path_hashes` is set on the **same** packet as `TextMessage.original_mc_packet` (see `text_messages/tests/test_heard_api.py`). ### Pre-prod evidence (read-only DB, one feeder) @@ -105,9 +112,10 @@ Implications for closing this gap (direction only — not scheduled here): ### Follow-up (tracking) -- [ ] **Tier 1 — server-led ingest (ship)** — [#385](https://github.com/pskillen/meshflow-api/issues/385): `path_hashes` on observation tied to `original_mc_packet` for channel `TextMessage` traffic; thin bot upload of TEXT_MSG/PATH `rx_log_data`; API twin-merge. Design: [tier-1-message-path-twin.md](./tier-1-message-path-twin.md). -- [ ] **Confirm with M2 spike** — whether `path_hash_mode` changes segment identity when we do get text paths. -- [ ] **Optional:** re-run pre-prod queries after deploy (`Meshflow/ai-env` + Django shell; local skill `MeshFlow/.cursor/skills/preprod-database/`) — breakdown by `payload_type` + `event_type`. +- [x] **Tier 1 — server-led ingest** — [#385](https://github.com/pskillen/meshflow-api/issues/385) / [api#390](https://github.com/pskillen/meshflow-api/pull/390) merged: twin-merge, bot TEXT_MSG/PATH upload. Design: [tier-1-message-path-twin.md](./tier-1-message-path-twin.md). +- [x] **Tier 2 — heard resolution (display)** — [api#395](https://github.com/pskillen/meshflow-api/pull/395), [ui#322](https://github.com/pskillen/meshflow-ui/pull/322) open; design [tier-2-heard-resolution.md](./tier-2-heard-resolution.md). +- [ ] **Confirm with M2 spike** — whether `path_hash_mode` changes segment identity when we do get text paths (composite key already used in #395 for heard). +- [ ] **Re-run pre-prod metrics** after tier-1 on all feeders — `meshflow-api/Meshflow/ai-env` + Django shell or `MeshFlow/.cursor/skills/preprod-database/scripts/query-preprod.sh` (auto-detects api `ai-env`). --- diff --git a/docs/features/meshcore/packet-path-tracing/packet-path-tracing-progress.md b/docs/features/meshcore/packet-path-tracing/packet-path-tracing-progress.md index 8598395..8094788 100644 --- a/docs/features/meshcore/packet-path-tracing/packet-path-tracing-progress.md +++ b/docs/features/meshcore/packet-path-tracing/packet-path-tracing-progress.md @@ -8,11 +8,16 @@ ## Overall status -**Status:** M1 PRs open (pending merge / deploy) +**Status:** M1 PRs open (pending merge / deploy). **Tier-1 message path twin** merged ([#390](https://github.com/pskillen/meshflow-api/pull/390)). **Tier-2 heard resolution** implemented in open PRs ([#395](https://github.com/pskillen/meshflow-api/pull/395), [ui#322](https://github.com/pskillen/meshflow-ui/pull/322)) — deploy API first, then UI. -**Branch:** `api-372/pskillen/meshcore-passive-path-m1` (meshflow-api, meshflow-bot, meshflow-ui) +**Branches (in flight):** -**PRs:** [meshflow-api#378](https://github.com/pskillen/meshflow-api/pull/378) · [meshflow-bot#122](https://github.com/pskillen/meshflow-bot/pull/122) · [meshflow-ui#310](https://github.com/pskillen/meshflow-ui/pull/310) +| Slice | meshflow-api | meshflow-bot | meshflow-ui | +| --- | --- | --- | --- | +| M1 passive path | `api-372/pskillen/meshcore-passive-path-m1` | (see bot PR) | (see ui PR) | +| Tier-2 heard map | `api-311/pskillen/mc-heard-path-resolution` | — | `ui-311/pskillen/mc-heard-path-map` | + +**PRs:** [api#378](https://github.com/pskillen/meshflow-api/pull/378) · [bot#122](https://github.com/pskillen/meshflow-bot/pull/122) · [ui#310](https://github.com/pskillen/meshflow-ui/pull/310) (M1) · [api#390](https://github.com/pskillen/meshflow-api/pull/390) (tier-1, **merged**) · [api#395](https://github.com/pskillen/meshflow-api/pull/395) · [ui#322](https://github.com/pskillen/meshflow-ui/pull/322) (tier-2) The display-only passive slice that precedes this subsystem has **shipped** (see Precursor below). M1 implementation is complete in branch; awaiting merge and deploy. @@ -86,7 +91,47 @@ Tracked as sub-issues of [#267](https://github.com/pskillen/meshflow-api/issues/ --- +## Tier-1 — message path twin (merged) + +**PR:** [api#390](https://github.com/pskillen/meshflow-api/pull/390) · Design: [tier-1-message-path-twin.md](./tier-1-message-path-twin.md) + +- Thin bot uploads TEXT_MSG/PATH `rx_log_data`; API `path_twin.py` copies `path_hashes` onto `channel_text` observations within `MESHCORE_DECODED_TWIN_WINDOW_SECONDS` (120s). +- Pre-prod: works when in-window twin exists; most channel texts still have empty path — see [bug-no-path-info.md](./bug-no-path-info.md). + +--- + +## Tier-2 — heard map resolution (PRs open, Jun 2026) + +**PRs:** [api#395](https://github.com/pskillen/meshflow-api/pull/395) · [ui#322](https://github.com/pskillen/meshflow-ui/pull/322) · UI issue [#311](https://github.com/pskillen/meshflow-ui/issues/311) +**Design:** [tier-2-heard-resolution.md](./tier-2-heard-resolution.md) (behaviour); matcher addendum in [traceroute ADR-0001](../../traceroute/adr/0001-mc-path-hash-resolution.md). + +**API (#395)** + +- `heard[]` includes `path_hash_mode`, `path_hash_size`; `resolved_path` from `MeshCorePathSegmentResolution` + guarded pubkey-suffix auto-matcher (`candidates[]` on ambiguity). +- `bulk_format_path_hops` cache keyed by `(hash_mode, hash_size, segment_hash)`. +- Tests: `meshcore_packets/tests/test_path_resolution.py`, `text_messages/tests/test_heard_api.py`. + +**UI (#322)** + +- `HeardPathMap`: logical dashed hop chain per feeder ([#311](https://github.com/pskillen/meshflow-ui/issues/311)). +- `HeardPathGeoMap`: geographic polylines when hop positions exist; partial segments when only some hops resolve. +- `HopPositionIcon` / `PathHopBadge`: known vs unknown position; ambiguity tooltips. +- Component docs: `meshflow-ui` `HeardPathMap.md`, `HeardPathGeoMap.md`, `docs/meshcore/heard-path-dialog.md`. + +**Pre-prod spot-check (2026-06-04):** `☘️GI7ULG☘️: Test` has `path_hashes` but segments stay `unknown`; sender shows as unknown in heard dialog while channel list links one candidate — no `NodeLatestStatus` on sender node. See [bug-no-path-info.md](./bug-no-path-info.md). + +**Deploy / verify** + +- Merge and deploy API #395 before UI #322. +- Open message heard dialog on a row with merged `path_hashes`; confirm hash chain, optional geo lines when positions exist. +- `pytest Meshflow/meshcore_packets/tests/test_path_resolution.py Meshflow/text_messages/tests/test_heard_api.py -v` + +--- + ## Next -- Merge PRs ([#378](https://github.com/pskillen/meshflow-api/pull/378), [#122](https://github.com/pskillen/meshflow-bot/pull/122), [#310](https://github.com/pskillen/meshflow-ui/pull/310)) and deploy. +- Merge M1 PRs ([#378](https://github.com/pskillen/meshflow-api/pull/378), [#122](https://github.com/pskillen/meshflow-bot/pull/122), [#310](https://github.com/pskillen/meshflow-ui/pull/310)) and deploy. +- Merge tier-2 PRs ([#395](https://github.com/pskillen/meshflow-api/pull/395), [ui#322](https://github.com/pskillen/meshflow-ui/pull/322)); deploy API before UI. +- Re-run pre-prod path-on-text metrics after tier-1 is live on all feeders ([bug-no-path-info.md](./bug-no-path-info.md) checklist). - Begin M2 resolution spike ([#373](https://github.com/pskillen/meshflow-api/issues/373)) using the diagnostic UI. +- Align heard-dialog sender with channel list when one `mc_sender_candidate` exists without position (outstanding). diff --git a/docs/features/traceroute/adr/0001-mc-path-hash-resolution.md b/docs/features/traceroute/adr/0001-mc-path-hash-resolution.md index c82929f..ce94044 100644 --- a/docs/features/traceroute/adr/0001-mc-path-hash-resolution.md +++ b/docs/features/traceroute/adr/0001-mc-path-hash-resolution.md @@ -52,3 +52,15 @@ We can reliably **parse and display** path segments per feeder observation. We * - Message heard map shows sender, feeder position(s), and **dashed** paths; hop labels show raw hash hex. - UI must not link unknown hops to node detail pages. - When a proven matcher lands, add tests that fail if heuristic matching is reintroduced without ADR update. + +## Addendum (2026-06) — message heard auto-matcher + +`bulk_format_path_hops` still prefers **`MeshCorePathSegmentResolution`** rows keyed by `(hash_mode, hash_size, segment_hash)`. + +When no staff row applies, a **guarded suffix matcher** runs: + +- Segment hex must match the **suffix** of `mc_pubkey_prefix` or `mc_pubkey` on `ObservedNode` (case-insensitive). +- **One** match → `status=resolved` with node fields and position when available. +- **Multiple** matches → `status=ambiguous` with `candidates[]` (same shape as `McSenderCandidate`); UI must not place ambiguous hops on the map. + +This is display-only for message `heard[]`; it does not prove on-air identity. Tests in `meshcore_packets/tests/test_path_resolution.py` lock ambiguity behaviour. diff --git a/openapi.yaml b/openapi.yaml index 413c2b4..5f003fc 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -2775,12 +2775,20 @@ components: long_name: type: string nullable: true + short_name: + type: string + nullable: true ambiguous: type: boolean position: $ref: '#/components/schemas/MapPosition' nullable: true description: Present when status is resolved and the linked ObservedNode has a latest position + candidates: + type: array + items: + $ref: '#/components/schemas/McSenderCandidate' + description: Present when status is ambiguous; potential ObservedNode suffix matches for this segment HeardObserver: type: object @@ -2838,6 +2846,14 @@ components: type: number format: float nullable: true + path_hash_mode: + type: integer + nullable: true + description: MeshCore path hash mode from the feeder observation + path_hash_size: + type: integer + nullable: true + description: MeshCore path hash size (bytes per segment) from the feeder observation path_hashes: type: array items: