diff --git a/Meshflow/common/mc_channel_labels.py b/Meshflow/common/mc_channel_labels.py index 456e5bc..961dcc8 100644 --- a/Meshflow/common/mc_channel_labels.py +++ b/Meshflow/common/mc_channel_labels.py @@ -6,15 +6,22 @@ from nodes.models import ManagedNode, ManagedNodeMcChannelLink +def _scope_suffix(channel: MessageChannel) -> str: + scope = (channel.region_scope or "").strip() + if scope: + return f" · {scope}" + return "" + + def mc_channel_admin_label(channel: MessageChannel) -> str: """Human label for admin lists: #hashtag for HASHTAG, plain name for PUBLIC.""" if channel.mc_channel_type == MeshCoreChannelType.HASHTAG: - tag = (channel.mc_hashtag or channel.name or "").strip().lstrip("#") + tag = (channel.name or "").strip().lstrip("#") if tag: - return f"#{tag}" + return f"#{tag}{_scope_suffix(channel)}" name = (channel.name or "").strip() if name: - return name + return f"{name}{_scope_suffix(channel)}" return str(channel.pk) @@ -53,10 +60,10 @@ def message_channel_to_apply_entry( "mc_channel_idx": mc_channel_idx, "name": channel.name, "mc_channel_type": ch_type, + "region_scope": channel.region_scope, } if channel.mc_channel_type == MeshCoreChannelType.HASHTAG: - tag = (channel.mc_hashtag or channel.name or "").strip().lstrip("#") - entry["mc_hashtag"] = tag[:64] if tag else None + tag = (channel.name or "").strip().lstrip("#") if tag: entry["name"] = tag[:100] return entry diff --git a/Meshflow/common/mc_region_scope.py b/Meshflow/common/mc_region_scope.py new file mode 100644 index 0000000..2e3619a --- /dev/null +++ b/Meshflow/common/mc_region_scope.py @@ -0,0 +1,31 @@ +"""Normalize MeshCore region scope names for storage and API validation.""" + +from __future__ import annotations + +import re + +REGION_SCOPE_MAX_BYTES = 29 + +# Lowercase alphanumeric and hyphen per MeshCore region naming rules. +_REGION_SCOPE_RE = re.compile(r"^[a-z0-9-]+$") + +_NULL_SCOPE_VALUES = frozenset({"", "*", "none", "null"}) + + +def normalize_region_scope(value: str | None) -> str | None: + """ + Return a canonical region name for DB storage, or None for null/legacy scope. + + Strips whitespace, lowercases, rejects invalid characters. Maps *, empty, and + "none" to None (null region — legacy flood everywhere on mesh). + """ + if value is None: + return None + raw = str(value).strip().lower().lstrip("#") + if not raw or raw in _NULL_SCOPE_VALUES: + return None + if len(raw.encode("utf-8")) > REGION_SCOPE_MAX_BYTES: + raise ValueError(f"region_scope exceeds {REGION_SCOPE_MAX_BYTES} UTF-8 bytes.") + if not _REGION_SCOPE_RE.match(raw): + raise ValueError("region_scope must be lowercase alphanumeric with hyphens only.") + return raw diff --git a/Meshflow/common/tests/test_mc_channel_labels.py b/Meshflow/common/tests/test_mc_channel_labels.py index 463c255..b5e4929 100644 --- a/Meshflow/common/tests/test_mc_channel_labels.py +++ b/Meshflow/common/tests/test_mc_channel_labels.py @@ -30,11 +30,23 @@ def test_mc_channel_admin_label_hashtag_prefix(create_constellation): constellation=constellation, protocol=Protocol.MESHCORE, mc_channel_type=MeshCoreChannelType.HASHTAG, - mc_hashtag="galloway", ) assert mc_channel_admin_label(ch) == "#galloway" +@pytest.mark.django_db +def test_mc_channel_admin_label_hashtag_with_scope(create_constellation): + constellation = create_constellation() + ch = MessageChannel.objects.create( + name="galloway", + constellation=constellation, + protocol=Protocol.MESHCORE, + mc_channel_type=MeshCoreChannelType.HASHTAG, + region_scope="sample-west", + ) + assert mc_channel_admin_label(ch) == "#galloway · sample-west" + + @pytest.mark.django_db def test_message_channel_to_apply_entry_hashtag(create_constellation): constellation = create_constellation() @@ -43,9 +55,9 @@ def test_message_channel_to_apply_entry_hashtag(create_constellation): constellation=constellation, protocol=Protocol.MESHCORE, mc_channel_type=MeshCoreChannelType.HASHTAG, - mc_hashtag="galloway", + region_scope="sample-west", ) entry = message_channel_to_apply_entry(ch, mc_channel_idx=1) assert entry["mc_channel_type"] == "HASHTAG" - assert entry["mc_hashtag"] == "galloway" assert entry["name"] == "galloway" + assert entry["region_scope"] == "sample-west" diff --git a/Meshflow/common/tests/test_mc_region_scope.py b/Meshflow/common/tests/test_mc_region_scope.py new file mode 100644 index 0000000..dce0ae9 --- /dev/null +++ b/Meshflow/common/tests/test_mc_region_scope.py @@ -0,0 +1,25 @@ +"""Tests for MeshCore region_scope normalization.""" + +import pytest + +from common.mc_region_scope import normalize_region_scope + + +def test_normalize_region_scope_null_values(): + assert normalize_region_scope(None) is None + assert normalize_region_scope("") is None + assert normalize_region_scope("*") is None + assert normalize_region_scope("none") is None + assert normalize_region_scope(" * ") is None + + +def test_normalize_region_scope_valid(): + assert normalize_region_scope("Sample-West") == "sample-west" + assert normalize_region_scope("#galloway") == "galloway" + + +def test_normalize_region_scope_invalid(): + with pytest.raises(ValueError, match="alphanumeric"): + normalize_region_scope("bad scope") + with pytest.raises(ValueError, match="UTF-8"): + normalize_region_scope("a" * 30) diff --git a/Meshflow/constellations/admin.py b/Meshflow/constellations/admin.py index 5e17769..6af92ee 100644 --- a/Meshflow/constellations/admin.py +++ b/Meshflow/constellations/admin.py @@ -116,7 +116,7 @@ class MeshCoreMessageChannelAdmin(admin.ModelAdmin): ("mc_channel_type", admin.ChoicesFieldListFilter), "constellation", ) - search_fields = ("name", "mc_hashtag", "constellation__name") + search_fields = ("name", "region_scope", "constellation__name") ordering = ("constellation__name", "name") list_select_related = ("constellation",) fieldsets = ( @@ -124,10 +124,10 @@ class MeshCoreMessageChannelAdmin(admin.ModelAdmin): ( _("Channel"), { - "fields": ("name", "mc_channel_type", "mc_hashtag"), + "fields": ("name", "mc_channel_type", "region_scope"), "description": _( - "PUBLIC channels use a plain name. HASHTAG channels use mc_hashtag " - "(no leading #); lists show #prefix for hashtags." + "PUBLIC channels use a plain name. HASHTAG channels store the tag in name " + "(no leading #); lists show #prefix. region_scope is optional (null = legacy scope)." ), }, ), diff --git a/Meshflow/constellations/migrations/0015_messagechannel_region_scope.py b/Meshflow/constellations/migrations/0015_messagechannel_region_scope.py new file mode 100644 index 0000000..40a7989 --- /dev/null +++ b/Meshflow/constellations/migrations/0015_messagechannel_region_scope.py @@ -0,0 +1,124 @@ +# Add region_scope, backfill from mc_hashtag, merge duplicates, drop mc_hashtag + +from django.db import migrations, models + +MC_PROTOCOL = 2 + + +def _normalize_tag(value): + if not value: + return "" + return str(value).strip().lstrip("#").lower()[:100] + + +def _logical_key(channel): + name = str(channel.name or "").strip().lower() + if channel.mc_channel_type == 2 and channel.mc_hashtag: + tag = _normalize_tag(channel.mc_hashtag) + if tag: + name = tag + return (channel.mc_channel_type, name, channel.region_scope) + + +def _repoint_channel_fks(apps, old_id, new_id): + if old_id == new_id: + return + TextMessage = apps.get_model("text_messages", "TextMessage") + TextMessage.objects.filter(channel_id=old_id).update(channel_id=new_id) + + try: + MeshCoreTextPacket = apps.get_model("meshcore_packets", "MeshCoreTextPacket") + MeshCoreTextPacket.objects.filter(channel_id=old_id).update(channel_id=new_id) + except LookupError: + pass + + try: + MeshCorePacketObservation = apps.get_model("meshcore_packets", "MeshCorePacketObservation") + MeshCorePacketObservation.objects.filter(channel_id=old_id).update(channel_id=new_id) + except LookupError: + pass + + Link = apps.get_model("nodes", "ManagedNodeMcChannelLink") + Link.objects.filter(message_channel_id=old_id).update(message_channel_id=new_id) + + +def backfill_hashtag_names(apps, schema_editor): + MessageChannel = apps.get_model("constellations", "MessageChannel") + for ch in MessageChannel.objects.filter(protocol=MC_PROTOCOL, mc_channel_type=2): + tag = _normalize_tag(ch.mc_hashtag or ch.name) + if tag and ch.name != tag: + ch.name = tag + ch.save(update_fields=["name"]) + + +def merge_duplicate_mc_channels(apps, schema_editor): + MessageChannel = apps.get_model("constellations", "MessageChannel") + channels = list(MessageChannel.objects.filter(protocol=MC_PROTOCOL)) + groups = {} + for ch in channels: + key = (ch.constellation_id, _logical_key(ch)) + groups.setdefault(key, []).append(ch) + + for _key, group in groups.items(): + if len(group) < 2: + continue + group.sort(key=lambda c: c.id) + survivor = group[0] + for dup in group[1:]: + _repoint_channel_fks(apps, dup.id, survivor.id) + MessageChannel.objects.filter(id=dup.id).delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ("constellations", "0014_mc_canonical_channels_finalize"), + ("text_messages", "0008_textmessage_mc_provenance"), + ("meshcore_packets", "0004_observation_path_hash_size_mode"), + ] + + operations = [ + migrations.AddField( + model_name="messagechannel", + name="region_scope", + field=models.CharField( + blank=True, + help_text=( + "MeshCore region scope when protocol is MeshCore (lowercase alphanumeric + " + "hyphen; null = legacy / no scope)." + ), + max_length=29, + null=True, + ), + ), + migrations.RunPython(backfill_hashtag_names, migrations.RunPython.noop), + migrations.RunPython(merge_duplicate_mc_channels, migrations.RunPython.noop), + migrations.RemoveConstraint( + model_name="messagechannel", + name="messagechannel_mc_hashtag_constellation_unique", + ), + migrations.RemoveConstraint( + model_name="messagechannel", + name="messagechannel_mc_public_name_constellation_unique", + ), + migrations.RemoveField( + model_name="messagechannel", + name="mc_hashtag", + ), + migrations.AddConstraint( + model_name="messagechannel", + constraint=models.UniqueConstraint( + condition=models.Q(("protocol", MC_PROTOCOL), ("region_scope__isnull", True)), + fields=("constellation", "protocol", "name", "mc_channel_type"), + name="messagechannel_mc_null_scope_unique", + ), + ), + migrations.AddConstraint( + model_name="messagechannel", + constraint=models.UniqueConstraint( + condition=models.Q(("protocol", MC_PROTOCOL), ("region_scope__isnull", False)), + fields=("constellation", "protocol", "name", "mc_channel_type", "region_scope"), + name="messagechannel_mc_region_scope_unique", + ), + ), + ] diff --git a/Meshflow/constellations/models.py b/Meshflow/constellations/models.py index 466d190..eb6f03b 100644 --- a/Meshflow/constellations/models.py +++ b/Meshflow/constellations/models.py @@ -53,11 +53,14 @@ class MessageChannel(models.Model): blank=True, help_text=_("MeshCore channel type when protocol is MeshCore."), ) - mc_hashtag = models.CharField( - max_length=64, + region_scope = models.CharField( + max_length=29, null=True, blank=True, - help_text=_("Hashtag string when mc_channel_type is HASHTAG (no leading #)."), + help_text=_( + "MeshCore region scope when protocol is MeshCore (lowercase alphanumeric + hyphen; " + "null = legacy / no scope)." + ), ) class Meta: @@ -65,21 +68,20 @@ class Meta: verbose_name_plural = _("Message channels") constraints = [ models.UniqueConstraint( - fields=["constellation", "protocol", "mc_hashtag"], + fields=["constellation", "protocol", "name", "mc_channel_type"], condition=models.Q( protocol=Protocol.MESHCORE, - mc_channel_type=MeshCoreChannelType.HASHTAG, - mc_hashtag__isnull=False, + region_scope__isnull=True, ), - name="messagechannel_mc_hashtag_constellation_unique", + name="messagechannel_mc_null_scope_unique", ), models.UniqueConstraint( - fields=["constellation", "protocol", "name"], + fields=["constellation", "protocol", "name", "mc_channel_type", "region_scope"], condition=models.Q( protocol=Protocol.MESHCORE, - mc_channel_type=MeshCoreChannelType.PUBLIC, + region_scope__isnull=False, ), - name="messagechannel_mc_public_name_constellation_unique", + name="messagechannel_mc_region_scope_unique", ), ] diff --git a/Meshflow/constellations/serializers.py b/Meshflow/constellations/serializers.py index f780be7..9d85bbd 100644 --- a/Meshflow/constellations/serializers.py +++ b/Meshflow/constellations/serializers.py @@ -15,7 +15,7 @@ def message_channel_payload(channel: MessageChannel) -> dict: "name": channel.name, "protocol": channel.protocol, "mc_channel_type": mc_type, - "mc_hashtag": channel.mc_hashtag, + "region_scope": channel.region_scope, "constellation": channel.constellation_id, } if channel.protocol == Protocol.MESHCORE: diff --git a/Meshflow/meshcore_packets/serializers_channel.py b/Meshflow/meshcore_packets/serializers_channel.py index 27f2867..48401bf 100644 --- a/Meshflow/meshcore_packets/serializers_channel.py +++ b/Meshflow/meshcore_packets/serializers_channel.py @@ -2,6 +2,7 @@ from rest_framework import serializers +from common.mc_region_scope import normalize_region_scope from constellations.models import MeshCoreChannelType, MessageChannel from nodes.models import ManagedNodeMcChannelLink @@ -16,7 +17,7 @@ class McChannelSnapshotEntrySerializer(serializers.Serializer): mc_channel_idx = serializers.IntegerField(min_value=0, max_value=63) name = serializers.CharField(max_length=100) mc_channel_type = serializers.ChoiceField(choices=MC_CHANNEL_TYPE_API_CHOICES) - mc_hashtag = serializers.CharField(max_length=64, required=False, allow_null=True, allow_blank=True) + region_scope = serializers.CharField(max_length=29, required=False, allow_null=True, allow_blank=True) class McChannelSyncSerializer(serializers.Serializer): @@ -35,7 +36,7 @@ class Meta: "id", "name", "mc_channel_type", - "mc_hashtag", + "region_scope", ] def get_mc_channel_type(self, obj): @@ -50,8 +51,8 @@ class FeederMcChannelMirrorSerializer(serializers.ModelSerializer): id = serializers.IntegerField(source="message_channel.id", read_only=True) name = serializers.CharField(source="message_channel.name", read_only=True) mc_channel_type = serializers.SerializerMethodField() - mc_hashtag = serializers.CharField( - source="message_channel.mc_hashtag", + region_scope = serializers.CharField( + source="message_channel.region_scope", read_only=True, allow_null=True, ) @@ -63,7 +64,7 @@ class Meta: "name", "mc_channel_idx", "mc_channel_type", - "mc_hashtag", + "region_scope", ] def get_mc_channel_type(self, obj): @@ -77,7 +78,7 @@ class McChannelApplyEntrySerializer(serializers.Serializer): mc_channel_idx = serializers.IntegerField(min_value=0, max_value=63, required=False) name = serializers.CharField(max_length=100) mc_channel_type = serializers.ChoiceField(choices=MC_CHANNEL_TYPE_API_CHOICES) - mc_hashtag = serializers.CharField(max_length=64, required=False, allow_null=True, allow_blank=True) + region_scope = serializers.CharField(max_length=29, required=False, allow_null=True, allow_blank=True) class McChannelApplySerializer(serializers.Serializer): @@ -86,10 +87,14 @@ class McChannelApplySerializer(serializers.Serializer): def validate(self, attrs): channels = attrs.get("channels") or [] for entry in channels: + if "region_scope" in entry: + try: + entry["region_scope"] = normalize_region_scope(entry.get("region_scope")) + except ValueError as exc: + raise serializers.ValidationError({"channels": str(exc)}) from exc if str(entry.get("mc_channel_type")).upper() == "HASHTAG": - tag = (entry.get("mc_hashtag") or entry.get("name") or "").strip().lstrip("#") + tag = (entry.get("name") or "").strip().lstrip("#") if not tag: - raise serializers.ValidationError({"channels": "Hashtag channels require a non-empty hashtag."}) - entry["mc_hashtag"] = tag[:64] + raise serializers.ValidationError({"channels": "Hashtag channels require a non-empty name."}) entry["name"] = tag[:100] return attrs diff --git a/Meshflow/meshcore_packets/services/channel_identity.py b/Meshflow/meshcore_packets/services/channel_identity.py index aee4fc4..807a877 100644 --- a/Meshflow/meshcore_packets/services/channel_identity.py +++ b/Meshflow/meshcore_packets/services/channel_identity.py @@ -1,18 +1,20 @@ -"""Canonical MeshCore MessageChannel identity (logical name/hashtag, not device index).""" +"""Canonical MeshCore MessageChannel identity (logical name/type/scope, not device index).""" from __future__ import annotations +from common.mc_region_scope import normalize_region_scope from common.protocol import Protocol from constellations.models import MeshCoreChannelType, MessageChannel MC_CHANNEL_IDX_MAX = 63 -def normalize_mc_hashtag(value: str | None) -> str | None: +def normalize_mc_hashtag_name(value: str | None) -> str | None: + """Normalize a HASHTAG channel tag (stored in MessageChannel.name, no leading #).""" if value is None: return None tag = str(value).strip().lstrip("#").lower() - return tag[:64] if tag else None + return tag[:100] if tag else None def normalize_mc_public_name(value: str | None) -> str: @@ -33,27 +35,48 @@ def _parse_channel_type(value: str | int | None) -> int: return MeshCoreChannelType.PUBLIC +def _parse_region_scope(entry: dict) -> str | None: + if "region_scope" not in entry: + return None + try: + return normalize_region_scope(entry.get("region_scope")) + except ValueError as exc: + raise ValueError(str(exc)) from exc + + +def snapshot_entry_matches_channel(entry: dict, channel: MessageChannel) -> bool: + """True when a device snapshot row describes the same logical channel row.""" + ch_type = _parse_channel_type(entry.get("mc_channel_type")) + if channel.mc_channel_type != ch_type: + return False + if ch_type == MeshCoreChannelType.HASHTAG: + tag = normalize_mc_hashtag_name(entry.get("name") or entry.get("mc_hashtag")) + return bool(tag) and tag == (channel.name or "").strip().lower() + name = normalize_mc_public_name(entry.get("name")) + return name == (channel.name or "").strip() + + def upsert_canonical_mc_channel(constellation, entry: dict) -> MessageChannel: """ - Resolve or create a constellation-scoped canonical MessageChannel from a device snapshot entry. + Resolve or create a constellation-scoped canonical MessageChannel from a device snapshot entry. - HASHTAG rows are keyed by normalized mc_hashtag; PUBLIC rows by normalized name. + MC rows are keyed by (name, mc_channel_type, region_scope). For HASHTAG, name is the tag + without #; for PUBLIC, name is the public channel name. """ ch_type = _parse_channel_type(entry.get("mc_channel_type")) + region_scope = _parse_region_scope(entry) if ch_type == MeshCoreChannelType.HASHTAG: - hashtag = normalize_mc_hashtag(entry.get("mc_hashtag") or entry.get("name")) - if not hashtag: + tag = normalize_mc_hashtag_name(entry.get("name") or entry.get("mc_hashtag")) + if not tag: raise ValueError("Hashtag channels require a non-empty hashtag.") - display_name = str(entry.get("name") or hashtag).strip().lstrip("#")[:100] or hashtag channel, _created = MessageChannel.objects.update_or_create( constellation=constellation, protocol=Protocol.MESHCORE, mc_channel_type=MeshCoreChannelType.HASHTAG, - mc_hashtag=hashtag, - defaults={ - "name": display_name, - }, + name=tag, + region_scope=region_scope, + defaults={}, ) return channel @@ -63,9 +86,8 @@ def upsert_canonical_mc_channel(constellation, entry: dict) -> MessageChannel: protocol=Protocol.MESHCORE, mc_channel_type=MeshCoreChannelType.PUBLIC, name=name, - defaults={ - "mc_hashtag": None, - }, + region_scope=region_scope, + defaults={}, ) return channel @@ -77,5 +99,6 @@ def placeholder_canonical_mc_channel(constellation, channel_idx: int) -> Message protocol=Protocol.MESHCORE, mc_channel_type=MeshCoreChannelType.PUBLIC, name=f"MC channel {channel_idx}", - defaults={"mc_hashtag": None}, + region_scope=None, + defaults={}, )[0] diff --git a/Meshflow/meshcore_packets/services/channel_sync.py b/Meshflow/meshcore_packets/services/channel_sync.py index 7758800..6bbd6b9 100644 --- a/Meshflow/meshcore_packets/services/channel_sync.py +++ b/Meshflow/meshcore_packets/services/channel_sync.py @@ -6,12 +6,49 @@ from django.utils import timezone from django.utils.dateparse import parse_datetime +from common.mc_region_scope import normalize_region_scope from common.protocol import Protocol from constellations.models import MessageChannel -from meshcore_packets.services.channel_identity import MC_CHANNEL_IDX_MAX, upsert_canonical_mc_channel +from meshcore_packets.services.channel_identity import ( + MC_CHANNEL_IDX_MAX, + snapshot_entry_matches_channel, + upsert_canonical_mc_channel, +) from nodes.models import ManagedNode, ManagedNodeMcChannelLink +def enrich_snapshot_region_scope(managed_node: ManagedNode, entry: dict) -> dict: + """ + When the device snapshot omits region_scope, preserve scope from the feeder's + existing slot link when name and type still match (companion protocol gap). + """ + entry = dict(entry) + if "region_scope" in entry: + try: + scope = normalize_region_scope(entry.get("region_scope")) + except ValueError: + scope = None + if scope is not None: + entry["region_scope"] = scope + return entry + + idx = entry.get("mc_channel_idx") + if idx is None: + return entry + + link = managed_node.mc_channel_links.filter(mc_channel_idx=int(idx)).select_related("message_channel").first() + if link is None: + return entry + + channel = link.message_channel + if not channel.region_scope: + return entry + if not snapshot_entry_matches_channel(entry, channel): + return entry + entry["region_scope"] = channel.region_scope + return entry + + def reconcile_mc_channels( managed_node: ManagedNode, channels: list[dict], @@ -20,7 +57,7 @@ def reconcile_mc_channels( """ Upsert canonical MC MessageChannel rows and feeder slot links from a device snapshot. - Each channel dict: mc_channel_idx, name, mc_channel_type (PUBLIC|HASHTAG), mc_hashtag (optional). + Each channel dict: mc_channel_idx, name, mc_channel_type (PUBLIC|HASHTAG), region_scope (optional). """ if managed_node.protocol != Protocol.MESHCORE: raise ValueError("mc-channel-sync is only valid for MeshCore managed nodes") @@ -36,7 +73,8 @@ def reconcile_mc_channels( attached: list[MessageChannel] = [] with transaction.atomic(): - for entry in channels: + for raw_entry in channels: + entry = enrich_snapshot_region_scope(managed_node, raw_entry) idx = entry.get("mc_channel_idx") if idx is None: continue diff --git a/Meshflow/meshcore_packets/tests/test_apply_channel.py b/Meshflow/meshcore_packets/tests/test_apply_channel.py index af50684..1b49954 100644 --- a/Meshflow/meshcore_packets/tests/test_apply_channel.py +++ b/Meshflow/meshcore_packets/tests/test_apply_channel.py @@ -36,7 +36,7 @@ def test_apply_returns_503_when_feeder_not_connected(create_user, create_managed "mc_channel_idx": 0, "name": "galloway", "mc_channel_type": "HASHTAG", - "mc_hashtag": "galloway", + "region_scope": "sample-west", } ] }, @@ -119,9 +119,8 @@ def test_apply_dispatches_when_feeder_connected(create_user, create_managed_node "channels": [ { "mc_channel_idx": 1, - "name": "tag", + "name": "galloway", "mc_channel_type": "HASHTAG", - "mc_hashtag": "#galloway", } ] }, @@ -131,7 +130,6 @@ def test_apply_dispatches_when_feeder_connected(create_user, create_managed_node assert response.status_code == 202 dispatch_mock.assert_called_once() sent_channels = dispatch_mock.call_args[0][1] - assert sent_channels[0]["mc_hashtag"] == "galloway" assert sent_channels[0]["name"] == "galloway" assert sent_channels[0]["mc_channel_type"] == "HASHTAG" assert type(sent_channels[0]["mc_channel_type"]) is str diff --git a/Meshflow/meshcore_packets/tests/test_canonical_channels.py b/Meshflow/meshcore_packets/tests/test_canonical_channels.py index f9bbc7e..b28ecbe 100644 --- a/Meshflow/meshcore_packets/tests/test_canonical_channels.py +++ b/Meshflow/meshcore_packets/tests/test_canonical_channels.py @@ -33,17 +33,18 @@ def test_two_feeders_same_hashtag_different_indices_one_canonical( reconcile_mc_channels( meshcore_feeder["node"], - [{"mc_channel_idx": 1, "name": "test", "mc_channel_type": "HASHTAG", "mc_hashtag": "test"}], + [{"mc_channel_idx": 1, "name": "test", "mc_channel_type": "HASHTAG"}], ) reconcile_mc_channels( feeder_b, - [{"mc_channel_idx": 2, "name": "test", "mc_channel_type": "HASHTAG", "mc_hashtag": "test"}], + [{"mc_channel_idx": 2, "name": "test", "mc_channel_type": "HASHTAG"}], ) canonical = MessageChannel.objects.filter( constellation=constellation, protocol=Protocol.MESHCORE, - mc_hashtag="test", + name="test", + mc_channel_type=2, ) assert canonical.count() == 1 @@ -52,6 +53,38 @@ def test_two_feeders_same_hashtag_different_indices_one_canonical( assert ch_a.id == ch_b.id +@pytest.mark.django_db +def test_same_hashtag_different_region_scope_two_canonical(meshcore_feeder): + """Same tag in different region scopes → distinct MessageChannel rows.""" + constellation = meshcore_feeder["node"].constellation + reconcile_mc_channels( + meshcore_feeder["node"], + [ + { + "mc_channel_idx": 1, + "name": "galloway", + "mc_channel_type": "HASHTAG", + "region_scope": "sample-west", + }, + { + "mc_channel_idx": 2, + "name": "galloway", + "mc_channel_type": "HASHTAG", + "region_scope": "uk-wide", + }, + ], + ) + rows = MessageChannel.objects.filter( + constellation=constellation, + protocol=Protocol.MESHCORE, + name="galloway", + mc_channel_type=2, + ) + assert rows.count() == 2 + scopes = {r.region_scope for r in rows} + assert scopes == {"sample-west", "uk-wide"} + + @pytest.mark.django_db def test_two_feeders_ingest_same_text_same_channel_id(meshcore_feeder, create_managed_node, create_node_api_key): constellation = meshcore_feeder["node"].constellation @@ -64,11 +97,11 @@ def test_two_feeders_ingest_same_text_same_channel_id(meshcore_feeder, create_ma ) reconcile_mc_channels( meshcore_feeder["node"], - [{"mc_channel_idx": 1, "name": "test", "mc_channel_type": "HASHTAG", "mc_hashtag": "test"}], + [{"mc_channel_idx": 1, "name": "test", "mc_channel_type": "HASHTAG"}], ) reconcile_mc_channels( feeder_b, - [{"mc_channel_idx": 2, "name": "test", "mc_channel_type": "HASHTAG", "mc_hashtag": "test"}], + [{"mc_channel_idx": 2, "name": "test", "mc_channel_type": "HASHTAG"}], ) now = timezone.now() diff --git a/Meshflow/meshcore_packets/tests/test_channel_apply_service.py b/Meshflow/meshcore_packets/tests/test_channel_apply_service.py index adc06dd..3871da1 100644 --- a/Meshflow/meshcore_packets/tests/test_channel_apply_service.py +++ b/Meshflow/meshcore_packets/tests/test_channel_apply_service.py @@ -15,11 +15,11 @@ def test_build_apply_channels_for_managed_node(meshcore_feeder): node = meshcore_feeder["node"] constellation = node.constellation ch = MessageChannel.objects.create( - name="tag", + name="meshflow", constellation=constellation, protocol=Protocol.MESHCORE, mc_channel_type=MeshCoreChannelType.HASHTAG, - mc_hashtag="meshflow", + region_scope="uk-wide", ) ManagedNodeMcChannelLink.objects.create( managed_node=node, @@ -31,7 +31,8 @@ def test_build_apply_channels_for_managed_node(meshcore_feeder): assert len(payload) == 1 assert payload[0]["mc_channel_idx"] == 2 assert payload[0]["mc_channel_type"] == "HASHTAG" - assert payload[0]["mc_hashtag"] == "meshflow" + assert payload[0]["name"] == "meshflow" + assert payload[0]["region_scope"] == "uk-wide" @pytest.mark.django_db diff --git a/Meshflow/meshcore_packets/tests/test_channel_sync.py b/Meshflow/meshcore_packets/tests/test_channel_sync.py index 9223fe3..45fd407 100644 --- a/Meshflow/meshcore_packets/tests/test_channel_sync.py +++ b/Meshflow/meshcore_packets/tests/test_channel_sync.py @@ -29,13 +29,11 @@ def test_reconcile_mc_channels_creates_and_links(meshcore_feeder): "mc_channel_idx": 0, "name": "Public", "mc_channel_type": "PUBLIC", - "mc_hashtag": None, }, { "mc_channel_idx": 1, - "name": "Galloway", + "name": "galloway", "mc_channel_type": "HASHTAG", - "mc_hashtag": "galloway", }, ], ) @@ -95,6 +93,91 @@ def test_mc_channel_sync_endpoint(sync_client, meshcore_feeder): assert meshcore_feeder["node"].mc_channels.count() == 1 +@pytest.mark.django_db +def test_reconcile_preserves_region_scope_when_snapshot_omits_scope(meshcore_feeder): + """Device read without scope must not create a duplicate unscoped row.""" + from constellations.models import MeshCoreChannelType + + node = meshcore_feeder["node"] + reconcile_mc_channels( + node, + [ + { + "mc_channel_idx": 0, + "name": "galloway", + "mc_channel_type": "HASHTAG", + "region_scope": "sample-west", + }, + ], + ) + scoped = MessageChannel.objects.get( + constellation=node.constellation, + name="galloway", + region_scope="sample-west", + ) + reconcile_mc_channels( + node, + [ + { + "mc_channel_idx": 0, + "name": "galloway", + "mc_channel_type": "HASHTAG", + }, + ], + ) + node.refresh_from_db() + link = node.mc_channel_links.get(mc_channel_idx=0) + assert link.message_channel_id == scoped.id + assert ( + MessageChannel.objects.filter( + constellation=node.constellation, + protocol=Protocol.MESHCORE, + name="galloway", + mc_channel_type=MeshCoreChannelType.HASHTAG, + ).count() + == 1 + ) + + +@pytest.mark.django_db +def test_reconcile_rescope_updates_canonical_row(meshcore_feeder): + """Applying a new region_scope for the same tag creates/uses a distinct canonical row.""" + from constellations.models import MeshCoreChannelType + + node = meshcore_feeder["node"] + reconcile_mc_channels( + node, + [ + { + "mc_channel_idx": 0, + "name": "galloway", + "mc_channel_type": "HASHTAG", + "region_scope": "sample-west", + }, + ], + ) + reconcile_mc_channels( + node, + [ + { + "mc_channel_idx": 0, + "name": "galloway", + "mc_channel_type": "HASHTAG", + "region_scope": "uk-wide", + }, + ], + ) + rows = MessageChannel.objects.filter( + constellation=node.constellation, + protocol=Protocol.MESHCORE, + name="galloway", + mc_channel_type=MeshCoreChannelType.HASHTAG, + ) + assert rows.count() == 2 + link = node.mc_channel_links.get(mc_channel_idx=0) + assert link.message_channel.region_scope == "uk-wide" + + @pytest.mark.django_db def test_resolve_mc_channel_prefers_m2m(meshcore_feeder): node = meshcore_feeder["node"] diff --git a/Meshflow/meshcore_packets/tests/test_cross_feeder_dedup.py b/Meshflow/meshcore_packets/tests/test_cross_feeder_dedup.py index 97db0b2..d8c860c 100644 --- a/Meshflow/meshcore_packets/tests/test_cross_feeder_dedup.py +++ b/Meshflow/meshcore_packets/tests/test_cross_feeder_dedup.py @@ -69,11 +69,11 @@ def second_mc_feeder(meshcore_feeder, create_managed_node, create_node_api_key): def _setup_hashtag_channels(meshcore_feeder, second_mc_feeder): reconcile_mc_channels( meshcore_feeder["node"], - [{"mc_channel_idx": 1, "name": "test", "mc_channel_type": "HASHTAG", "mc_hashtag": "test"}], + [{"mc_channel_idx": 1, "name": "test", "mc_channel_type": "HASHTAG"}], ) reconcile_mc_channels( second_mc_feeder["node"], - [{"mc_channel_idx": 2, "name": "test", "mc_channel_type": "HASHTAG", "mc_hashtag": "test"}], + [{"mc_channel_idx": 2, "name": "test", "mc_channel_type": "HASHTAG"}], ) diff --git a/docs/features/meshcore/README.md b/docs/features/meshcore/README.md index 3845002..0cf5751 100644 --- a/docs/features/meshcore/README.md +++ b/docs/features/meshcore/README.md @@ -19,7 +19,8 @@ Cross-repo MeshCore work is tracked on GitHub epics [#264](https://github.com/ps **Feature guides** (how-to, not phase status): - [feeder-bootstrap.md](./feeder-bootstrap.md) — first MC feeder + API key + `mc_pubkey` -- [text-message-channels.md](./text-message-channels.md) — Phase 2.2 text + channels (device → API sync, apply-to-radio, ops troubleshooting) +- [mc-channel-sync/](./mc-channel-sync/) — feeder channel mirror (`mc-channel-sync`, apply-to-radio, data model, ops) +- [text-message-channels.md](./text-message-channels.md) — MC text ingest, `TextMessage`, wire format (channel config → [mc-channel-sync/](./mc-channel-sync/)) - [packet-path-tracing/](./packet-path-tracing/) — proposed passive MC packet path subsystem (capture, resolution, Neo4j, realtime/history UI) - [../node-lifecycle/node-claims-meshcore.md](../node-lifecycle/node-claims-meshcore.md) — ownership claims via contact/DM proof diff --git a/docs/features/meshcore/feeder-bootstrap.md b/docs/features/meshcore/feeder-bootstrap.md index d91e1f0..08977e5 100644 --- a/docs/features/meshcore/feeder-bootstrap.md +++ b/docs/features/meshcore/feeder-bootstrap.md @@ -86,21 +86,7 @@ Restart the bot. Confirm logs show feeder-scoped paths, e.g. `POST /api/meshcore On connect, **meshflow-bot** reads the device channel table and calls **`POST /api/meshcore/feeders/{prefix}/mc-channel-sync/`** so the API mirror matches the radio. -See [text-message-channels.md](./text-message-channels.md). - -### Django admin (operators) - -- **MeshCore channels** — constellation MC channel catalog only (proxy admin); hashtag rows show with **`#` prefix** in lists. -- **Managed nodes** (MeshCore) — **read-only** “device mirror” table (slot, type, label); **`mc_channels_synced_at`** from last bot sync. -- **Push MC channel config to feeder device** — admin action sends the current mirror to the radio over WebSocket (same mechanism as UI apply). Requires bot connected on **primary** API WS (see dual-API below). - -Editing channel definitions in admin does **not** auto-push; use the push action after the mirror reflects what you want on air. - -### Dual API upload (`STORAGE_API_2_*`) - -If the bot sets **`STORAGE_API_2_ROOT`** + token with `MESHCORE_UPLOAD_ENABLED`, it uploads packets, bot version, and **`mc-channel-sync`** to **both** APIs. - -**WebSocket** (apply-to-radio, traceroute) is only started from **`STORAGE_API_ROOT`** (or `MESHFLOW_WS_URL` pointing at the primary). API 2 receives passive mirror updates but will not receive `apply_mc_channel_config` unless WS is also pointed there. +Full flow, data model, Django admin push action, and dual-API behaviour: **[mc-channel-sync/](mc-channel-sync/)** (see [operations.md](mc-channel-sync/operations.md) for `STORAGE_API_2_*` and apply troubleshooting). ## Operator trial (24h) diff --git a/docs/features/meshcore/mc-channel-sync/README.md b/docs/features/meshcore/mc-channel-sync/README.md new file mode 100644 index 0000000..1f5e0d1 --- /dev/null +++ b/docs/features/meshcore/mc-channel-sync/README.md @@ -0,0 +1,116 @@ +# MeshCore channel sync (`mc-channel-sync`) + +Meshflow **feeders** (MeshCore `ManagedNode`s) listen on group channels configured on the companion radio. The API cannot read the radio directly; **meshflow-bot** uploads a **device channel snapshot** after connect (and again after operator **apply-to-radio**). The API reconciles that snapshot into a **per-feeder mirror** and a **constellation-scoped channel catalog** used for ingest, text history, and the UI. + +This feature is **configuration only** — it does not upload mesh traffic. Packet ingest uses the same feeder identity URLs but a separate endpoint ([packet-ingestion](../../packet-ingestion/README.md)). + +--- + +## Implementation status + +| Area | Status | Notes | +|------|--------|--------| +| `POST …/mc-channel-sync/` (device → API) | Shipped | Feeder Node API key; full snapshot reconcile | +| Bot read on connect + after apply | Shipped | `meshcore.commands.get_channel`; optional dual API POST | +| `POST …/apply-mc-channel-config/` (UI → device) | Shipped | WebSocket + Redis channel layer | +| Canonical `MessageChannel` + `ManagedNodeMcChannelLink` | Shipped | Logical identity separate from device slot ([#379](https://github.com/pskillen/meshflow-api/issues/379)) | +| `region_scope` on channels | Shipped | [#391](https://github.com/pskillen/meshflow-api/issues/391) | +| API-only mirror edits (no device round-trip) | Not supported | Next connect sync overwrites API from device | +| Periodic background sync | Not supported | Connect + post-apply only | + +--- + +## Documentation map + +| Doc | Contents | +|-----|----------| +| [flow.md](flow.md) | End-to-end sequences, HTTP/WebSocket contracts, reconcile behaviour | +| [data-model.md](data-model.md) | `MessageChannel`, feeder links, placeholders at ingest | +| [operations.md](operations.md) | Auth, dual API, scaling, troubleshooting | +| [region-scope.md](region-scope.md) | Region scope semantics, API field, bot/UI behaviour | +| [region-scope-progress.md](region-scope-progress.md) | Execution log ([#391](https://github.com/pskillen/meshflow-api/issues/391)) | +| [region-scope-outstanding.md](region-scope-outstanding.md) | Protocol gaps and follow-ups | + +**Related (other features)** + +| Doc | Relationship | +|-----|----------------| +| [feeder-bootstrap.md](../feeder-bootstrap.md) | First feeder, API key, env vars | +| [text-message-channels.md](../text-message-channels.md) | MC text ingest, `TextMessage`, wire format | +| [ADR-0002](../../packet-ingestion/adr/0002-mc-channel-modelling.md) | Normative channel modelling | +| [meshflow-bot `docs/MESHCORE.md`](https://github.com/pskillen/meshflow-bot/blob/main/docs/MESHCORE.md) | Bot env and transport | + +**OpenAPI:** `McChannelSyncRequest`, `McChannelSyncResponse`, `McChannelApplyRequest` under `/meshcore/`. + +--- + +## Concepts + +| Term | Meaning | +|------|---------| +| **Device slot** | `mc_channel_idx` (0–63) on the companion channel table. On the wire, `channel_message` carries **index only**, not name or hashtag. | +| **Canonical channel** | Constellation-scoped `MessageChannel` row (`protocol=MESHCORE`). Identity is **logical** (name / type / `region_scope`), not device index. | +| **Feeder mirror** | `ManagedNode.mc_channels` M2M **through** `ManagedNodeMcChannelLink`: which canonical channel sits in which device slot **for this feeder**. | +| **Snapshot** | Full list of configured slots from the device; reconcile **replaces** the feeder link rows to match (no partial merge). | +| **Source of truth** | **Device** for names and types; API mirror is overwritten on each successful sync. UI/admin **apply** writes the device first, then syncs back. | + +**Meshtastic contrast:** MT feeders map eight fixed slots via `meshtastic_channel_0..7` FKs on `ManagedNode`. MC feeders use the mirror + sync pipeline instead. + +--- + +## Architecture (overview) + +```mermaid +sequenceDiagram + participant Radio as MeshCore_companion + participant Bot as meshflow_bot + participant API as meshflow_api + participant UI as meshflow_ui + + Note over Radio,API: Default path — bot connect + Bot->>Radio: get_channel 0..N + Bot->>API: POST feeders/{prefix}/mc-channel-sync + API->>API: reconcile_mc_channels + + Note over UI,Radio: Operator edit + UI->>API: POST apply-mc-channel-config + API->>Bot: WS apply_mc_channel_config + Bot->>Radio: set_channel per slot + Bot->>API: POST mc-channel-sync +``` + +**Drift:** If apply fails or the API catalog is edited without a device write, the **next successful sync from the device** wins. There is no three-way merge. + +--- + +## Code anchors (meshflow-api) + +| Piece | Path | +|-------|------| +| Sync view | `Meshflow/meshcore_packets/views.py` — `MeshCoreMcChannelSyncView` | +| Apply view | `Meshflow/meshcore_packets/views.py` — `ManagedNodeMcChannelApplyView` | +| Reconcile | `Meshflow/meshcore_packets/services/channel_sync.py` — `reconcile_mc_channels` | +| Canonical upsert | `Meshflow/meshcore_packets/services/channel_identity.py` — `upsert_canonical_mc_channel` | +| Ingest resolution | `Meshflow/meshcore_packets/services/channel.py` — `resolve_mc_channel` | +| WS dispatch | `Meshflow/meshcore_packets/services/channel_apply.py` | +| Serializers | `Meshflow/meshcore_packets/serializers_channel.py` | +| URLs | `Meshflow/meshcore_packets/urls.py` | +| Models | `Meshflow/constellations/models.py`, `Meshflow/nodes/models.py` (`ManagedNodeMcChannelLink`) | +| Feeder auth | `Meshflow/common/meshcore_feeder_auth.py` | +| Apply labels | `Meshflow/common/mc_channel_labels.py` | + +**meshflow-bot:** `src/meshcore/channel_sync.py`, `src/meshcore/channels.py`, `src/bot.py` (`apply_mc_channel_config`). + +**meshflow-ui:** `MeshCoreChannelEditor` in `src/components/nodes/MeshCoreChannelEditor.tsx`, `src/lib/mc-channel-editor.ts`. + +--- + +## Consumers + +| Consumer | Use | +|----------|-----| +| **Packet ingest** | `resolve_mc_channel(observer, channel_idx)` → `MessageChannel` FK on `MeshCoreTextPacket` | +| **Text messages API** | `TextMessage.channel`; history filtered by canonical channel id | +| **Node Settings UI** | `GET` managed node `mc_channels` mirror; apply pushes to radio | +| **Django admin** | Read-only mirror on MC managed nodes; **Push MC channel config** action | +| **Constellation API** | Nested `channels` on constellation detail (MC catalog) | diff --git a/docs/features/meshcore/mc-channel-sync/data-model.md b/docs/features/meshcore/mc-channel-sync/data-model.md new file mode 100644 index 0000000..038c377 --- /dev/null +++ b/docs/features/meshcore/mc-channel-sync/data-model.md @@ -0,0 +1,109 @@ +# mc-channel-sync — data model + +Purpose: which database rows exist for MeshCore channels, and how snapshot entries map to them. + +Normative design: [ADR-0002](../../packet-ingestion/adr/0002-mc-channel-modelling.md). + +--- + +## Two layers + +```text +Constellation + └── MessageChannel (canonical, protocol=MESHCORE) + ▲ + │ FK message_channel +ManagedNode (feeder) ── ManagedNodeMcChannelLink ── mc_channel_idx (device slot) +``` + +| Layer | Model | Scoped by | Holds device index? | +|-------|--------|-----------|---------------------| +| **Canonical** | `MessageChannel` | `constellation` | No | +| **Feeder mirror** | `ManagedNodeMcChannelLink` | `managed_node` | Yes (`mc_channel_idx`, unique per feeder) | + +The same logical channel (e.g. hashtag `galloway`) is **one** `MessageChannel` row even when two feeders use different device indices. Each feeder has its own link row pointing at that canonical row. + +Meshtastic channels on the same table use `protocol=MESHTASTIC` and `ManagedNode.meshtastic_channel_0..7` — not the MC link table. + +--- + +## `MessageChannel` (canonical) + +Django: [`Meshflow/constellations/models.py`](../../../Meshflow/constellations/models.py) + +| Field | MC usage | +|-------|----------| +| `name` | Operator-facing label; for `HASHTAG`, typically the tag without `#` | +| `constellation` | FK | +| `protocol` | `MESHCORE` (2) | +| `mc_channel_type` | `PUBLIC` (1) or `HASHTAG` (2) | +| `region_scope` | Optional MeshCore region name (null = legacy / no scope); see [region-scope.md](region-scope.md) | + +**Uniqueness (MC rows):** `(constellation, protocol, name, mc_channel_type, region_scope)` — partial indexes for null vs non-null `region_scope`. + +`MeshCoreMessageChannel` is a **proxy model** for admin listing MC rows only. + +**Not stored on canonical rows:** `mc_channel_idx`, PSK, channel secret (MeshCore companion stores secrets on device; not mirrored in API v1). + +--- + +## `ManagedNodeMcChannelLink` + +Django: [`Meshflow/nodes/models.py`](../../../Meshflow/nodes/models.py) + +| Field | Description | +|-------|-------------| +| `managed_node` | MeshCore feeder | +| `message_channel` | Canonical `MessageChannel` | +| `mc_channel_idx` | Device slot 0–63 | + +Unique: `(managed_node, mc_channel_idx)`. + +`ManagedNode.mc_channels` is the M2M through this table. `ManagedNode.mc_channels_synced_at` is updated on every successful reconcile. + +--- + +## Snapshot → canonical mapping + +[`upsert_canonical_mc_channel`](../../../Meshflow/meshcore_packets/services/channel_identity.py): + +| `mc_channel_type` | Match key | `defaults` | +|-------------------|-----------|------------| +| `HASHTAG` | `(constellation, protocol, name, mc_channel_type, region_scope)` — `name` = tag without `#` | — | +| `PUBLIC` | same key shape — `name` = public channel name | — | + +`region_scope` normalized via `common/mc_region_scope.py` (`*`, empty → `NULL`). + +Invalid hashtag (empty after normalize) → `ValueError` → sync `400`. + +--- + +## Placeholder channels (ingest before sync) + +[`placeholder_canonical_mc_channel`](../../../Meshflow/meshcore_packets/services/channel_identity.py) creates: + +- `name`: `MC channel {idx}` +- `mc_channel_type`: `PUBLIC` +- Link row for `(observer, channel_idx)` + +Reconcile replaces the link’s `message_channel` when the device reports real metadata for that slot. Orphan placeholders may remain in the catalog if never configured on device. + +--- + +## API serialization surfaces + +| Surface | Serializer | Contents | +|---------|------------|----------| +| Sync request | `McChannelSnapshotEntrySerializer` | Device snapshot entries | +| Sync response / managed node | `FeederMcChannelMirrorSerializer` | Canonical `id`, `name`, `mc_channel_type`, `region_scope`, plus `mc_channel_idx` from link | +| Constellation nested | `message_channel_payload` in `constellations/serializers.py` | Includes `display_label` from `mc_channel_display_label` | +| Apply payload | `McChannelApplyEntrySerializer` | Same entry shape; validated hashtag rules | + +`display_label` is read-only on API responses (`#tag` for hashtags). + +--- + +## Related + +- [flow.md](flow.md) — reconcile algorithm and endpoints +- [text-message-channels.md](../text-message-channels.md) — `MeshCoreTextPacket.channel`, `TextMessage` diff --git a/docs/features/meshcore/mc-channel-sync/flow.md b/docs/features/meshcore/mc-channel-sync/flow.md new file mode 100644 index 0000000..7fe7dfe --- /dev/null +++ b/docs/features/meshcore/mc-channel-sync/flow.md @@ -0,0 +1,161 @@ +# mc-channel-sync — flow and API + +Purpose: how a device channel table becomes API state, and how operator edits return to the radio. + +--- + +## 1. Device → API (`mc-channel-sync`) + +### Trigger + +| When | Who calls | +|------|-----------| +| Bot connects to companion (after `SELF_INFO`) | `meshflow-bot` — `sync_channels_to_storage_apis_async` | +| After WebSocket `apply_mc_channel_config` succeeds | Bot re-syncs to **all** configured storage APIs | +| Django admin **Push MC channel config** | Same as apply (mirror payload), then bot re-syncs | + +The bot does **not** pull channel config from the API on startup. + +### Bot read path + +1. Wait `CHANNEL_READ_DELAY_S` (2 s) after connect so the channel table is stable. +2. For each index `0 .. max_channels-1` (default 16), `meshcore.commands.get_channel(idx)`. +3. Map `CHANNEL_INFO` to snapshot rows in [`channels.py`](https://github.com/pskillen/meshflow-bot/blob/main/src/meshcore/channels.py): + - Name starting with `#` → `mc_channel_type: HASHTAG`, `name` = tag without `#` + - Otherwise → `PUBLIC` + - `region_scope` when companion exposes it; otherwise `null` ([region-scope.md](region-scope.md)) +4. Build body: `{ "channels": [...], "synced_at": "" }`. +5. `POST` to each `StorageAPIWrapper` in `storage_apis` (primary + optional secondary). + +Empty named slots are omitted from the snapshot. If no channels are found, the bot logs a per-slot scan warning. + +### HTTP + +| | | +|--|--| +| **Method / path** | `POST /api/meshcore/feeders/{feeder_pubkey_prefix}/mc-channel-sync/` | +| **Auth** | `NodeAPIKeyAuthentication` + `MeshCoreFeederPermission` | +| **Feeder resolution** | URL **12-hex prefix** must match `pubkey_to_prefix(ManagedNode.mc_pubkey)`; optional `X-MeshCore-Feeder-Pubkey` (64 hex) must match when `mc_pubkey` is set | + +See [feeder-bootstrap.md](../feeder-bootstrap.md) for 403 codes (`feeder_not_linked`, `feeder_pubkey_mismatch`, etc.). + +**Request body** (`McChannelSyncSerializer`): + +| Field | Required | Description | +|-------|----------|-------------| +| `channels` | Yes | Array of snapshot entries | +| `synced_at` | No | Client timestamp; defaults to server now | + +**Each channel entry:** + +| Field | Required | Description | +|-------|----------|-------------| +| `mc_channel_idx` | Yes | Device slot, 0–63 | +| `name` | Yes | Channel name (hashtag channels: tag without leading `#` in stored canonical `name` where applicable) | +| `mc_channel_type` | Yes | `PUBLIC` or `HASHTAG` (plain strings, not gettext) | +| `region_scope` | No | MeshCore region name; `null` / `*` / empty = legacy scope | + +Example: + +```json +{ + "channels": [ + { "mc_channel_idx": 0, "name": "Public", "mc_channel_type": "PUBLIC", "region_scope": null }, + { "mc_channel_idx": 1, "name": "galloway", "mc_channel_type": "HASHTAG", "region_scope": "sample-west" } + ], + "synced_at": "2026-05-20T12:00:00Z" +} +``` + +**Response** (`200`): + +| Field | Description | +|-------|-------------| +| `status` | `"success"` | +| `synced_at` | `ManagedNode.mc_channels_synced_at` after reconcile | +| `mc_channels` | Feeder mirror: canonical fields + `mc_channel_idx` per link | + +**Errors:** `400` validation or `mc_channel_idx` out of range; `403` feeder auth. + +### API reconcile (`reconcile_mc_channels`) + +Transactional steps ([`channel_sync.py`](../../../Meshflow/meshcore_packets/services/channel_sync.py)): + +1. For each snapshot entry: `upsert_canonical_mc_channel(constellation, entry)` → `MessageChannel`. +2. `ManagedNodeMcChannelLink.update_or_create(managed_node, mc_channel_idx, message_channel=canonical)`. +3. Delete any link rows for this feeder whose `mc_channel_idx` was **not** in the snapshot. +4. Set `managed_node.mc_channels_synced_at` from `synced_at` or now. + +An **empty** `channels` array clears all feeder links (catalog rows may remain for other feeders or history). + +--- + +## 2. UI / admin → device (`apply-mc-channel-config`) + +### Trigger + +| When | Who | +|------|-----| +| Node Settings **Apply to radio** | Owner JWT | +| Django admin **Push MC channel config to feeder device** | Staff; uses current mirror via `build_apply_channels_for_managed_node` | + +### HTTP + +| | | +|--|--| +| **Method / path** | `POST /api/meshcore/managed-nodes/{internal_id}/apply-mc-channel-config/` | +| **Auth** | `IsAuthenticated`; managed node must belong to `request.user` | +| **Body** | `McChannelApplySerializer`: `{ "channels": [ ... ] }` — same entry shape as sync (may omit `mc_channel_idx` on entries that only update catalog intent; mirror apply includes indices) | + +**Responses:** + +| Status | Meaning | +|--------|---------| +| `200` | `apply_mc_channel_config` dispatched to bot group | +| `404` | Unknown or non-owned managed node | +| `503` `feeder_bot_not_connected` | No bot WebSocket on `node_mc_{internal_id}` Redis group | +| `503` `command_dispatch_unavailable` | Channel layer / Redis unavailable | + +### WebSocket command + +Dispatched via Django Channels (`dispatch_node_command`) to group `node_mc_{managed_node.internal_id}`: + +```json +{ + "type": "apply_mc_channel_config", + "channels": [ { "mc_channel_idx": 0, "name": "...", "mc_channel_type": "PUBLIC", "region_scope": null } ] +} +``` + +**meshflow-bot** (`MeshflowBot` handler): + +1. `apply_device_channels` — `set_channel(idx, name)` per row; hashtag names written with `#` prefix on device. +2. Re-run channel sync to all `storage_apis`. + +Bot WebSocket connects when `STORAGE_API_ROOT` + token are set; URL is derived from API base unless `MESHFLOW_WS_URL` is set. Feeder **12-hex prefix** is appended on connect. + +### Horizontal scaling + +Apply uses **Redis DB 0** channel layer ([`docs/REDIS.md`](../../../REDIS.md)), not in-process signals. Any API worker can `group_send`; the bot must share that Redis and register on the feeder group. Details: [operations.md](operations.md). + +--- + +## 3. Ingest coupling (read-only here) + +When `channel_text` arrives before the first sync, [`resolve_mc_channel`](../../../Meshflow/meshcore_packets/services/channel.py): + +1. Clamps `channel_idx` to 0–63. +2. Looks up `ManagedNodeMcChannelLink` for `(observer, channel_idx)`. +3. If missing: creates placeholder canonical `MessageChannel` (`MC channel N`) + link. + +The next **mc-channel-sync** overwrites placeholder metadata from the device. Multiple feeders with the same logical hashtag at **different** indices share one canonical row after sync ([ADR-0002](../../packet-ingestion/adr/0002-mc-channel-modelling.md)). + +Text dedup keys include `message_channel_id`, so canonical identity must be stable across feeders. + +--- + +## Related + +- [data-model.md](data-model.md) — fields and uniqueness +- [operations.md](operations.md) — dual API, local dev 503s +- [text-message-channels.md](../text-message-channels.md) — wire format and `TextMessage` pipeline diff --git a/docs/features/meshcore/mc-channel-sync/operations.md b/docs/features/meshcore/mc-channel-sync/operations.md new file mode 100644 index 0000000..be459e5 --- /dev/null +++ b/docs/features/meshcore/mc-channel-sync/operations.md @@ -0,0 +1,121 @@ +# mc-channel-sync — operations + +Purpose: authentication, multi-API deployments, scaling, and common failure modes. Timeless reference for operators and developers. + +--- + +## Authentication + +| Direction | Auth | Feeder identity | +|-----------|------|-----------------| +| Device → API (`mc-channel-sync`) | Node API key (`NodeAuth`) | URL `{feeder_pubkey_prefix}` + optional `X-MeshCore-Feeder-Pubkey` | +| UI → API (`apply-mc-channel-config`) | User JWT | Managed node `owner` | +| API → bot (WebSocket) | Bot connected with same API key / feeder group | `node_mc_{internal_id}` | + +Shared constellation API keys are supported: each `ManagedNode` must have a distinct `mc_pubkey`; the URL prefix disambiguates the observer. + +See [feeder-bootstrap.md](../feeder-bootstrap.md) for bootstrap and 403 JSON codes. + +--- + +## meshflow-bot environment + +| Variable | Role | +|----------|------| +| `RADIO_PROTOCOL=meshcore` | MeshCore radio + channel sync | +| `MESHCORE_UPLOAD_ENABLED=true` | POST sync (and packets); otherwise capture-only | +| `STORAGE_API_ROOT` / `STORAGE_API_TOKEN` | Primary API; WebSocket unless `MESHFLOW_WS_URL` set | +| `STORAGE_API_2_ROOT` / `STORAGE_API_2_TOKEN` | Optional second POST target for sync + ingest | +| `MESHCORE_SERIAL_DEVICE` or `MESHCORE_BLE_ADDRESS` | Companion transport | + +Full list: [meshflow-bot `docs/MESHCORE.md`](https://github.com/pskillen/meshflow-bot/blob/main/docs/MESHCORE.md). + +--- + +## Dual API (`STORAGE_API_2_*`) + +When the bot uploads to two API bases: + +| Traffic | Primary (`STORAGE_API_ROOT`) | Secondary (`STORAGE_API_2_ROOT`) | +|---------|------------------------------|----------------------------------| +| `mc-channel-sync` | Yes | Yes (same snapshot after connect / apply) | +| WebSocket / `apply_mc_channel_config` | Yes | No (unless `MESHFLOW_WS_URL` points at secondary) | +| UI apply against secondary only | — | **503** — bot not on that deployment’s WS groups | + +Operators should use one deployment for UI + bot WS, or accept mirror-only updates on the secondary. + +--- + +## Horizontal scaling (API + bot) + +| Component | Requirement | +|-----------|-------------| +| API workers | `channels_redis` on Redis DB 0 ([`docs/REDIS.md`](../../../REDIS.md)) | +| Bot | Single WebSocket per feeder; any ASGI instance that shares Redis | +| Presence check | `feeder_ws_group_has_subscribers` probes Redis ZSET for `asgi:group:node_mc_{uuid}` | + +In-memory channel layer (tests only) breaks apply from real workers. + +--- + +## Troubleshooting + +### Apply returns 503 `feeder_bot_not_connected` + +| Cause | What to check | +|-------|----------------| +| Bot not running or WS down | Bot logs: `MeshflowWSClient: connected` | +| UI and bot on different API hosts | Align `MESHFLOW_API_URL` (UI) with bot `STORAGE_API_ROOT` | +| Redis mismatch | Same `REDIS_HOST` / DB 0 as pre-prod when mixing local + remote | +| Wrong feeder | `mc_pubkey` on `ManagedNode` matches device; WS URL includes correct 12-hex prefix | + +### Apply returns 503 `command_dispatch_unavailable` + +Channel layer or Redis error during `group_send`. Check API logs and Redis connectivity. + +### Sync succeeds but UI mirror empty + +| Cause | What to check | +|-------|----------------| +| Device has no named channels | Bot warning: zero channels from `get_channel` scan | +| Wrong feeder prefix | 403 on sync if prefix does not match `mc_pubkey` | +| Empty snapshot accepted | Reconcile with `channels: []` clears all links | + +### API catalog differs from radio + +Expected until next successful sync from device. **Device wins** on sync. Edit via apply-to-radio, not by changing canonical rows alone in admin (admin catalog edits do not push automatically). + +### Placeholder `MC channel N` in messages + +Ingest arrived before first sync for that slot. Run bot connect or fix device config and re-sync. + +--- + +## Verification checklist + +1. Bot log: `POST /api/meshcore/feeders/{prefix}/mc-channel-sync/` with channel count. +2. `ManagedNode.mc_channels_synced_at` updated in admin or `GET /api/nodes/managed-nodes/mine/`. +3. Nested `mc_channels` on managed node matches device slot order and names. +4. After apply: bot log shows `set_channel` then second sync POST. +5. `channel_message` ingest: `MeshCoreTextPacket.channel` FK resolves to expected canonical row. + +--- + +## Known gaps + +| Gap | Notes | +|-----|--------| +| Per-channel **region scope** in sync payload | Planned [#391](https://github.com/pskillen/meshflow-api/issues/391); companion `GET_CHANNEL` does not return scope today | +| API-only mirror edits | No REST to patch mirror without device; use apply or wait for connect sync | +| Empty device channel table | Ops: configure channels on companion or via MeshCore app first | +| `region_scope` / transport code on ingest | Out of scope for channel sync; wire observation ≠ feeder config | + +Track execution debt in phase/outstanding docs only when discovered during delivery — not duplicated here. + +--- + +## Related + +- [flow.md](flow.md) +- [feeder-bootstrap.md](../feeder-bootstrap.md) +- [API keys & WebSocket](../../../API_KEYS.md) diff --git a/docs/features/meshcore/mc-channel-sync/region-scope-outstanding.md b/docs/features/meshcore/mc-channel-sync/region-scope-outstanding.md new file mode 100644 index 0000000..224969f --- /dev/null +++ b/docs/features/meshcore/mc-channel-sync/region-scope-outstanding.md @@ -0,0 +1,14 @@ +# MeshCore region scope — outstanding + +Items **skipped**, **incomplete**, or **discovered during execution** for [#391](https://github.com/pskillen/meshflow-api/issues/391) — not the plan backlog. + +**Tracking:** [region-scope-progress.md](./region-scope-progress.md) + +--- + +## Protocol / device + +- [ ] Per-channel `region_scope` in companion `CHANNEL_INFO` — not on wire today; bot may sync `null` until meshcore_py/firmware expose it +- [ ] Apply path: confirm whether `set_flood_scope` is per-channel or global on hardware (spike on real feeder) +- [x] Bot merges `region_scope` from apply payload into post-apply sync (CHANNEL_INFO does not return scope) +- [ ] Per-channel scope in `CHANNEL_INFO` / extended `SET_CHANNEL` — firmware still name+secret only; active scope is `set_flood_scope` (global) per slot during apply diff --git a/docs/features/meshcore/mc-channel-sync/region-scope-progress.md b/docs/features/meshcore/mc-channel-sync/region-scope-progress.md new file mode 100644 index 0000000..09294fd --- /dev/null +++ b/docs/features/meshcore/mc-channel-sync/region-scope-progress.md @@ -0,0 +1,61 @@ +# MeshCore region scope — progress + +**Tracking:** [meshflow-api#391](https://github.com/pskillen/meshflow-api/issues/391) +**Plan:** `.cursor/plans/meshcore_region_scopes_db255499.plan.md` +**Repos:** meshflow-api, meshflow-bot, meshflow-ui + +--- + +## Overall status + +**Status:** Ready for PRs + +**Branch (API):** `api-391/pskillen/region-scope` (from `api-391/pskillen/mc-channel-sync-docs`) + +--- + +## Documentation + +**Status:** Done + +- `region-scope.md`, progress/outstanding pair, hub doc map +- `data-model.md`, `flow.md`, ADR-0002 amended + +--- + +## API — schema and identity + +**Status:** Done + +- Migration `0015_messagechannel_region_scope`, `region_scope` field, drop `mc_hashtag` +- `channel_identity`, `mc_region_scope`, labels, tests + +--- + +## API — contract and timeless docs + +**Status:** Done + +- Serializers, `openapi.yaml`, admin + +--- + +## meshflow-bot + +**Status:** Done (branch `bot-391/pskillen/region-scope`) + +- `channels.py` read/apply `region_scope`; tests updated + +--- + +## meshflow-ui + +**Status:** Done (branch `ui-391/pskillen/region-scope`) + +- Editor, message labels, models, tests + +--- + +## Next + +- Atomic commits + PRs (api, bot, ui) via github-personal MCP diff --git a/docs/features/meshcore/mc-channel-sync/region-scope.md b/docs/features/meshcore/mc-channel-sync/region-scope.md new file mode 100644 index 0000000..a2f5619 --- /dev/null +++ b/docs/features/meshcore/mc-channel-sync/region-scope.md @@ -0,0 +1,54 @@ +# MeshCore region scope + +**Tracking:** [meshflow-api#391](https://github.com/pskillen/meshflow-api/issues/391) + +MeshCore **region filtering** limits which flood traffic a node forwards. Operators assign a **scope name** per channel (or use null scope for legacy “hear everything” behaviour). Meshflow stores scope on the **canonical** `MessageChannel` row so the same hashtag in different regions is distinct for ingest, history, and UI. + +**Background:** [Region filtering](https://blog.meshcore.io/2026/01/20/region-filtering) · [Default scope](https://blog.meshcore.io/2026/04/17/default-scope) + +--- + +## Naming rules + +| Rule | Detail | +|------|--------| +| Characters | Lowercase letters, digits, hyphen (`a-z`, `0-9`, `-`) | +| Length | Max **29 UTF-8 bytes** | +| Null scope | `*`, empty string, or `none` → stored as `NULL` (legacy / no scope) | +| Wire `#` | Do **not** store leading `#` on `region_scope`; hashtags use `name` without `#` | + +--- + +## API field + +On `MessageChannel` when `protocol=MESHCORE`: + +- **`region_scope`** — optional `CharField(max_length=29)`, nullable +- **`name`** — for `HASHTAG`, the tag without `#`; for `PUBLIC`, the public channel name +- **Uniqueness** — `(constellation, protocol, name, mc_channel_type, region_scope)` with separate partial indexes for null vs non-null scope + +Sync and apply payloads include `region_scope` on each entry ([flow.md](flow.md)). Normalization lives in `Meshflow/common/mc_region_scope.py`. + +--- + +## UI and labels + +Display helpers append scope when set, e.g. `#galloway · sample-west` (`Meshflow/common/mc_channel_labels.py`). Node Settings channel editor validates scope like MeshCore rules; Messages channel picker uses the same labels. + +--- + +## Bot / device + +On **read**, the bot includes `region_scope` when `CHANNEL_INFO` exposes it. Today the response is **name + secret only**, so: + +- After **apply**, the bot re-reads the device and **merges `region_scope` from the apply payload** by `mc_channel_idx` before posting `mc-channel-sync` (avoids duplicate unscoped canonical rows). +- On **connect** sync, the API may **preserve** scope from the feeder’s existing slot link when the snapshot omits scope but name/type still match. + +On **apply**, after each `set_channel`, the bot calls `meshcore.commands.set_flood_scope` for that row’s scope (companion **active** flood scope; only the last slot’s scope remains active on the radio until the operator changes channel in the MeshCore app). + +--- + +## Related + +- [data-model.md](data-model.md) — canonical channel identity +- [ADR-0002](../../packet-ingestion/adr/0002-mc-channel-modelling.md) — normative modelling diff --git a/docs/features/meshcore/phase-2-outstanding.md b/docs/features/meshcore/phase-2-outstanding.md index 5ec4f32..f87a3c2 100644 --- a/docs/features/meshcore/phase-2-outstanding.md +++ b/docs/features/meshcore/phase-2-outstanding.md @@ -103,7 +103,7 @@ Follow-up after local UI + pre-prod bot + shared Postgres/Redis. Tracked on [#29 - [ ] **Auto-set `mc_pubkey` on first connect** — still manual in admin ([#279](https://github.com/pskillen/meshflow-api/issues/279)). - [ ] **OpenAPI** — confirm all MC feeder paths and apply responses match deployed code after #335 merge (code was ahead of spec in places during staging). - [ ] **Admin push action** — pushes the feeder’s **linked** channel list (`ManagedNodeMcChannelLink` order/slots); editing constellation **MeshCore channels** admin does not auto-assign links or push (use Node Settings / bot sync). -- [ ] **Dual API (`STORAGE_API_2_*`)** — bot POSTs `mc-channel-sync` (and packets) to **both** APIs when upload enabled; **WebSocket / apply** only on primary `STORAGE_API_ROOT`. Documented in [text-message-channels.md](./text-message-channels.md); UI apply against API 2 while bot WS on API 1 will always 503. +- [ ] **Dual API (`STORAGE_API_2_*`)** — bot POSTs `mc-channel-sync` (and packets) to **both** APIs when upload enabled; **WebSocket / apply** only on primary `STORAGE_API_ROOT`. Documented in [mc-channel-sync/operations.md](./mc-channel-sync/operations.md); UI apply against API 2 while bot WS on API 1 will always 503. ### Intentional / by design (document only) diff --git a/docs/features/meshcore/text-message-channels.md b/docs/features/meshcore/text-message-channels.md index 9590451..1c6a6c7 100644 --- a/docs/features/meshcore/text-message-channels.md +++ b/docs/features/meshcore/text-message-channels.md @@ -11,7 +11,7 @@ How MeshCore **group text** and **channel configuration** flow from the radio th **Normative ADRs:** [ADR-0002](../packet-ingestion/adr/0002-mc-channel-modelling.md) (channels), [ADR-0003](../packet-ingestion/adr/0003-mc-broadcast-semantics.md) (broadcast vs DM), [ADR-0001](../packet-ingestion/adr/0001-mc-node-identity.md) (sender identity on text). -**Related:** [feeder-bootstrap.md](feeder-bootstrap.md), [README.md](README.md) (phase docs), [MESHCORE_PACKET_FIELDS.md](../packet-ingestion/MESHCORE_PACKET_FIELDS.md), [message-sender.md](message-sender.md) (channel `Name: body` sender inference). +**Related:** [mc-channel-sync/](mc-channel-sync/) (feeder channel mirror, apply-to-radio), [feeder-bootstrap.md](feeder-bootstrap.md), [README.md](README.md) (phase docs), [MESHCORE_PACKET_FIELDS.md](../packet-ingestion/MESHCORE_PACKET_FIELDS.md), [message-sender.md](message-sender.md) (channel `Name: body` sender inference). --- @@ -52,76 +52,11 @@ sequenceDiagram API-->>TM: create TextMessage row ``` -### Configuration path ([#297](https://github.com/pskillen/meshflow-api/issues/297)) +### Channel configuration -The **MeshCore device / companion** is the **source of truth** for channel names, types (public / hashtag), and indices. The API holds a **mirror** per feeder (`ManagedNode.mc_channels`). On every bot connect the bot reads the device and **uploads a full snapshot**; the API reconciles its own state. Operator edits in the UI go **to the device first** (WebSocket), then the bot re-syncs so the API matches the radio again. +Feeder channel mirror, `mc-channel-sync`, apply-to-radio, scaling, and troubleshooting are documented in **[mc-channel-sync/](mc-channel-sync/)** ([#297](https://github.com/pskillen/meshflow-api/issues/297)). -**Feeder identity:** ingest, channel sync, and bot version use **`/api/meshcore/feeders/{feeder_pubkey_prefix}/…`** (12-hex prefix from device pubkey). See [feeder-bootstrap.md](feeder-bootstrap.md) ([#295](https://github.com/pskillen/meshflow-api/issues/295)). - -```mermaid -sequenceDiagram - participant Radio as MeshCore_device - participant Bot as meshflow_bot - participant API as meshflow_api - participant UI as meshflow_ui - - Note over Radio,API: On bot start - Bot->>Radio: read channel table - Bot->>API: POST …/feeders/{prefix}/mc-channel-sync - API->>API: reconcile MessageChannel + mc_channels - - Note over UI,Radio: Optional operator edit - UI->>API: apply to radio (WS) - API->>Bot: apply_mc_channel_config - Bot->>Radio: write channels - Bot->>API: POST …/feeders/{prefix}/mc-channel-sync -``` - -**Drift:** if the API mirror and device disagree (e.g. failed push), the **next connect sync overwrites the API** from the device. No three-way merge in v1. - -The bot starts **`MeshflowWSClient`** for MeshCore when `STORAGE_API_ROOT` + token are set (WebSocket URL derived from the API base URL unless `MESHFLOW_WS_URL` is set). The feeder’s 12-hex pubkey prefix is appended automatically on connect (not an operator env var). - -### Horizontal scaling (API web tier) - -Apply and traceroute commands use **Django Channels + Redis DB 0** ([`docs/REDIS.md`](../../REDIS.md)), not in-process Django signals. Signals only run inside one Python process and do not reach other workers or hosts. - -| Piece | Scales horizontally? | -|-------|----------------------| -| `POST apply-mc-channel-config` on any Gunicorn/Uvicorn worker | Yes — handler calls `channel_layer.group_send` via Redis | -| `feeder_ws_group_has_subscribers` on any worker | Yes — `channels_redis` stores group membership in Redis DB 0 | -| Bot `NodeConsumer` WebSocket | One connection per bot; must land on **some** ASGI instance in the deployment that shares that Redis | -| Browser UI WebSocket (`text_messages`, JWT) | Separate consumer/groups; unrelated to feeder commands | - -You do **not** need a separate “signal” bus for multi-worker API: Redis channel layer already is that bus. - -### Local API + pre-prod bot (common 503 cause) - -Sharing **Postgres** and **Redis** is not enough if the **HTTP hosts differ**: - -- UI / `apply-mc-channel-config` → `http://localhost:8000` (local Django) -- Bot → `STORAGE_API_ROOT=https://pre-prod…/api` and WebSocket to **pre-prod** - -The bot registers in Redis group `node_mc_{uuid}` through **pre-prod’s** `NodeConsumer`. Local Django also reads Redis DB 0, so presence *can* work if both use the same `REDIS_HOST` / password and the managed node `internal_id` + `mc_pubkey` match the device. If local settings use **`InMemoryChannelLayer`** (tests only) or a different Redis DB/host, local apply always sees **503 feeder not connected** even though pre-prod logs show `MeshflowWSClient: connected`. - -**Practical dev setups (pick one):** - -1. Point **meshflow-ui** at pre-prod API (browser and bot share one deployment), or -2. Point **bot** `STORAGE_API_ROOT` at local API and run bot + API locally, or -3. Keep split hosts but verify local `CHANNEL_LAYERS` is `channels_redis` to the **same** Redis DB 0 as pre-prod. - -**Common 503 causes (fixed on api #335 / bot #108 branch):** - -| Symptom | Cause | Fix | -|---------|--------|-----| -| 503 `feeder_bot_not_connected` but Redis has `asgi:group:node_mc_{uuid}` | Presence check called `group_channels()` (not in channels_redis 4.x) | ZSET probe on `asgi:group:…` | -| 503 `command_dispatch_unavailable`, log `__proxy__` | gettext labels in apply payload | Plain `PUBLIC`/`HASHTAG` + `_ws_json_safe()` | -| Apply works on pre-prod UI but not localhost | Bot WS registered on pre-prod only | Align UI API URL with bot `STORAGE_API_ROOT` or shared Redis + local `channels_redis` | - -**Dual API (`STORAGE_API_2_*`):** with `MESHCORE_UPLOAD_ENABLED`, the bot POSTs **`mc-channel-sync`** (and packets) to **both** `STORAGE_API_ROOT` and `STORAGE_API_2_ROOT`. **WebSocket** and **apply** use **primary** only (`STORAGE_API_ROOT` / `MESHFLOW_WS_URL`). API 2 gets mirror updates on connect; UI apply against API 2 will 503 unless WS is also on API 2. - -**Django admin:** read-only device mirror on MeshCore managed nodes; **MeshCore channels** admin for constellation catalog; **Push MC channel config to feeder device** action (see [feeder-bootstrap.md](feeder-bootstrap.md) §5). - -**Implementation plan:** Cursor workspace plan `mc_text_textmessage_pipeline_2c3e9fb8.plan.md` (historical); **this doc is canonical** in git for operators and PRs. +Brief mental model: the **companion device** is source of truth for names, types, and slot indices; the API holds a per-feeder mirror plus constellation-scoped canonical `MessageChannel` rows for ingest and UI. Ingest, channel sync, and bot version share **`/api/meshcore/feeders/{feeder_pubkey_prefix}/…`** (see [feeder-bootstrap.md](feeder-bootstrap.md)). --- @@ -194,66 +129,13 @@ Example ingest body (channel text, illustrative): } ``` -### Bot channel configuration - -| Step | Behaviour | -|------|-----------| -| **On connect** | `read_device_channels` via `meshcore.commands.get_channel`; `POST` snapshot to **`/api/meshcore/feeders/{prefix}/mc-channel-sync/`** for each entry in `bot.storage_apis` (primary + optional API 2). | -| **On UI / admin “apply”** | WebSocket `apply_mc_channel_config` on **primary** WS only → `set_channel` on device → re-sync to **all** `storage_apis`. | -| **Channel types (v1)** | **PUBLIC** and **HASHTAG** only. | - -The bot does **not** pull API channel config and push to device on startup. If the device reports zero named channels, logs include a per-slot scan warning ([#107](https://github.com/pskillen/meshflow-bot/pull/107)) — still an open ops issue when other tools show channels on the same radio. - -**Example sync payload** (illustrative): - -```json -{ - "channels": [ - { "mc_channel_idx": 0, "name": "Public", "mc_channel_type": "PUBLIC", "mc_hashtag": null }, - { "mc_channel_idx": 1, "name": "Galloway", "mc_channel_type": "HASHTAG", "mc_hashtag": "galloway" } - ], - "synced_at": "2026-05-20T12:00:00Z" -} -``` +Channel sync and apply behaviour: [mc-channel-sync/flow.md](mc-channel-sync/flow.md). The bot does **not** pull API channel config on startup. --- ## meshflow-api — data models -### `MessageChannel` (constellation-scoped) - -[`Meshflow/constellations/models.py`](../../../Meshflow/constellations/models.py) - -| Field | Role | -|-------|------| -| `name` | Operator-facing label (UI/admin); not on wire | -| `constellation` | FK | -| `protocol` | `MESHCORE` for MC rows | -| `mc_channel_type` | `PUBLIC` / `HASHTAG` | -| `mc_hashtag` | Hashtag string when type is `HASHTAG` (no leading `#` in DB); unique per constellation for HASHTAG | -| *(no `mc_channel_idx`)* | Logical identity is name/hashtag, not device slot ([#379](https://github.com/pskillen/meshflow-api/issues/379)) | - -Meshtastic channels use PSK-backed slots on the managed node (`meshtastic_channel_0..7`). MeshCore does **not** use those slots. - -### `ManagedNodeMcChannelLink` (per-feeder device slot) - -| Field | Role | -|-------|------| -| `managed_node` | Feeder | -| `message_channel` | Canonical `MessageChannel` for this slot’s logical channel | -| `mc_channel_idx` | Device slot `0..63` (unique per managed node) | - -### `ManagedNode` (feeder) - -| Field | Role | -|-------|------| -| `protocol` | `MESHCORE` for MC feeders | -| `mc_pubkey` | Full 64-hex feeder identity ([#295](https://github.com/pskillen/meshflow-api/issues/295)) | -| `meshtastic_channel_0..7` | Meshtastic only | -| `mc_channels` | M2M to canonical channels **through** `ManagedNodeMcChannelLink` | -| `mc_channels_synced_at` | Last successful bot `mc-channel-sync` | - -Authentication: Node API key via `NodeAuth`; feeder resolved by URL **`{feeder_pubkey_prefix}`** + optional **`X-MeshCore-Feeder-Pubkey`** header ([feeder-bootstrap.md](feeder-bootstrap.md)). +Channel catalogue and feeder mirror: [mc-channel-sync/data-model.md](mc-channel-sync/data-model.md). Below focuses on **text ingest** models. ### Raw ingest: `MeshCoreTextPacket` @@ -322,34 +204,12 @@ Identity receiver **skips** channel text (no `from_pubkey` / prefix). Contact te --- -## meshflow-api — channel mirror and WebSocket - -### Primary: `POST …/feeders/{prefix}/mc-channel-sync` (device → API) - -- **Caller:** meshflow-bot with Node API key, after reading the device. -- **Body:** full channel list (`mc_channel_idx`, `name`, `mc_channel_type`, `mc_hashtag`); optional `synced_at`. -- **Effect:** upsert canonical `MessageChannel` rows by logical identity; set `ManagedNodeMcChannelLink` rows to match snapshot ([`reconcile_mc_channels`](../../../Meshflow/meshcore_packets/services/channel_sync.py)). -- **Read:** managed node API returns nested `mc_channels` for UI. - -### Secondary: push UI / admin intent → device - -- **REST:** `POST /api/meshcore/managed-nodes/{internal_id}/apply-mc-channel-config/` (owner JWT) → [`channel_apply`](../../../Meshflow/meshcore_packets/services/channel_apply.py) → Redis `group_send` → bot WS. -- **WebSocket command:** `apply_mc_channel_config` with `channels` array (plain string types). -- **Django admin:** action **Push MC channel config to feeder device** uses the same dispatch with mirror payload. -- Bot writes device, then re-syncs to all configured storage APIs. - -**Permissions:** sync uses feeder key + prefix; apply uses authenticated node owner. - -**Not v1:** API-only mirror edits without device round-trip (drift until next connect sync). - ---- - ## meshflow-ui - **Meshtastic feeders:** [Node Settings](https://github.com/pskillen/meshflow-ui/blob/main/src/pages/user/NodeSettings.tsx) — slots 0–7 → `meshtastic_channel_*`. -- **MeshCore feeders:** mirror from API; **Apply to radio** via `apply-mc-channel-config` (PUBLIC / HASHTAG, hashtag). Toasts on 503; richer offline UX still deferred. +- **MeshCore feeders:** channel editor and **Apply to radio** — see [mc-channel-sync/README.md](mc-channel-sync/README.md#consumers). -**Message history UI** for MC remains deferred; API supports `GET /api/messages/text/?protocol=meshcore` for channel broadcast. +**Message history UI** for MC: API supports `GET /api/messages/text/?protocol=meshcore` for channel broadcast. --- @@ -369,23 +229,17 @@ Identity receiver **skips** channel text (no `from_pubkey` / prefix). Contact te | Capability | Status | |------------|--------| | Bot upload `channel_text` / `contact_text` | **Done** | -| Feeder-scoped ingest + `mc-channel-sync` URLs ([#295](https://github.com/pskillen/meshflow-api/issues/295)) | **Done** (main + #335 fixes) | -| `ManagedNode.mc_channels` + `mc_channel_type` / `mc_hashtag` | **Done** | +| Feeder-scoped ingest ([#295](https://github.com/pskillen/meshflow-api/issues/295)) | **Done** | | `TextMessage` + MC provenance ([#296](https://github.com/pskillen/meshflow-api/issues/296)) | **Done** | -| Bot sync + WS apply ([#297](https://github.com/pskillen/meshflow-api/issues/297)) | **Done** | -| UI mirror + apply-to-radio | **Done** | -| Canonical channels + per-feeder links ([#379](https://github.com/pskillen/meshflow-api/issues/379)) | **Done** — [api #380](https://github.com/pskillen/meshflow-api/pull/380), [ui #313](https://github.com/pskillen/meshflow-ui/pull/313) | -| Apply 503 / msgpack / WS group fixes | **Done** on [api #335](https://github.com/pskillen/meshflow-api/pull/335), [bot #108](https://github.com/pskillen/meshflow-bot/pull/108) (merge pending) | -| Django admin MC channels + push action | **Done** (#335) | -| Dual API channel sync (no WS on API 2) | **Done** (bot behaviour; documented) | -| Empty device channel table / auto `mc_pubkey` | **Outstanding** — [phase-2-outstanding.md](./phase-2-outstanding.md) | +| Channel sync + apply ([#297](https://github.com/pskillen/meshflow-api/issues/297), [#379](https://github.com/pskillen/meshflow-api/issues/379)) | **Done** — see [mc-channel-sync/](mc-channel-sync/) | | MC message history in UI | **Deferred** | -| Three-way merge / periodic sync | **Deferred** | +| Empty device channel table / auto `mc_pubkey` | **Outstanding** — [phase-2-outstanding.md](./phase-2-outstanding.md) | --- ## References +- [mc-channel-sync/](mc-channel-sync/) — feeder channel mirror and apply - [ADR-0002 — MC channel modelling](../packet-ingestion/adr/0002-mc-channel-modelling.md) - [ADR-0003 — MC broadcast semantics](../packet-ingestion/adr/0003-mc-broadcast-semantics.md) - [ADR-0001 — MC node identity](../packet-ingestion/adr/0001-mc-node-identity.md) diff --git a/docs/features/packet-ingestion/adr/0002-mc-channel-modelling.md b/docs/features/packet-ingestion/adr/0002-mc-channel-modelling.md index b2b271e..59dbd95 100644 --- a/docs/features/packet-ingestion/adr/0002-mc-channel-modelling.md +++ b/docs/features/packet-ingestion/adr/0002-mc-channel-modelling.md @@ -1,6 +1,6 @@ # ADR-0002 — MeshCore channel modelling -**Status:** Accepted (amended 2026-06-01) +**Status:** Accepted (amended 2026-06-03) **Date:** 2026-05-12 **Tracking:** [meshflow-api#276](https://github.com/pskillen/meshflow-api/issues/276), [meshflow-api#379](https://github.com/pskillen/meshflow-api/issues/379) @@ -25,9 +25,10 @@ We still need a way to: 1. **Add `MessageChannel.protocol`** (same enum as `ObservedNode.protocol`). Channels are single-protocol; an MT channel and an MC channel are distinct rows even if their names happen to match. 2. **Canonical `MessageChannel` rows (constellation-scoped logical identity):** - - `name`, `mc_channel_type` (`PUBLIC` / `HASHTAG`), `mc_hashtag` (when HASHTAG). + - `name`, `mc_channel_type` (`PUBLIC` / `HASHTAG`), optional `region_scope` (MeshCore region filter; null = legacy scope). + - For `HASHTAG`, `name` is the tag **without** `#`; display adds `#` in UI helpers. - **No `mc_channel_idx` on `MessageChannel`.** Device slot index is per-feeder, not global. - - Uniqueness: `(constellation, protocol, mc_hashtag)` for HASHTAG rows; `(constellation, protocol, name)` for PUBLIC rows (normalized in services). + - Uniqueness: `(constellation, protocol, name, mc_channel_type, region_scope)` with partial indexes for null vs non-null scope (normalized in services). See [region-scope.md](../../meshcore/mc-channel-sync/region-scope.md). 3. **Defer `mc_channel_hash`.** Revisit if/when a firmware revision starts emitting one on the wire. No column added now. 4. **Per-feeder slot mapping via `ManagedNodeMcChannelLink` (M2M through table):** - `managed_node`, `message_channel` (canonical), `mc_channel_idx` (`0..63`, unique per managed node). diff --git a/openapi.yaml b/openapi.yaml index 6cfb4c0..413c2b4 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -2972,10 +2972,13 @@ components: nullable: true enum: [PUBLIC, HASHTAG] description: MeshCore channel type when protocol is MeshCore - mc_hashtag: + region_scope: type: string nullable: true - description: Hashtag string when mc_channel_type is HASHTAG (no leading #) + maxLength: 29 + description: > + MeshCore region scope (lowercase alphanumeric + hyphen; null = legacy / no scope). + HASHTAG rows use name for the tag without #. constellation: type: integer description: The ID of the constellation this channel belongs to @@ -2998,9 +3001,10 @@ components: type: string nullable: true enum: [PUBLIC, HASHTAG] - mc_hashtag: + region_scope: type: string nullable: true + maxLength: 29 McChannelSyncEntry: type: object @@ -3015,9 +3019,10 @@ components: mc_channel_type: type: string enum: [PUBLIC, HASHTAG] - mc_hashtag: + region_scope: type: string nullable: true + maxLength: 29 McChannelSyncRequest: type: object