diff --git a/core/ipc_handler.go b/core/ipc_handler.go index f687b24b..3e22f93b 100644 --- a/core/ipc_handler.go +++ b/core/ipc_handler.go @@ -212,14 +212,20 @@ func buildEndpoints(neigh *state.Neighbour) []*protocol.EndpointInfo { if ap, err := nep.DynEP.Get(); err == nil { resolved = stringPtr(ap.String()) } + bindSource := "" + if nep.Bind.Source.IsValid() { + bindSource = nep.Bind.Source.String() + } eps = append(eps, &protocol.EndpointInfo{ - Address: nep.DynEP.Value, - Resolved: resolved, - Active: ep.IsActive(), - RemoteInit: ep.IsRemote(), - Metric: ep.Metric(), - FilteredRttNs: int64(nep.FilteredPing()), - StabilizedRttNs: int64(nep.StabilizedPing()), + Address: nep.DynEP.Value, + Resolved: resolved, + Active: ep.IsActive(), + RemoteInit: ep.IsRemote(), + Metric: ep.Metric(), + FilteredRttNs: int64(nep.FilteredPing()), + StabilizedRttNs: int64(nep.StabilizedPing()), + LocalBindInterface: nep.Bind.Interface, + LocalBindSource: bindSource, }) } slices.SortFunc(eps, func(a, b *protocol.EndpointInfo) int { diff --git a/core/nylon_apply.go b/core/nylon_apply.go index b5ad501b..9f7041f5 100644 --- a/core/nylon_apply.go +++ b/core/nylon_apply.go @@ -70,7 +70,7 @@ func (n *Nylon) reconcileRouterState(next *state.CentralCfg) error { continue } // configure existing neighbours - reconcileConfiguredEndpoints(neigh, cfg.Endpoints, &n.RouterTunables) + reconcileConfiguredEndpoints(neigh, configuredEndpoints(n.LocalCfg.Binds, cfg.Endpoints, &n.RouterTunables)) neighs = append(neighs, neigh) delete(desired, neigh.Id) } @@ -88,9 +88,7 @@ func (n *Nylon) reconcileRouterState(next *state.CentralCfg) error { Routes: make(map[netip.Prefix]state.NeighRoute), Eps: make([]state.Endpoint, 0, len(cfg.Endpoints)), } - for _, ep := range cfg.Endpoints { - stNeigh.Eps = append(stNeigh.Eps, state.NewEndpoint(ep, false, nil, &n.RouterTunables)) - } + stNeigh.Eps = append(stNeigh.Eps, configuredEndpoints(n.LocalCfg.Binds, cfg.Endpoints, &n.RouterTunables)...) neighs = append(neighs, stNeigh) } n.RouterState.Neighbours = neighs @@ -107,14 +105,64 @@ func (n *Nylon) reconcileRouterState(next *state.CentralCfg) error { return nil } -func reconcileConfiguredEndpoints(neigh *state.Neighbour, desired []*state.DynamicEndpoint, t *state.RouterTunables) { - desiredByValue := make(map[string]*state.DynamicEndpoint, len(desired)) +func configuredEndpoints(binds []state.LocalBind, endpoints []*state.DynamicEndpoint, t *state.RouterTunables) []state.Endpoint { + if len(binds) == 0 { + eps := make([]state.Endpoint, 0, len(endpoints)) + for _, ep := range endpoints { + eps = append(eps, state.NewEndpoint(ep, false, nil, t)) + } + return eps + } + + eps := make([]state.Endpoint, 0, len(endpoints)*len(binds)) + for _, ep := range endpoints { + for _, bind := range binds { + if !bindMatchesEndpoint(bind, ep) { + continue + } + nep := state.NewEndpoint(ep, false, nil, t) + nep.Bind = bind + eps = append(eps, nep) + } + } + return eps +} + +func bindMatchesEndpoint(bind state.LocalBind, ep *state.DynamicEndpoint) bool { + if !bind.Source.IsValid() { + return true + } + host, _, err := ep.Parse() + if err != nil { + return false + } + addr, err := netip.ParseAddr(host) + if err != nil { + return true + } + return state.SameIPFamily(bind.Source, addr) +} + +type endpointIdentity struct { + value string + bind state.LocalBind +} + +func endpointKey(ep *state.NylonEndpoint) endpointIdentity { + return endpointIdentity{ + value: ep.DynEP.Value, + bind: ep.Bind, + } +} + +func reconcileConfiguredEndpoints(neigh *state.Neighbour, desired []state.Endpoint) { + desiredByKey := make(map[endpointIdentity]state.Endpoint, len(desired)) for _, ep := range desired { - desiredByValue[ep.Value] = ep + desiredByKey[endpointKey(ep.AsNylonEndpoint())] = ep } eps := make([]state.Endpoint, 0, len(neigh.Eps)+len(desired)) - seen := make(map[string]struct{}, len(desired)) + seen := make(map[endpointIdentity]struct{}, len(desired)) for _, ep := range neigh.Eps { nep := ep.AsNylonEndpoint() if ep.IsRemote() { @@ -122,16 +170,18 @@ func reconcileConfiguredEndpoints(neigh *state.Neighbour, desired []*state.Dynam continue } // only keep if desired - if desiredEp, ok := desiredByValue[nep.DynEP.Value]; ok { + key := endpointKey(nep) + if _, ok := desiredByKey[key]; ok { eps = append(eps, ep) - seen[desiredEp.Value] = struct{}{} + seen[key] = struct{}{} } } for _, ep := range desired { - if _, ok := seen[ep.Value]; ok { + key := endpointKey(ep.AsNylonEndpoint()) + if _, ok := seen[key]; ok { continue } - eps = append(eps, state.NewEndpoint(ep, false, nil, t)) + eps = append(eps, ep) } neigh.Eps = eps } diff --git a/core/nylon_apply_test.go b/core/nylon_apply_test.go new file mode 100644 index 00000000..1242c0a3 --- /dev/null +++ b/core/nylon_apply_test.go @@ -0,0 +1,38 @@ +package core + +import ( + "net/netip" + "testing" + + "github.com/encodeous/nylon/state" + "github.com/stretchr/testify/assert" +) + +func TestConfiguredEndpointsExpandsBindsAcrossMatchingEndpointFamilies(t *testing.T) { + tunables := state.DefaultRouterTunables() + eps := configuredEndpoints([]state.LocalBind{ + {Source: netip.MustParseAddr("192.0.2.10")}, + {Source: netip.MustParseAddr("2001:db8::10")}, + }, []*state.DynamicEndpoint{ + state.NewDynamicEndpoint("198.51.100.10:57175"), + state.NewDynamicEndpoint("[2001:db8::20]:57175"), + }, &tunables) + + assert.Len(t, eps, 2) + assert.Equal(t, netip.MustParseAddr("192.0.2.10"), eps[0].AsNylonEndpoint().Bind.Source) + assert.Equal(t, "198.51.100.10:57175", eps[0].AsNylonEndpoint().DynEP.Value) + assert.Equal(t, netip.MustParseAddr("2001:db8::10"), eps[1].AsNylonEndpoint().Bind.Source) + assert.Equal(t, "[2001:db8::20]:57175", eps[1].AsNylonEndpoint().DynEP.Value) +} + +func TestConfiguredEndpointsUsesDefaultEndpointWhenNoBindsConfigured(t *testing.T) { + tunables := state.DefaultRouterTunables() + eps := configuredEndpoints(nil, []*state.DynamicEndpoint{ + state.NewDynamicEndpoint("198.51.100.10:57175"), + state.NewDynamicEndpoint("[2001:db8::20]:57175"), + }, &tunables) + + assert.Len(t, eps, 2) + assert.False(t, eps[0].AsNylonEndpoint().Bind.Source.IsValid()) + assert.False(t, eps[1].AsNylonEndpoint().Bind.Source.IsValid()) +} diff --git a/core/nylon_endpoints.go b/core/nylon_endpoints.go index 22b435a5..7cdeba50 100644 --- a/core/nylon_endpoints.go +++ b/core/nylon_endpoints.go @@ -15,6 +15,8 @@ import ( type EpPing struct { TimeSent time.Time + Node state.NodeId + Endpoint *state.NylonEndpoint } func (n *Nylon) Probe(node state.NodeId, ep *state.NylonEndpoint, waitErr bool) error { @@ -35,6 +37,8 @@ func (n *Nylon) Probe(node state.NodeId, ep *state.NylonEndpoint, waitErr bool) n.PingBuf.Set(token, EpPing{ TimeSent: time.Now(), + Node: node, + Endpoint: ep, }, ttlcache.DefaultTTL) wg := sync.WaitGroup{} @@ -130,33 +134,29 @@ func handleProbePing(n *Nylon, node state.NodeId, wgEndpoint conn.Endpoint) { } func handleProbePong(n *Nylon, node state.NodeId, token uint64, ep conn.Endpoint) { - // check if link exists - for _, neigh := range n.RouterState.Neighbours { - for _, dep := range neigh.Eps { - dpLink := dep.AsNylonEndpoint() - ap, err := dpLink.DynEP.Get() - if err == nil && ap == ep.DstIPPort() && neigh.Id == node { - linkHealth, ok := n.PingBuf.GetAndDelete(token) - if ok { - health := linkHealth.Value() - latency := time.Since(health.TimeSent) - // we have a link - if n.DBG_log_probe { - n.Log.Debug("probe back", "peer", node, "ping", latency) - } - dpLink.Renew() - dpLink.UpdatePing(latency) - - // update wireguard endpoint - dpLink.WgEndpoint = ep - - ComputeRoutes(n.RouterState, n) - } - return - } - } + linkHealth, ok := n.PingBuf.GetAndDelete(token) + if !ok { + n.Log.Warn("probe came back and couldn't find token", "from", ep.DstToString(), "node", node) + return + } + health := linkHealth.Value() + dpLink := health.Endpoint + if health.Node != node || dpLink == nil { + n.Log.Warn("probe came back for unexpected node", "from", ep.DstToString(), "node", node, "expected", health.Node) + return } - n.Log.Warn("probe came back and couldn't find link", "from", ep.DstToString(), "node", node) + latency := time.Since(health.TimeSent) + // we have a link + if n.DBG_log_probe { + n.Log.Debug("probe back", "peer", node, "ping", latency) + } + dpLink.Renew() + dpLink.UpdatePing(latency) + + // update wireguard endpoint + dpLink.WgEndpoint = ep + + ComputeRoutes(n.RouterState, n) } func (n *Nylon) probeLinks(active bool) error { diff --git a/docs/reference/config.mdx b/docs/reference/config.mdx index 566822ad..d67fe3e2 100644 --- a/docs/reference/config.mdx +++ b/docs/reference/config.mdx @@ -22,6 +22,14 @@ log_path: "" # write logs to this file (empty = stderr only) interface_name: "" # override the interface name (default: "nylon", or utunX on macOS) dns_resolvers: [] # DNS servers for nylon's own lookups, e.g. ["1.1.1.1:53"] +# Local endpoint selectors. Nylon probes each peer endpoint through matching +# binds and uses the best active link for the routing edge. +binds: + - interface: eth0 + source: 192.0.2.10 + - interface: eth0 + source: 2001:db8::10 + # Bootstrap: fetch central.yaml from a remote bundle on first start dist: url: https://static.example.com/network1.nybundle diff --git a/docs/reference/endpoint-local-binds.mdx b/docs/reference/endpoint-local-binds.mdx new file mode 100644 index 00000000..8f91f8c9 --- /dev/null +++ b/docs/reference/endpoint-local-binds.mdx @@ -0,0 +1,77 @@ +# Endpoint Local Binds + +Nylon keeps routing decisions separate from transport endpoint selection. +Routing chooses which node to visit next. Endpoint selection chooses which +underlay address, source address, and local interface are used for the UDP +packet sent to that next node. + +## Model + +At the routing level, a neighbour is represented once. Multiple underlay +endpoints for the same neighbour are link-level candidates for the same routing +edge, and Nylon surfaces the best active endpoint metric to the router. + +At the link level, a Nylon endpoint may carry an optional local bind selector: + +- source address: the local address to use for the outgoing UDP packet +- interface index: the local interface to constrain the outgoing UDP packet to + +The source address and endpoint address must be in the same IP family when a +source address is configured. If no local bind selector is configured, endpoint +traffic uses the kernel's normal source address and route selection. + +Users configure local bind selectors in `node.yaml`: + +```yaml +binds: + - interface: eth0 + source: 192.0.2.10 + - interface: eth0 + source: 2001:db8::10 +``` + +For each peer endpoint, Nylon creates link-level candidates for the configured +binds whose source address family matches the remote endpoint. Interface-only +binds are applied to all endpoint families. If `binds` is empty, Nylon keeps the +default behaviour and creates one unbound candidate per endpoint. + +This design does not create one socket per local bind. A Nylon device owns one +IPv4 UDP socket and one IPv6 UDP socket when both address families are +available. Local bind selectors are attached to endpoint sends. + +## Linux Transport + +On Linux, endpoint-local binds are encoded as per-message ancillary data: + +- IPv4 uses `IP_PKTINFO` with `in_pktinfo.ipi_spec_dst` and + `in_pktinfo.ipi_ifindex`. +- IPv6 uses `IPV6_PKTINFO` with `in6_pktinfo.ipi6_addr` and + `in6_pktinfo.ipi6_ifindex`. + +When both a source address and interface index are provided, Nylon sends both in +the packet info control message. The source address remains the requested source +address; the interface index constrains output interface selection. When only an +interface is provided, the packet info source address is left unspecified and +the kernel may select an address for that interface. + +This keeps probe packets and other control traffic precise without changing the +socket-level bind. It also allows different endpoints for the same peer to use +different local bind selectors while still sharing the device sockets. + +## Non-Linux Platforms + +The endpoint-local bind mechanism is currently implemented only for Linux. +Other platforms keep the default socket behaviour. A cross-platform user-facing +configuration should only expose selectors that the current platform can honor, +or should fail validation before the device starts. + +## Routing Behaviour + +Endpoint-local binds affect only the link-level endpoint used to send packets. +They do not create additional routing-level neighbours or independent routed +edges. Nylon still computes routes over nodes and uses the best active endpoint +metric for each neighbour. + +Bandwidth aggregation, ECMP, and max-flow style forwarding are outside this +model. Supporting those would require different routing semantics rather than +additional endpoint-local bind selectors. diff --git a/polyamide/conn/bind_std.go b/polyamide/conn/bind_std.go index 27f5fe86..6938d788 100644 --- a/polyamide/conn/bind_std.go +++ b/polyamide/conn/bind_std.go @@ -374,12 +374,12 @@ func (s *StdNetBind) Send(bufs [][]byte, endpoint Endpoint) error { defer s.udpAddrPool.Put(ua) if is6 { as16 := endpoint.DstIP().As16() - copy(ua.IP, as16[:]) ua.IP = ua.IP[:16] + copy(ua.IP, as16[:]) } else { as4 := endpoint.DstIP().As4() - copy(ua.IP, as4[:]) ua.IP = ua.IP[:4] + copy(ua.IP, as4[:]) } ua.Port = int(endpoint.(*StdNetEndpoint).Port()) var ( @@ -526,6 +526,7 @@ func splitCoalescedMessages(msgs []ipv6.Message, firstMsgAt int, getGSO getGSOFu if err != nil { return n, err } + control := append([]byte(nil), msg.OOB[:msg.NN]...) if gsoSize > 0 { numToSplit = (msg.N + gsoSize - 1) / gsoSize end = gsoSize @@ -536,6 +537,8 @@ func splitCoalescedMessages(msgs []ipv6.Message, firstMsgAt int, getGSO getGSOFu } copied := copy(msgs[n].Buffers[0], msg.Buffers[0][start:end]) msgs[n].N = copied + msgs[n].OOB = append(msgs[n].OOB[:0], control...) + msgs[n].NN = len(control) msgs[n].Addr = msg.Addr start = end end += gsoSize diff --git a/polyamide/conn/bind_std_test.go b/polyamide/conn/bind_std_test.go index 34a3c9ac..cecf4afd 100644 --- a/polyamide/conn/bind_std_test.go +++ b/polyamide/conn/bind_std_test.go @@ -248,3 +248,27 @@ func Test_splitCoalescedMessages(t *testing.T) { }) } } + +func Test_splitCoalescedMessagesPreservesControl(t *testing.T) { + msgs := []ipv6.Message{ + {Buffers: [][]byte{make([]byte, 1<<16-1)}, OOB: make([]byte, 0, 2)}, + {Buffers: [][]byte{make([]byte, 1<<16-1)}, OOB: make([]byte, 0, 2)}, + {Buffers: [][]byte{make([]byte, 1<<16-1)}, N: 2, NN: 2, OOB: []byte{1, 0}}, + {Buffers: [][]byte{make([]byte, 1<<16-1)}, OOB: make([]byte, 0, 2)}, + } + got, err := splitCoalescedMessages(msgs, 2, mockGetGSOSize) + if err != nil { + t.Fatalf("err: %v", err) + } + if got != 2 { + t.Fatalf("got to eval: %d want: %d", got, 2) + } + for i := 0; i < got; i++ { + if msgs[i].NN != 2 { + t.Fatalf("msg[%d].NN: %d want: 2", i, msgs[i].NN) + } + if gotGSO, err := mockGetGSOSize(msgs[i].OOB[:msgs[i].NN]); err != nil || gotGSO != 1 { + t.Fatalf("msg[%d] gso: %d err: %v", i, gotGSO, err) + } + } +} diff --git a/polyamide/conn/sticky_default.go b/polyamide/conn/sticky_default.go index 15b65af8..c05ba49f 100644 --- a/polyamide/conn/sticky_default.go +++ b/polyamide/conn/sticky_default.go @@ -21,6 +21,8 @@ func (e *StdNetEndpoint) SrcToString() string { return "" } +func (e *StdNetEndpoint) SetSrc(addr netip.Addr, ifidx int32) {} + // TODO: macOS, FreeBSD and other BSDs likely do support the sticky sockets // {get,set}srcControl feature set, but use alternatively named flags and need // ports and require testing. diff --git a/polyamide/conn/sticky_linux.go b/polyamide/conn/sticky_linux.go index adfedc17..8d4d0b5b 100644 --- a/polyamide/conn/sticky_linux.go +++ b/polyamide/conn/sticky_linux.go @@ -45,6 +45,54 @@ func (e *StdNetEndpoint) SrcToString() string { return e.SrcIP().String() } +func (e *StdNetEndpoint) SetSrc(addr netip.Addr, ifidx int32) { + if !addr.IsValid() && ifidx == 0 { + e.ClearSrc() + return + } + if !addr.IsValid() { + addr = netip.AddrFrom16([16]byte{}) + if e.DstIP().Is4() { + addr = netip.AddrFrom4([4]byte{}) + } + } + if addr.Is4() { + if e.src == nil || cap(e.src) < unix.CmsgSpace(unix.SizeofInet4Pktinfo) { + e.src = make([]byte, 0, unix.CmsgSpace(unix.SizeofInet4Pktinfo)) + } + e.src = e.src[:unix.CmsgSpace(unix.SizeofInet4Pktinfo)] + hdr := unix.Cmsghdr{ + Level: unix.IPPROTO_IP, + Type: unix.IP_PKTINFO, + } + hdr.SetLen(unix.CmsgLen(unix.SizeofInet4Pktinfo)) + copy(e.src, unsafe.Slice((*byte)(unsafe.Pointer(&hdr)), int(unsafe.Sizeof(hdr)))) + info := unix.Inet4Pktinfo{ + Ifindex: ifidx, + Spec_dst: addr.As4(), + } + copy(e.src[unix.CmsgLen(0):], unsafe.Slice((*byte)(unsafe.Pointer(&info)), unix.SizeofInet4Pktinfo)) + return + } + if addr.Is6() { + if e.src == nil || cap(e.src) < unix.CmsgSpace(unix.SizeofInet6Pktinfo) { + e.src = make([]byte, 0, unix.CmsgSpace(unix.SizeofInet6Pktinfo)) + } + e.src = e.src[:unix.CmsgSpace(unix.SizeofInet6Pktinfo)] + hdr := unix.Cmsghdr{ + Level: unix.IPPROTO_IPV6, + Type: unix.IPV6_PKTINFO, + } + hdr.SetLen(unix.CmsgLen(unix.SizeofInet6Pktinfo)) + copy(e.src, unsafe.Slice((*byte)(unsafe.Pointer(&hdr)), int(unsafe.Sizeof(hdr)))) + info := unix.Inet6Pktinfo{ + Ifindex: uint32(ifidx), + Addr: addr.As16(), + } + copy(e.src[unix.CmsgLen(0):], unsafe.Slice((*byte)(unsafe.Pointer(&info)), unix.SizeofInet6Pktinfo)) + } +} + // getSrcFromControl parses the control for PKTINFO and if found updates ep with // the source information found. func getSrcFromControl(control []byte, ep *StdNetEndpoint) { diff --git a/polyamide/conn/sticky_linux_test.go b/polyamide/conn/sticky_linux_test.go index 1b1ee683..185a4199 100644 --- a/polyamide/conn/sticky_linux_test.go +++ b/polyamide/conn/sticky_linux_test.go @@ -54,6 +54,49 @@ func setSrc(ep *StdNetEndpoint, addr netip.Addr, ifidx int32) { } func Test_setSrcControl(t *testing.T) { + t.Run("SetSrcStoresPacketInfo", func(t *testing.T) { + ep := &StdNetEndpoint{ + AddrPort: netip.MustParseAddrPort("127.0.0.1:1234"), + } + ep.SetSrc(netip.MustParseAddr("127.0.0.1"), 5) + + control := make([]byte, stickyControlSize) + setSrcControl(&control, ep) + + info := (*unix.Inet4Pktinfo)(unsafe.Pointer(&control[unix.CmsgLen(0)])) + if info.Spec_dst != [4]byte{127, 0, 0, 1} { + t.Errorf("unexpected address: %v", info.Spec_dst) + } + if info.Ifindex != 5 { + t.Errorf("unexpected ifindex: %d", info.Ifindex) + } + }) + + t.Run("SetSrcStoresInterfaceOnlyPacketInfo", func(t *testing.T) { + ep := &StdNetEndpoint{ + AddrPort: netip.MustParseAddrPort("127.0.0.1:1234"), + } + ep.SetSrc(netip.Addr{}, 5) + + control := make([]byte, stickyControlSize) + setSrcControl(&control, ep) + + hdr := (*unix.Cmsghdr)(unsafe.Pointer(&control[0])) + if hdr.Level != unix.IPPROTO_IP { + t.Errorf("unexpected level: %d", hdr.Level) + } + if hdr.Type != unix.IP_PKTINFO { + t.Errorf("unexpected type: %d", hdr.Type) + } + info := (*unix.Inet4Pktinfo)(unsafe.Pointer(&control[unix.CmsgLen(0)])) + if info.Spec_dst != [4]byte{} { + t.Errorf("unexpected address: %v", info.Spec_dst) + } + if info.Ifindex != 5 { + t.Errorf("unexpected ifindex: %d", info.Ifindex) + } + }) + t.Run("IPv4", func(t *testing.T) { ep := &StdNetEndpoint{ AddrPort: netip.MustParseAddrPort("127.0.0.1:1234"), diff --git a/polyamide/device/peer.go b/polyamide/device/peer.go index 9d1afc7b..6497742f 100644 --- a/polyamide/device/peer.go +++ b/polyamide/device/peer.go @@ -136,11 +136,6 @@ func (peer *Peer) SendBuffers(buffers [][]byte, eps []conn.Endpoint) error { for _, ep := range endpoints { ep.ClearSrc() } - for _, ep := range eps { - if ep != nil { - ep.ClearSrc() - } - } peer.endpoints.clearSrcOnTx = false } peer.endpoints.Unlock() diff --git a/protocol/nylon_ipc.pb.go b/protocol/nylon_ipc.pb.go index 5be9758f..4616eab1 100644 --- a/protocol/nylon_ipc.pb.go +++ b/protocol/nylon_ipc.pb.go @@ -578,16 +578,18 @@ func (x *Advertisement) GetPassiveHold() bool { } type EndpointInfo struct { - state protoimpl.MessageState `protogen:"open.v1"` - Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"` - Resolved *string `protobuf:"bytes,2,opt,name=resolved,proto3,oneof" json:"resolved,omitempty"` - Active bool `protobuf:"varint,3,opt,name=active,proto3" json:"active,omitempty"` - RemoteInit bool `protobuf:"varint,4,opt,name=remote_init,json=remoteInit,proto3" json:"remote_init,omitempty"` - Metric uint32 `protobuf:"varint,5,opt,name=metric,proto3" json:"metric,omitempty"` - FilteredRttNs int64 `protobuf:"varint,7,opt,name=filtered_rtt_ns,json=filteredRttNs,proto3" json:"filtered_rtt_ns,omitempty"` - StabilizedRttNs int64 `protobuf:"varint,8,opt,name=stabilized_rtt_ns,json=stabilizedRttNs,proto3" json:"stabilized_rtt_ns,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"` + Resolved *string `protobuf:"bytes,2,opt,name=resolved,proto3,oneof" json:"resolved,omitempty"` + Active bool `protobuf:"varint,3,opt,name=active,proto3" json:"active,omitempty"` + RemoteInit bool `protobuf:"varint,4,opt,name=remote_init,json=remoteInit,proto3" json:"remote_init,omitempty"` + Metric uint32 `protobuf:"varint,5,opt,name=metric,proto3" json:"metric,omitempty"` + FilteredRttNs int64 `protobuf:"varint,7,opt,name=filtered_rtt_ns,json=filteredRttNs,proto3" json:"filtered_rtt_ns,omitempty"` + StabilizedRttNs int64 `protobuf:"varint,8,opt,name=stabilized_rtt_ns,json=stabilizedRttNs,proto3" json:"stabilized_rtt_ns,omitempty"` + LocalBindInterface string `protobuf:"bytes,10,opt,name=local_bind_interface,json=localBindInterface,proto3" json:"local_bind_interface,omitempty"` + LocalBindSource string `protobuf:"bytes,11,opt,name=local_bind_source,json=localBindSource,proto3" json:"local_bind_source,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *EndpointInfo) Reset() { @@ -669,6 +671,20 @@ func (x *EndpointInfo) GetStabilizedRttNs() int64 { return 0 } +func (x *EndpointInfo) GetLocalBindInterface() string { + if x != nil { + return x.LocalBindInterface + } + return "" +} + +func (x *EndpointInfo) GetLocalBindSource() string { + if x != nil { + return x.LocalBindSource + } + return "" +} + type WireGuardPeerStats struct { state protoimpl.MessageState `protogen:"open.v1"` LatestHandshakeUnix int64 `protobuf:"varint,1,opt,name=latest_handshake_unix,json=latestHandshakeUnix,proto3" json:"latest_handshake_unix,omitempty"` @@ -1799,7 +1815,7 @@ const file_protocol_nylon_ipc_proto_rawDesc = "" + "\x06metric\x18\x03 \x01(\rR\x06metric\x12\x1f\n" + "\vexpiry_unix\x18\x04 \x01(\x03R\n" + "expiryUnix\x12!\n" + - "\fpassive_hold\x18\x05 \x01(\bR\vpassiveHold\"\xfb\x01\n" + + "\fpassive_hold\x18\x05 \x01(\bR\vpassiveHold\"\xd9\x02\n" + "\fEndpointInfo\x12\x18\n" + "\aaddress\x18\x01 \x01(\tR\aaddress\x12\x1f\n" + "\bresolved\x18\x02 \x01(\tH\x00R\bresolved\x88\x01\x01\x12\x16\n" + @@ -1808,7 +1824,10 @@ const file_protocol_nylon_ipc_proto_rawDesc = "" + "remoteInit\x12\x16\n" + "\x06metric\x18\x05 \x01(\rR\x06metric\x12&\n" + "\x0ffiltered_rtt_ns\x18\a \x01(\x03R\rfilteredRttNs\x12*\n" + - "\x11stabilized_rtt_ns\x18\b \x01(\x03R\x0fstabilizedRttNsB\v\n" + + "\x11stabilized_rtt_ns\x18\b \x01(\x03R\x0fstabilizedRttNs\x120\n" + + "\x14local_bind_interface\x18\n" + + " \x01(\tR\x12localBindInterface\x12*\n" + + "\x11local_bind_source\x18\v \x01(\tR\x0flocalBindSourceB\v\n" + "\t_resolved\"\xf0\x01\n" + "\x12WireGuardPeerStats\x122\n" + "\x15latest_handshake_unix\x18\x01 \x01(\x03R\x13latestHandshakeUnix\x12\x19\n" + diff --git a/protocol/nylon_ipc.proto b/protocol/nylon_ipc.proto index 080276ca..9d454467 100644 --- a/protocol/nylon_ipc.proto +++ b/protocol/nylon_ipc.proto @@ -63,6 +63,8 @@ message EndpointInfo { uint32 metric = 5; int64 filtered_rtt_ns = 7; int64 stabilized_rtt_ns = 8; + string local_bind_interface = 10; + string local_bind_source = 11; } message WireGuardPeerStats { diff --git a/state/config.go b/state/config.go index 083493e0..7df37074 100644 --- a/state/config.go +++ b/state/config.go @@ -64,6 +64,7 @@ type LocalCfg struct { PreDown []string `yaml:"pre_down,omitempty"` // a list of commands executed in order before the nylon interface is brought down PostUp []string `yaml:"post_up,omitempty"` // a list of commands executed in order after the nylon interface is brought up PostDown []string `yaml:"post_down,omitempty"` // a list of commands executed in order after the nylon interface is brought down + Binds []LocalBind `yaml:"binds,omitempty"` // local source/interface selectors used for endpoint probing } func (c *CentralCfg) Clone() (error, *CentralCfg) { diff --git a/state/config_test.go b/state/config_test.go index 92a63912..78d7e930 100644 --- a/state/config_test.go +++ b/state/config_test.go @@ -1,9 +1,12 @@ package state import ( + "net/netip" + "runtime" "strings" "testing" + "github.com/goccy/go-yaml" "github.com/stretchr/testify/assert" ) @@ -149,3 +152,47 @@ func TestParseGraph_InvalidGraph(t *testing.T) { failGraph(t, `,,,,,,,,,,,,,,,,`) failGraph(t, `a=a`) } + +func TestLocalBindsParse(t *testing.T) { + data := []byte(` +key: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= +id: alice +port: 57175 +binds: + - interface: eth0 + source: 203.0.113.10 +`) + var local LocalCfg + err := yaml.Unmarshal(data, &local) + assert.NoError(t, err) + assert.Len(t, local.Binds, 1) + assert.Equal(t, "eth0", local.Binds[0].Interface) + assert.Equal(t, netip.MustParseAddr("203.0.113.10"), local.Binds[0].Source) +} + +func TestNodeConfigValidatorRejectsSelectorlessBind(t *testing.T) { + local := LocalCfg{ + Id: "alice", + Key: [32]byte{1}, + Port: 57175, + Binds: []LocalBind{{}}, + } + + err := NodeConfigValidator(nil, &local) + assert.ErrorContains(t, err, "must specify source or interface") +} + +func TestNodeConfigValidatorAllowsBind(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("local binds are linux-only") + } + local := LocalCfg{ + Id: "alice", + Key: [32]byte{1}, + Port: 57175, + Binds: []LocalBind{{Source: netip.MustParseAddr("203.0.113.10")}}, + } + + err := NodeConfigValidator(nil, &local) + assert.NoError(t, err) +} diff --git a/state/endpoint.go b/state/endpoint.go index 2bdd3a1d..32420e87 100644 --- a/state/endpoint.go +++ b/state/endpoint.go @@ -23,6 +23,13 @@ type Endpoint interface { AsNylonEndpoint() *NylonEndpoint } +func SameIPFamily(a, b netip.Addr) bool { + if !a.IsValid() || !b.IsValid() { + return true + } + return a.BitLen() == b.BitLen() +} + /* DynamicEndpoint represents either an ip:port or a dns name. This may be resolved to a different address at any time @@ -164,6 +171,7 @@ type NylonEndpoint struct { remoteInit bool WgEndpoint conn.Endpoint DynEP *DynamicEndpoint + Bind LocalBind } func (ep *NylonEndpoint) AsNylonEndpoint() *NylonEndpoint { @@ -175,6 +183,9 @@ func (ep *NylonEndpoint) GetWgEndpoint(device *device.Device) (conn.Endpoint, er if err != nil { return nil, err } + if !SameIPFamily(ep.Bind.Source, ap.Addr()) { + return nil, fmt.Errorf("bind source %s does not match endpoint %s", ep.Bind.Source, ap) + } if ep.WgEndpoint == nil || ep.WgEndpoint.DstIPPort() != ap { wgEp, err := device.Bind().ParseEndpoint(ap.String()) @@ -183,6 +194,19 @@ func (ep *NylonEndpoint) GetWgEndpoint(device *device.Device) (conn.Endpoint, er } ep.WgEndpoint = wgEp } + if setter, ok := ep.WgEndpoint.(interface { + SetSrc(netip.Addr, int32) + }); ok && (ep.Bind.Source.IsValid() || ep.Bind.Interface != "") { + ifidx := int32(0) + if ep.Bind.Interface != "" { + iface, err := net.InterfaceByName(ep.Bind.Interface) + if err != nil { + return nil, fmt.Errorf("failed to resolve bind interface %s: %w", ep.Bind.Interface, err) + } + ifidx = int32(iface.Index) + } + setter.SetSrc(ep.Bind.Source, ifidx) + } return ep.WgEndpoint, nil } diff --git a/state/endpoint_test.go b/state/endpoint_test.go index b7ad2a7d..a6cfc0a4 100644 --- a/state/endpoint_test.go +++ b/state/endpoint_test.go @@ -3,12 +3,20 @@ package state import ( "math" "math/rand/v2" + "net/netip" "testing" "time" "github.com/stretchr/testify/assert" ) +func TestSameIPFamily(t *testing.T) { + assert.True(t, SameIPFamily(netip.MustParseAddr("192.0.2.1"), netip.MustParseAddr("198.51.100.1"))) + assert.True(t, SameIPFamily(netip.MustParseAddr("2001:db8::1"), netip.MustParseAddr("2001:db8::2"))) + assert.False(t, SameIPFamily(netip.MustParseAddr("192.0.2.1"), netip.MustParseAddr("2001:db8::1"))) + assert.True(t, SameIPFamily(netip.Addr{}, netip.MustParseAddr("2001:db8::1"))) +} + type DataSource struct { Name string Data []time.Duration diff --git a/state/routing.go b/state/routing.go index 2de499f6..74d2bb50 100644 --- a/state/routing.go +++ b/state/routing.go @@ -11,6 +11,11 @@ import ( type NodeId string +type LocalBind struct { + Interface string `yaml:"interface,omitempty"` + Source netip.Addr `yaml:"source,omitempty"` +} + // Source is a pair of a router-id and a prefix (Babel Section 2.7). type Source struct { NodeId diff --git a/state/validation.go b/state/validation.go index 5c0e7009..a0fe0c53 100644 --- a/state/validation.go +++ b/state/validation.go @@ -5,6 +5,7 @@ import ( "net/netip" "net/url" "regexp" + "runtime" "slices" ) @@ -43,6 +44,19 @@ func NodeConfigValidator(central *CentralCfg, node *LocalCfg) error { return err } } + seenBinds := make(map[LocalBind]struct{}) + for _, bind := range node.Binds { + if bind.Interface == "" && !bind.Source.IsValid() { + return fmt.Errorf("bind must specify source or interface") + } + if _, ok := seenBinds[bind]; ok { + return fmt.Errorf("duplicate bind %s %s", bind.Interface, bind.Source) + } + seenBinds[bind] = struct{}{} + if runtime.GOOS != "linux" { + return fmt.Errorf("local binds are only supported on linux") + } + } if len(node.DnsResolvers) != 0 { for _, resolver := range node.DnsResolvers { if _, err := netip.ParseAddrPort(resolver); err != nil {