feat: store all reported listen multi addresses

cmd/nebula/cmd_crawl.go

@@ -36,7 +36,6 @@ var crawlConfig = &config.Crawl{
 	Network:           string(config.NetworkIPFS),
 	BootstrapPeers:    cli.NewStringSlice(),
 	Protocols:         cli.NewStringSlice(string(kaddht.ProtocolDHT)),
-	AddrTrackTypeStr:  "public",
 	AddrDialTypeStr:   "public",
 	KeepENR:           false,
 	CheckExposed:      false,
@@ -74,17 +73,6 @@ var CrawlCommand = &cli.Command{
 			fmt.Println(crawlConfig.String())
 		}
 
-		switch config.AddrType(strings.ToLower(crawlConfig.AddrTrackTypeStr)) {
-		case config.AddrTypePrivate:
-			crawlConfig.AddrTrackTypeStr = string(config.AddrTypePrivate)
-		case config.AddrTypePublic:
-			crawlConfig.AddrTrackTypeStr = string(config.AddrTypePublic)
-		case config.AddrTypeAny:
-			crawlConfig.AddrTrackTypeStr = string(config.AddrTypeAny)
-		default:
-			return fmt.Errorf("unknown type of addresses to track: %s (supported values are private, public, any)", crawlConfig.AddrTrackTypeStr)
-		}
-
 		switch config.AddrType(strings.ToLower(crawlConfig.AddrDialTypeStr)) {
 		case config.AddrTypePrivate:
 			crawlConfig.AddrDialTypeStr = string(config.AddrTypePrivate)
@@ -153,25 +141,19 @@ var CrawlCommand = &cli.Command{
 			Destination: &crawlConfig.PersistNeighbors,
 		},
 		&cli.StringFlag{
-			Name:        "addr-track-type",
-			Usage:       "Which type addresses should be stored to the database (private, public, any)",
-			EnvVars:     []string{"NEBULA_CRAWL_ADDR_TRACK_TYPE"},
-			Value:       crawlConfig.AddrTrackTypeStr,
-			Destination: &crawlConfig.AddrTrackTypeStr,
+			Name:        "network",
+			Usage:       "Which network should be crawled. Presets default bootstrap peers and protocol. Run: `nebula networks` for more information.",
+			EnvVars:     []string{"NEBULA_CRAWL_NETWORK"},
+			Value:       crawlConfig.Network,
+			Destination: &crawlConfig.Network,
 		},
 		&cli.StringFlag{
 			Name:        "addr-dial-type",
-			Usage:       "Which type of addresses should Nebula try to dial (private, public, any)",
+			Usage:       "LIBP2P/NON-ETHEREUM: Which type of addresses should Nebula try to dial (private, public, any)",
 			EnvVars:     []string{"NEBULA_CRAWL_ADDR_DIAL_TYPE"},
 			Value:       crawlConfig.AddrDialTypeStr,
 			Destination: &crawlConfig.AddrDialTypeStr,
-		},
-		&cli.StringFlag{
-			Name:        "network",
-			Usage:       "Which network should be crawled. Presets default bootstrap peers and protocol. Run: `nebula networks` for more information.",
-			EnvVars:     []string{"NEBULA_CRAWL_NETWORK"},
-			Value:       crawlConfig.Network,
-			Destination: &crawlConfig.Network,
+			Category:    flagCategoryNetwork,
 		},
 		&cli.BoolFlag{
 			Name:        "gossipsub-px",
@@ -318,7 +300,6 @@ func CrawlAction(c *cli.Context) error {
 			CrawlWorkerCount: cfg.CrawlWorkerCount,
 			BootstrapPeers:   bpEnodes,
 			AddrDialType:     cfg.AddrDialType(),
-			AddrTrackType:    cfg.AddrTrackType(),
 			TracerProvider:   cfg.Root.TracerProvider,
 			MeterProvider:    cfg.Root.MeterProvider,
 			LogErrors:        cfg.Root.LogErrors,
@@ -424,7 +405,6 @@ func CrawlAction(c *cli.Context) error {
 			BootstrapPeers:    bpEnodes,
 			CrawlWorkerCount:  cfg.CrawlWorkerCount,
 			AddrDialType:      cfg.AddrDialType(),
-			AddrTrackType:     cfg.AddrTrackType(),
 			KeepENR:           crawlConfig.KeepENR,
 			TracerProvider:    cfg.Root.TracerProvider,
 			MeterProvider:     cfg.Root.MeterProvider,
@@ -473,7 +453,6 @@ func CrawlAction(c *cli.Context) error {
 			CheckExposed:   cfg.CheckExposed,
 			BootstrapPeers: bpAddrInfos,
 			AddrDialType:   cfg.AddrDialType(),
-			AddrTrackType:  cfg.AddrTrackType(),
 			TracerProvider: cfg.Root.TracerProvider,
 			MeterProvider:  cfg.Root.MeterProvider,
 			GossipSubPX:    cfg.EnableGossipSubPX,

config/config.go

@@ -402,9 +402,6 @@ type Crawl struct {
 	// The network to crawl
 	Network string
 
-	// Which type addresses should be stored to the database (private, public, both)
-	AddrTrackTypeStr string
-
 	// Which type of addresses should Nebula try to dial (private, public, both)
 	AddrDialTypeStr string
 
@@ -427,10 +424,6 @@ type Crawl struct {
 	EnableGossipSubPX bool
 }
 
-func (c *Crawl) AddrTrackType() AddrType {
-	return AddrType(c.AddrTrackTypeStr)
-}
-
 func (c *Crawl) AddrDialType() AddrType {
 	return AddrType(c.AddrDialTypeStr)
 }

core/engine_test.go

@@ -12,7 +12,6 @@ import (
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
 
-	"github.com/dennis-tra/nebula-crawler/config"
 	"github.com/dennis-tra/nebula-crawler/db"
 	"github.com/dennis-tra/nebula-crawler/utils"
 )
@@ -246,9 +245,7 @@ func TestNewEngine_Run_parking_peers(t *testing.T) {
 	}
 
 	crawler := newTestCrawler()
-	writerCfg := &CrawlWriterConfig{
-		AddrTrackType: config.AddrTypeAny,
-	}
+	writerCfg := &CrawlWriterConfig{}
 	writer := NewCrawlWriter[*testPeerInfo]("1", db.NewNoopClient(), writerCfg)
 
 	crawler.On("Work", mock.IsType(ctx), bootstrapPeer).

core/handler_crawl.go

@@ -53,6 +53,9 @@ type CrawlResult[I PeerInfo[I]] struct {
 	// discovery protocol.
 	ExtraMaddrs []ma.Multiaddr
 
+	// All addresses that the remote peer claims to listen on.
+	ListenMaddrs []ma.Multiaddr
+
 	// The multi address of the connection that we have established to the peer
 	ConnectMaddr ma.Multiaddr
 
@@ -99,11 +102,15 @@ func (r CrawlResult[I]) PeerInfo() I {
 }
 
 func (r CrawlResult[I]) LogEntry() *log.Entry {
+	rtSize := -1
+	if r.RoutingTable != nil {
+		rtSize = len(r.RoutingTable.Neighbors)
+	}
 	logEntry := log.WithFields(log.Fields{
 		"remoteID":   r.Info.ID().ShortString(),
 		"isDialable": r.ConnectError == nil && r.CrawlError == nil,
 		"duration":   r.CrawlDuration(),
-		"rtSize":     len(r.RoutingTable.Neighbors),
+		"rtSize":     rtSize,
 	})
 
 	if r.ConnectError != nil {

core/writer_crawl.go

@@ -8,14 +8,10 @@ import (
 	"github.com/libp2p/go-libp2p/core/peer"
 	log "github.com/sirupsen/logrus"
 
-	"github.com/dennis-tra/nebula-crawler/config"
 	"github.com/dennis-tra/nebula-crawler/db"
-	"github.com/dennis-tra/nebula-crawler/utils"
 )
 
-type CrawlWriterConfig struct {
-	AddrTrackType config.AddrType
-}
+type CrawlWriterConfig struct{}
 
 // CrawlWriter handles the insert/upsert/update operations for a particular crawl result.
 type CrawlWriter[I PeerInfo[I]] struct {
@@ -47,16 +43,6 @@ func (w *CrawlWriter[I]) Work(ctx context.Context, task CrawlResult[I]) (WriteRe
 	logEntry.Debugln("Storing peer")
 	defer logEntry.Debugln("Stored peer")
 
-	maddrs := task.Info.Addrs()
-	switch w.cfg.AddrTrackType {
-	case config.AddrTypePrivate:
-		maddrs = utils.FilterPublicMaddrs(maddrs)
-	case config.AddrTypePublic:
-		maddrs = utils.FilterPrivateMaddrs(maddrs)
-	default:
-		// noop
-	}
-
 	var (
 		errorBits        uint16
 		neighbors        []peer.ID
@@ -80,6 +66,7 @@ func (w *CrawlWriter[I]) Work(ctx context.Context, task CrawlResult[I]) (WriteRe
 		DialMaddrs:       task.DialMaddrs,
 		FilteredMaddrs:   task.FilteredMaddrs,
 		ExtraMaddrs:      task.ExtraMaddrs,
+		ListenMaddrs:     task.ListenMaddrs,
 		ConnectMaddr:     task.ConnectMaddr,
 		DialErrors:       task.DialErrors,
 		ConnectDuration:  task.ConnectDuration(),

db/ch.go

@@ -385,6 +385,7 @@ type ClickHouseVisit struct {
 	DialMaddrs     []string        `ch:"dial_maddrs"`
 	FilteredMaddrs []string        `ch:"filtered_maddrs"`
 	ExtraMaddrs    []string        `ch:"extra_maddrs"`
+	ListenMaddrs   []string        `ch:"listen_maddrs"`
 	DialErrors     []string        `ch:"dial_errors"`
 	ConnectMaddr   *string         `ch:"connect_maddr"`
 	CrawlError     *string         `ch:"crawl_error"`
@@ -605,6 +606,7 @@ func (c *ClickHouseClient) InsertVisit(ctx context.Context, args *VisitArgs) err
 		DialMaddrs:     utils.MaddrsToAddrs(args.DialMaddrs),
 		FilteredMaddrs: utils.MaddrsToAddrs(args.FilteredMaddrs),
 		ExtraMaddrs:    utils.MaddrsToAddrs(args.ExtraMaddrs),
+		ListenMaddrs:   utils.MaddrsToAddrs(args.ListenMaddrs),
 		ConnectMaddr:   connMaddrStr,
 		DialErrors:     args.DialErrors,
 		CrawlError:     crawlErrStr,

db/client.go

@@ -46,6 +46,7 @@ type VisitArgs struct {
 	DialMaddrs       []ma.Multiaddr
 	FilteredMaddrs   []ma.Multiaddr
 	ExtraMaddrs      []ma.Multiaddr
+	ListenMaddrs     []ma.Multiaddr
 	DialErrors       []string
 	ConnectMaddr     ma.Multiaddr
 	DialDuration     time.Duration

db/json.go

@@ -161,6 +161,9 @@ func (c *JSONClient) InsertCrawlProperties(ctx context.Context, properties map[s
 type JSONVisit struct {
 	PeerID          peer.ID
 	Maddrs          []ma.Multiaddr
+	FilteredMaddrs  []ma.Multiaddr
+	ListenMaddrs    []ma.Multiaddr
+	ConnectMaddr    ma.Multiaddr
 	Protocols       []string
 	AgentVersion    string
 	ConnectDuration string
@@ -175,7 +178,10 @@ type JSONVisit struct {
 func (c *JSONClient) InsertVisit(ctx context.Context, args *VisitArgs) error {
 	data := JSONVisit{
 		PeerID:          args.PeerID,
-		Maddrs:          slices.Concat(args.DialMaddrs, args.FilteredMaddrs, args.ExtraMaddrs),
+		Maddrs:          slices.Concat(args.DialMaddrs),
+		FilteredMaddrs:  args.FilteredMaddrs,
+		ListenMaddrs:    args.ListenMaddrs,
+		ConnectMaddr:    args.ConnectMaddr,
 		Protocols:       args.Protocols,
 		AgentVersion:    args.AgentVersion,
 		ConnectDuration: args.ConnectDuration.String(),

db/migrations/chcluster/000006_add_missing_maddrs_column_to_visits_table.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE visits
+    DROP COLUMN listen_maddrs;

db/migrations/chcluster/000006_add_missing_maddrs_column_to_visits_table.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE visits
+    ADD COLUMN listen_maddrs Array(String) DEFAULT [] AFTER extra_maddrs;

db/migrations/chlocal/000003_create_visits_table.up.sql

@@ -1,3 +1,5 @@
+-- DO NOT EDIT: This file was generated with: just generate-local-clickhouse-migrations
+
 -- Captures the information when crawling a peer
 CREATE TABLE visits
 (

db/migrations/chlocal/000004_create_neighbors_table.up.sql

@@ -1,3 +1,5 @@
+-- DO NOT EDIT: This file was generated with: just generate-local-clickhouse-migrations
+
 CREATE TABLE neighbors
 (
     -- identifies the crawl that produced this neighbor mapping

db/migrations/chlocal/000006_add_missing_maddrs_column_to_visits_table.down.sql

@@ -0,0 +1,4 @@
+-- DO NOT EDIT: This file was generated with: just generate-local-clickhouse-migrations
+
+ALTER TABLE visits
+    DROP COLUMN listen_maddrs;

db/migrations/chlocal/000006_add_missing_maddrs_column_to_visits_table.up.sql

@@ -0,0 +1,4 @@
+-- DO NOT EDIT: This file was generated with: just generate-local-clickhouse-migrations
+
+ALTER TABLE visits
+    ADD COLUMN listen_maddrs Array(String) DEFAULT [] AFTER extra_maddrs;

discv4/crawler.go

@@ -157,6 +157,7 @@ func (c *Crawler) Work(ctx context.Context, task PeerInfo) (core.CrawlResult[Pee
 		DialMaddrs:          devp2pResult.DialMaddrs,
 		FilteredMaddrs:      filteredMaddrs,
 		ExtraMaddrs:         []ma.Multiaddr{},
+		ListenMaddrs:        []ma.Multiaddr{},
 		ConnectMaddr:        connectMaddr,
 		DialErrors:          db.MaddrErrors(devp2pResult.DialMaddrs, devp2pResult.ConnectError),
 		ConnectError:        devp2pResult.ConnectError,

discv4/driver_crawler.go

@@ -155,7 +155,6 @@ type CrawlDriverConfig struct {
 	DialTimeout      time.Duration
 	BootstrapPeers   []*enode.Node
 	AddrDialType     config.AddrType
-	AddrTrackType    config.AddrType
 	MeterProvider    metric.MeterProvider
 	TracerProvider   trace.TracerProvider
 	LogErrors        bool
@@ -175,9 +174,7 @@ func (cfg *CrawlDriverConfig) CrawlerConfig() *CrawlerConfig {
 }
 
 func (cfg *CrawlDriverConfig) WriterConfig() *core.CrawlWriterConfig {
-	return &core.CrawlWriterConfig{
-		AddrTrackType: cfg.AddrTrackType,
-	}
+	return &core.CrawlWriterConfig{}
 }
 
 type CrawlDriver struct {