collectors: tolerate DeadlineExceeded in all scrape-cycle RPCs

collectors/channels_collector.go

@@ -202,7 +202,12 @@ func NewChannelsCollector(lnd lndclient.LightningClient, errChan chan<- error,
 		for {
 			err := collector.refreshClosedChannelsCache()
 			if err != nil {
-				errChan <- err
+				Logger.Errorf("ChannelsCollector refreshClosedChannelsCache "+
+					"failed with: %v", err)
+
+				if !IsDeadlineExceeded(err) {
+					errChan <- err
+				}
 			}
 
 			select {
@@ -293,8 +298,14 @@ func (c *ChannelsCollector) Collect(ch chan<- prometheus.Metric) {
 	// pending channel balances.
 	chanBalResp, err := c.lnd.ChannelBalance(context.Background())
 	if err != nil {
-		c.errChan <- fmt.Errorf("ChannelsCollector ChannelBalance "+
-			"failed with: %v", err)
+		errWithContext := fmt.Errorf("ChannelsCollector ChannelBalance "+
+			"failed with: %w", err)
+		Logger.Error(errWithContext)
+
+		if !IsDeadlineExceeded(err) {
+			c.errChan <- errWithContext
+		}
+
 		return
 	}
 
@@ -341,8 +352,14 @@ func (c *ChannelsCollector) Collect(ch chan<- prometheus.Metric) {
 	// as well as the number of pending HTLCs.
 	listChannelsResp, err := c.lnd.ListChannels(context.Background(), false, false)
 	if err != nil {
-		c.errChan <- fmt.Errorf("ChannelsCollector ListChannels "+
-			"failed with: %v", err)
+		errWithContext := fmt.Errorf("ChannelsCollector ListChannels "+
+			"failed with: %w", err)
+		Logger.Error(errWithContext)
+
+		if !IsDeadlineExceeded(err) {
+			c.errChan <- errWithContext
+		}
+
 		return
 	}
 
@@ -452,8 +469,14 @@ func (c *ChannelsCollector) Collect(ch chan<- prometheus.Metric) {
 	// Get the list of pending channels
 	pendingChannelsResp, err := c.lnd.PendingChannels(context.Background())
 	if err != nil {
-		c.errChan <- fmt.Errorf("ChannelsCollector PendingChannels "+
-			"failed with: %v", err)
+		errWithContext := fmt.Errorf("ChannelsCollector PendingChannels "+
+			"failed with: %w", err)
+		Logger.Error(errWithContext)
+
+		if !IsDeadlineExceeded(err) {
+			c.errChan <- errWithContext
+		}
+
 		return
 	}
 	ch <- prometheus.MustNewConstMetric(
@@ -539,8 +562,14 @@ func (c *ChannelsCollector) Collect(ch chan<- prometheus.Metric) {
 	// Get all remote policies
 	remotePolicies, err := c.getRemotePolicies(getInfoResp.IdentityPubkey)
 	if err != nil {
-		c.errChan <- fmt.Errorf("ChannelsCollector getRemotePolicies "+
-			"failed with: %v", err)
+		errWithContext := fmt.Errorf("ChannelsCollector getRemotePolicies "+
+			"failed with: %w", err)
+		Logger.Error(errWithContext)
+
+		if !IsDeadlineExceeded(err) {
+			c.errChan <- errWithContext
+		}
+
 		return
 	}
 

collectors/graph_collector.go

@@ -322,8 +322,14 @@ func (g *GraphCollector) Describe(ch chan<- *prometheus.Desc) {
 func (g *GraphCollector) Collect(ch chan<- prometheus.Metric) {
 	resp, err := g.lnd.DescribeGraph(context.Background(), false)
 	if err != nil {
-		g.errChan <- fmt.Errorf("GraphCollector DescribeGraph failed "+
-			"with: %v", err)
+		errWithContext := fmt.Errorf("GraphCollector DescribeGraph failed "+
+			"with: %w", err)
+		Logger.Error(errWithContext)
+
+		if !IsDeadlineExceeded(err) {
+			g.errChan <- errWithContext
+		}
+
 		return
 	}
 
@@ -340,8 +346,14 @@ func (g *GraphCollector) Collect(ch chan<- prometheus.Metric) {
 
 	networkInfo, err := g.lnd.NetworkInfo(context.Background())
 	if err != nil {
-		g.errChan <- fmt.Errorf("GraphCollector NetworkInfo failed "+
-			"with: %v", err)
+		errWithContext := fmt.Errorf("GraphCollector NetworkInfo failed "+
+			"with: %w", err)
+		Logger.Error(errWithContext)
+
+		if !IsDeadlineExceeded(err) {
+			g.errChan <- errWithContext
+		}
+
 		return
 	}
 

collectors/info_collector.go

@@ -50,8 +50,17 @@ func (c *InfoCollector) Describe(ch chan<- *prometheus.Desc) {
 func (c *InfoCollector) Collect(ch chan<- prometheus.Metric) {
 	resp, err := c.lnd.GetInfo(context.Background())
 	if err != nil {
-		c.errChan <- fmt.Errorf("InfoCollector GetInfo failed with: "+
-			"%v", err)
+		errWithContext := fmt.Errorf("InfoCollector GetInfo failed with: "+
+			"%w", err)
+		Logger.Error(errWithContext)
+
+		// A deadline exceeded is expected if lnd is temporarily slow
+		// to respond (e.g. due to database load). We just skip this
+		// scrape cycle and let Prometheus retry on the next interval.
+		if !IsDeadlineExceeded(err) {
+			c.errChan <- errWithContext
+		}
+
 		return
 	}
 

collectors/wallet_collector.go

@@ -115,8 +115,14 @@ func (u *WalletCollector) Collect(ch chan<- prometheus.Metric) {
 		context.Background(), 0, math.MaxInt32,
 	)
 	if err != nil {
-		u.errChan <- fmt.Errorf("WalletCollector ListUnspent failed "+
-			"with: %v", err)
+		errWithContext := fmt.Errorf("WalletCollector ListUnspent failed "+
+			"with: %w", err)
+		Logger.Error(errWithContext)
+
+		if !IsDeadlineExceeded(err) {
+			u.errChan <- errWithContext
+		}
+
 		return
 	}
 
@@ -171,8 +177,14 @@ func (u *WalletCollector) Collect(ch chan<- prometheus.Metric) {
 	// balance at this instance.
 	walletBal, err := u.lnd.Client.WalletBalance(context.Background())
 	if err != nil {
-		u.errChan <- fmt.Errorf("WalletCollector WalletBalance "+
-			"failed with: %v", err)
+		errWithContext := fmt.Errorf("WalletCollector WalletBalance "+
+			"failed with: %w", err)
+		Logger.Error(errWithContext)
+
+		if !IsDeadlineExceeded(err) {
+			u.errChan <- errWithContext
+		}
+
 		return
 	}
 
@@ -187,8 +199,14 @@ func (u *WalletCollector) Collect(ch chan<- prometheus.Metric) {
 
 	accounts, err := u.lnd.WalletKit.ListAccounts(context.Background(), "", 0)
 	if err != nil {
-		u.errChan <- fmt.Errorf("WalletCollector ListAccounts"+
-			"failed with: %v", err)
+		errWithContext := fmt.Errorf("WalletCollector ListAccounts "+
+			"failed with: %w", err)
+		Logger.Error(errWithContext)
+
+		if !IsDeadlineExceeded(err) {
+			u.errChan <- errWithContext
+		}
+
 		return
 	}
 

collectors/wt_client_collector.go

@@ -75,8 +75,14 @@ func (c *WtClientCollector) Collect(ch chan<- prometheus.Metric) {
 			return
 		}
 
-		c.errChan <- fmt.Errorf("WtClientCollector ListTowers failed "+
-			"with: %v", err)
+		errWithContext := fmt.Errorf("WtClientCollector ListTowers failed "+
+			"with: %w", err)
+		Logger.Error(errWithContext)
+
+		if !IsDeadlineExceeded(err) {
+			c.errChan <- errWithContext
+		}
+
 		return
 	}