From 0a2570a24c701727b12c6733ea8dab1f279593f4 Mon Sep 17 00:00:00 2001 From: ziggie Date: Mon, 23 Feb 2026 07:50:20 +0100 Subject: [PATCH] collectors: tolerate DeadlineExceeded in all scrape-cycle RPCs GetInfo and other RPC calls are known to occasionally take longer than expected due to database interactions on the lnd side. A transient DeadlineExceeded during a Prometheus scrape should not cause lndmon to exit. All collectors now log the error and skip the current scrape cycle instead of propagating to errChan, mirroring the existing behaviour in ChainCollector and PeerCollector. Affected collectors: InfoCollector, ChannelsCollector, WalletCollector, GraphCollector, WtClientCollector. --- collectors/channels_collector.go | 47 +++++++++++++++++++++++++------ collectors/graph_collector.go | 20 ++++++++++--- collectors/info_collector.go | 13 +++++++-- collectors/wallet_collector.go | 30 ++++++++++++++++---- collectors/wt_client_collector.go | 10 +++++-- 5 files changed, 97 insertions(+), 23 deletions(-) diff --git a/collectors/channels_collector.go b/collectors/channels_collector.go index f2ad2ea..db56032 100644 --- a/collectors/channels_collector.go +++ b/collectors/channels_collector.go @@ -202,7 +202,12 @@ func NewChannelsCollector(lnd lndclient.LightningClient, errChan chan<- error, for { err := collector.refreshClosedChannelsCache() if err != nil { - errChan <- err + Logger.Errorf("ChannelsCollector refreshClosedChannelsCache "+ + "failed with: %v", err) + + if !IsDeadlineExceeded(err) { + errChan <- err + } } select { @@ -293,8 +298,14 @@ func (c *ChannelsCollector) Collect(ch chan<- prometheus.Metric) { // pending channel balances. chanBalResp, err := c.lnd.ChannelBalance(context.Background()) if err != nil { - c.errChan <- fmt.Errorf("ChannelsCollector ChannelBalance "+ - "failed with: %v", err) + errWithContext := fmt.Errorf("ChannelsCollector ChannelBalance "+ + "failed with: %w", err) + Logger.Error(errWithContext) + + if !IsDeadlineExceeded(err) { + c.errChan <- errWithContext + } + return } @@ -341,8 +352,14 @@ func (c *ChannelsCollector) Collect(ch chan<- prometheus.Metric) { // as well as the number of pending HTLCs. listChannelsResp, err := c.lnd.ListChannels(context.Background(), false, false) if err != nil { - c.errChan <- fmt.Errorf("ChannelsCollector ListChannels "+ - "failed with: %v", err) + errWithContext := fmt.Errorf("ChannelsCollector ListChannels "+ + "failed with: %w", err) + Logger.Error(errWithContext) + + if !IsDeadlineExceeded(err) { + c.errChan <- errWithContext + } + return } @@ -452,8 +469,14 @@ func (c *ChannelsCollector) Collect(ch chan<- prometheus.Metric) { // Get the list of pending channels pendingChannelsResp, err := c.lnd.PendingChannels(context.Background()) if err != nil { - c.errChan <- fmt.Errorf("ChannelsCollector PendingChannels "+ - "failed with: %v", err) + errWithContext := fmt.Errorf("ChannelsCollector PendingChannels "+ + "failed with: %w", err) + Logger.Error(errWithContext) + + if !IsDeadlineExceeded(err) { + c.errChan <- errWithContext + } + return } ch <- prometheus.MustNewConstMetric( @@ -539,8 +562,14 @@ func (c *ChannelsCollector) Collect(ch chan<- prometheus.Metric) { // Get all remote policies remotePolicies, err := c.getRemotePolicies(getInfoResp.IdentityPubkey) if err != nil { - c.errChan <- fmt.Errorf("ChannelsCollector getRemotePolicies "+ - "failed with: %v", err) + errWithContext := fmt.Errorf("ChannelsCollector getRemotePolicies "+ + "failed with: %w", err) + Logger.Error(errWithContext) + + if !IsDeadlineExceeded(err) { + c.errChan <- errWithContext + } + return } diff --git a/collectors/graph_collector.go b/collectors/graph_collector.go index 4537ec6..de7956b 100644 --- a/collectors/graph_collector.go +++ b/collectors/graph_collector.go @@ -322,8 +322,14 @@ func (g *GraphCollector) Describe(ch chan<- *prometheus.Desc) { func (g *GraphCollector) Collect(ch chan<- prometheus.Metric) { resp, err := g.lnd.DescribeGraph(context.Background(), false) if err != nil { - g.errChan <- fmt.Errorf("GraphCollector DescribeGraph failed "+ - "with: %v", err) + errWithContext := fmt.Errorf("GraphCollector DescribeGraph failed "+ + "with: %w", err) + Logger.Error(errWithContext) + + if !IsDeadlineExceeded(err) { + g.errChan <- errWithContext + } + return } @@ -340,8 +346,14 @@ func (g *GraphCollector) Collect(ch chan<- prometheus.Metric) { networkInfo, err := g.lnd.NetworkInfo(context.Background()) if err != nil { - g.errChan <- fmt.Errorf("GraphCollector NetworkInfo failed "+ - "with: %v", err) + errWithContext := fmt.Errorf("GraphCollector NetworkInfo failed "+ + "with: %w", err) + Logger.Error(errWithContext) + + if !IsDeadlineExceeded(err) { + g.errChan <- errWithContext + } + return } diff --git a/collectors/info_collector.go b/collectors/info_collector.go index a5384b6..3765647 100644 --- a/collectors/info_collector.go +++ b/collectors/info_collector.go @@ -50,8 +50,17 @@ func (c *InfoCollector) Describe(ch chan<- *prometheus.Desc) { func (c *InfoCollector) Collect(ch chan<- prometheus.Metric) { resp, err := c.lnd.GetInfo(context.Background()) if err != nil { - c.errChan <- fmt.Errorf("InfoCollector GetInfo failed with: "+ - "%v", err) + errWithContext := fmt.Errorf("InfoCollector GetInfo failed with: "+ + "%w", err) + Logger.Error(errWithContext) + + // A deadline exceeded is expected if lnd is temporarily slow + // to respond (e.g. due to database load). We just skip this + // scrape cycle and let Prometheus retry on the next interval. + if !IsDeadlineExceeded(err) { + c.errChan <- errWithContext + } + return } diff --git a/collectors/wallet_collector.go b/collectors/wallet_collector.go index c787129..fbcfb2f 100644 --- a/collectors/wallet_collector.go +++ b/collectors/wallet_collector.go @@ -115,8 +115,14 @@ func (u *WalletCollector) Collect(ch chan<- prometheus.Metric) { context.Background(), 0, math.MaxInt32, ) if err != nil { - u.errChan <- fmt.Errorf("WalletCollector ListUnspent failed "+ - "with: %v", err) + errWithContext := fmt.Errorf("WalletCollector ListUnspent failed "+ + "with: %w", err) + Logger.Error(errWithContext) + + if !IsDeadlineExceeded(err) { + u.errChan <- errWithContext + } + return } @@ -171,8 +177,14 @@ func (u *WalletCollector) Collect(ch chan<- prometheus.Metric) { // balance at this instance. walletBal, err := u.lnd.Client.WalletBalance(context.Background()) if err != nil { - u.errChan <- fmt.Errorf("WalletCollector WalletBalance "+ - "failed with: %v", err) + errWithContext := fmt.Errorf("WalletCollector WalletBalance "+ + "failed with: %w", err) + Logger.Error(errWithContext) + + if !IsDeadlineExceeded(err) { + u.errChan <- errWithContext + } + return } @@ -187,8 +199,14 @@ func (u *WalletCollector) Collect(ch chan<- prometheus.Metric) { accounts, err := u.lnd.WalletKit.ListAccounts(context.Background(), "", 0) if err != nil { - u.errChan <- fmt.Errorf("WalletCollector ListAccounts"+ - "failed with: %v", err) + errWithContext := fmt.Errorf("WalletCollector ListAccounts "+ + "failed with: %w", err) + Logger.Error(errWithContext) + + if !IsDeadlineExceeded(err) { + u.errChan <- errWithContext + } + return } diff --git a/collectors/wt_client_collector.go b/collectors/wt_client_collector.go index 6bcd44d..8525d18 100644 --- a/collectors/wt_client_collector.go +++ b/collectors/wt_client_collector.go @@ -75,8 +75,14 @@ func (c *WtClientCollector) Collect(ch chan<- prometheus.Metric) { return } - c.errChan <- fmt.Errorf("WtClientCollector ListTowers failed "+ - "with: %v", err) + errWithContext := fmt.Errorf("WtClientCollector ListTowers failed "+ + "with: %w", err) + Logger.Error(errWithContext) + + if !IsDeadlineExceeded(err) { + c.errChan <- errWithContext + } + return }