diff --git a/collectors/channels_collector.go b/collectors/channels_collector.go index f2ad2ea..db56032 100644 --- a/collectors/channels_collector.go +++ b/collectors/channels_collector.go @@ -202,7 +202,12 @@ func NewChannelsCollector(lnd lndclient.LightningClient, errChan chan<- error, for { err := collector.refreshClosedChannelsCache() if err != nil { - errChan <- err + Logger.Errorf("ChannelsCollector refreshClosedChannelsCache "+ + "failed with: %v", err) + + if !IsDeadlineExceeded(err) { + errChan <- err + } } select { @@ -293,8 +298,14 @@ func (c *ChannelsCollector) Collect(ch chan<- prometheus.Metric) { // pending channel balances. chanBalResp, err := c.lnd.ChannelBalance(context.Background()) if err != nil { - c.errChan <- fmt.Errorf("ChannelsCollector ChannelBalance "+ - "failed with: %v", err) + errWithContext := fmt.Errorf("ChannelsCollector ChannelBalance "+ + "failed with: %w", err) + Logger.Error(errWithContext) + + if !IsDeadlineExceeded(err) { + c.errChan <- errWithContext + } + return } @@ -341,8 +352,14 @@ func (c *ChannelsCollector) Collect(ch chan<- prometheus.Metric) { // as well as the number of pending HTLCs. listChannelsResp, err := c.lnd.ListChannels(context.Background(), false, false) if err != nil { - c.errChan <- fmt.Errorf("ChannelsCollector ListChannels "+ - "failed with: %v", err) + errWithContext := fmt.Errorf("ChannelsCollector ListChannels "+ + "failed with: %w", err) + Logger.Error(errWithContext) + + if !IsDeadlineExceeded(err) { + c.errChan <- errWithContext + } + return } @@ -452,8 +469,14 @@ func (c *ChannelsCollector) Collect(ch chan<- prometheus.Metric) { // Get the list of pending channels pendingChannelsResp, err := c.lnd.PendingChannels(context.Background()) if err != nil { - c.errChan <- fmt.Errorf("ChannelsCollector PendingChannels "+ - "failed with: %v", err) + errWithContext := fmt.Errorf("ChannelsCollector PendingChannels "+ + "failed with: %w", err) + Logger.Error(errWithContext) + + if !IsDeadlineExceeded(err) { + c.errChan <- errWithContext + } + return } ch <- prometheus.MustNewConstMetric( @@ -539,8 +562,14 @@ func (c *ChannelsCollector) Collect(ch chan<- prometheus.Metric) { // Get all remote policies remotePolicies, err := c.getRemotePolicies(getInfoResp.IdentityPubkey) if err != nil { - c.errChan <- fmt.Errorf("ChannelsCollector getRemotePolicies "+ - "failed with: %v", err) + errWithContext := fmt.Errorf("ChannelsCollector getRemotePolicies "+ + "failed with: %w", err) + Logger.Error(errWithContext) + + if !IsDeadlineExceeded(err) { + c.errChan <- errWithContext + } + return } diff --git a/collectors/graph_collector.go b/collectors/graph_collector.go index 4537ec6..de7956b 100644 --- a/collectors/graph_collector.go +++ b/collectors/graph_collector.go @@ -322,8 +322,14 @@ func (g *GraphCollector) Describe(ch chan<- *prometheus.Desc) { func (g *GraphCollector) Collect(ch chan<- prometheus.Metric) { resp, err := g.lnd.DescribeGraph(context.Background(), false) if err != nil { - g.errChan <- fmt.Errorf("GraphCollector DescribeGraph failed "+ - "with: %v", err) + errWithContext := fmt.Errorf("GraphCollector DescribeGraph failed "+ + "with: %w", err) + Logger.Error(errWithContext) + + if !IsDeadlineExceeded(err) { + g.errChan <- errWithContext + } + return } @@ -340,8 +346,14 @@ func (g *GraphCollector) Collect(ch chan<- prometheus.Metric) { networkInfo, err := g.lnd.NetworkInfo(context.Background()) if err != nil { - g.errChan <- fmt.Errorf("GraphCollector NetworkInfo failed "+ - "with: %v", err) + errWithContext := fmt.Errorf("GraphCollector NetworkInfo failed "+ + "with: %w", err) + Logger.Error(errWithContext) + + if !IsDeadlineExceeded(err) { + g.errChan <- errWithContext + } + return } diff --git a/collectors/info_collector.go b/collectors/info_collector.go index a5384b6..3765647 100644 --- a/collectors/info_collector.go +++ b/collectors/info_collector.go @@ -50,8 +50,17 @@ func (c *InfoCollector) Describe(ch chan<- *prometheus.Desc) { func (c *InfoCollector) Collect(ch chan<- prometheus.Metric) { resp, err := c.lnd.GetInfo(context.Background()) if err != nil { - c.errChan <- fmt.Errorf("InfoCollector GetInfo failed with: "+ - "%v", err) + errWithContext := fmt.Errorf("InfoCollector GetInfo failed with: "+ + "%w", err) + Logger.Error(errWithContext) + + // A deadline exceeded is expected if lnd is temporarily slow + // to respond (e.g. due to database load). We just skip this + // scrape cycle and let Prometheus retry on the next interval. + if !IsDeadlineExceeded(err) { + c.errChan <- errWithContext + } + return } diff --git a/collectors/wallet_collector.go b/collectors/wallet_collector.go index c787129..fbcfb2f 100644 --- a/collectors/wallet_collector.go +++ b/collectors/wallet_collector.go @@ -115,8 +115,14 @@ func (u *WalletCollector) Collect(ch chan<- prometheus.Metric) { context.Background(), 0, math.MaxInt32, ) if err != nil { - u.errChan <- fmt.Errorf("WalletCollector ListUnspent failed "+ - "with: %v", err) + errWithContext := fmt.Errorf("WalletCollector ListUnspent failed "+ + "with: %w", err) + Logger.Error(errWithContext) + + if !IsDeadlineExceeded(err) { + u.errChan <- errWithContext + } + return } @@ -171,8 +177,14 @@ func (u *WalletCollector) Collect(ch chan<- prometheus.Metric) { // balance at this instance. walletBal, err := u.lnd.Client.WalletBalance(context.Background()) if err != nil { - u.errChan <- fmt.Errorf("WalletCollector WalletBalance "+ - "failed with: %v", err) + errWithContext := fmt.Errorf("WalletCollector WalletBalance "+ + "failed with: %w", err) + Logger.Error(errWithContext) + + if !IsDeadlineExceeded(err) { + u.errChan <- errWithContext + } + return } @@ -187,8 +199,14 @@ func (u *WalletCollector) Collect(ch chan<- prometheus.Metric) { accounts, err := u.lnd.WalletKit.ListAccounts(context.Background(), "", 0) if err != nil { - u.errChan <- fmt.Errorf("WalletCollector ListAccounts"+ - "failed with: %v", err) + errWithContext := fmt.Errorf("WalletCollector ListAccounts "+ + "failed with: %w", err) + Logger.Error(errWithContext) + + if !IsDeadlineExceeded(err) { + u.errChan <- errWithContext + } + return } diff --git a/collectors/wt_client_collector.go b/collectors/wt_client_collector.go index 6bcd44d..8525d18 100644 --- a/collectors/wt_client_collector.go +++ b/collectors/wt_client_collector.go @@ -75,8 +75,14 @@ func (c *WtClientCollector) Collect(ch chan<- prometheus.Metric) { return } - c.errChan <- fmt.Errorf("WtClientCollector ListTowers failed "+ - "with: %v", err) + errWithContext := fmt.Errorf("WtClientCollector ListTowers failed "+ + "with: %w", err) + Logger.Error(errWithContext) + + if !IsDeadlineExceeded(err) { + c.errChan <- errWithContext + } + return }