Fix for load-watcher crash in dev52 when datadog returned 2 grouptags for a series

Dockerfile

@@ -1,10 +1,24 @@
 ARG BUILD_SPEC="build.amd64"
+# Builder
 FROM golang:1.22
 
+# Optional corporate CA (BuildKit secret: --secret id=corp_ca,src=paypal-crypto-root-ca.crt)
+RUN --mount=type=secret,id=corp_ca,required=false,target=/tmp/corp-ca.crt \
+    if [ -f /tmp/corp-ca.crt ]; then \
+      cp /tmp/corp-ca.crt /usr/local/share/ca-certificates/corp-ca.crt && update-ca-certificates; \
+    fi
+
+ARG GOPROXY=direct
+ARG GOSUMDB=off
+ENV GOPROXY=${GOPROXY}
+ENV GOSUMDB=${GOSUMDB}
+
 WORKDIR /go/src/github.com/paypal/load-watcher
 COPY . .
+
 RUN make ${BUILD_SPEC}
 
+# Runtime
 FROM alpine:3.12
 
 COPY --from=0 /go/src/github.com/paypal/load-watcher/bin/load-watcher /bin/load-watcher

Makefile

@@ -24,7 +24,7 @@ build:
 
 .PHONY:	build.amd64
 build.amd64:
-	GOOS=linux$(BUILDENVVAR) GOARCH=amd64 go build -o bin/load-watcher main.go
+	GOOS=linux $(BUILDENVVAR) GOARCH=amd64 go build -o bin/load-watcher main.go
 
 .PHONY:	build.arm64
 build.arm64:

README.md

@@ -8,6 +8,7 @@ The following metrics provider clients are currently supported:
 1) SignalFx
 2) Kubernetes Metrics Server
 3) Prometheus
+4) Datadog
 
 These clients fetch CPU usage currently, support for other resources will be added later as needed.
 
@@ -19,10 +20,39 @@ The default `main.go` is configured to watch Kubernetes Metrics Server.
 You can change this to any available metrics provider in `pkg/metricsprovider`.
 To build a client for new metrics provider, you will need to implement `FetcherClient` interface.
 
-From the root folder, run the following commands to build docker image of load watcher, tag it and push to your docker repository:
+## Building behind a corporate TLS proxy
+
+Some corporate environments may fail during Go module downloads or base image pulls with errors such as:
+
+```text
+x509: certificate signed by unknown authority
+
+Export the corporate root CA certificate:
+
+security find-certificate -c "PayPal Crypto Mgt Corp Root CA" -p \
+  /Library/Keychains/System.keychain > paypal-crypto-root-ca.crt
+
+From the root folder, build the load-watcher image using Docker BuildKit:
+
+docker buildx build \
+  --platform linux/amd64 \
+  --secret id=corp_ca,src="$(pwd)/paypal-crypto-root-ca.crt" \
+  --build-arg BUILD_SPEC=build.amd64 \
+  --build-arg GOPROXY=direct \
+  --build-arg GOSUMDB=off \
+  --load \
+  -t load-watcher:<version> \
+  .
+
+Notes:
+
+corp_ca is an optional BuildKit secret used to install the corporate root CA inside the builder container.
+GOPROXY=direct and GOSUMDB=off can help in environments where access to proxy.golang.org is restricted.
+linux/arm64 builds succeed locally. linux/amd64 builds on Apple Silicon/Rancher Desktop can intermittently hit Go compiler segmentation faults under emulation, so amd64 images should be validated on CI or an amd64 runner.
+
+Tag the docker image and push to your docker repository:
 
 ```
-docker build -t load-watcher:<version> .
 docker tag load-watcher:<version> <your-docker-repo>:<version>
 docker push <your-docker-repo>
 ```
@@ -43,6 +73,8 @@ This will return metrics for all nodes. A query parameter to filter by host can
 
 - To use the SignalFx client, please configure environment variables `METRICS_PROVIDER_NAME`, `METRICS_PROVIDER_ADDRESS` and `METRICS_PROVIDER_TOKEN` to `SignalFx`, SignalFx address and auth token respectively. Default value of address set is `https://api.signalfx.com` for SignalFx client.
 
+- To use the Datadog client, configure `METRICS_PROVIDER_NAME`, `METRICS_PROVIDER_ADDRESS`, `METRICS_PROVIDER_TOKEN`, `METRICS_PROVIDER_APP_KEY`, `DATADOG_ClUSTER_FILTER`, and `DATADOG_CLUSTER_NAME`.
+
 ## Deploy `load-watcher` as a service
 To deploy `load-watcher` as a monitoring service in your Kubernetes cluster, you should replace the values in the `[]` with your own cluster monitoring stack and then you can run the following.
 ```bash

pkg/watcher/internal/metricsprovider/datadog.go

@@ -254,11 +254,18 @@ func getMetricsFromTimeSeriesResponse(resp datadogV2.TimeseriesFormulaQueryRespo
 		return metrics, errors.New("No values from timeseries attributes.")
 	}
 
+	// The number of series should match with the number of values since each series corresponds to
+	// an array of values across time intervals. If they don't match, return error and empty metrics.
+	if len(*timeSeriesDataSeriesPtr) != len(*timeSeriesDataValuesPtr) {
+		errMsg := "Number of series does not match number of values in timeseries response."
+		log.Error(errMsg)
+		return metrics, errors.New(errMsg)
+	}
+
 	hosts := make([]string, len(*timeSeriesDataSeriesPtr))
 	isCPU := make([]bool, len(*timeSeriesDataSeriesPtr))
-	index := 0
 	// Populate host array and corresponding query index
-	for _, timeSeriesDataSeries := range *timeSeriesDataSeriesPtr {
+	for index, timeSeriesDataSeries := range *timeSeriesDataSeriesPtr {
 		queryIndex, ok := timeSeriesDataSeries.GetQueryIndexOk()
 		if !ok {
 			log.Error("Error when getting query index from timeseries series.")
@@ -268,7 +275,6 @@ func getMetricsFromTimeSeriesResponse(resp datadogV2.TimeseriesFormulaQueryRespo
 			log.Error("No query index from timeseries series.")
 			continue
 		}
-		isCPU[index] = (*queryIndex == 0)
 		groupTagsPtr, ok := timeSeriesDataSeries.GetGroupTagsOk()
 		if !ok {
 			log.Error("Error when getting group tags from timeseries series.")
@@ -278,20 +284,37 @@ func getMetricsFromTimeSeriesResponse(resp datadogV2.TimeseriesFormulaQueryRespo
 			log.Error("No group tags from timeseries series.")
 			continue
 		}
-		for _, groupTags := range *groupTagsPtr {
-			log.Debugf("%v\n", groupTags)
-			hosts[index] = getHostName(groupTags)
-			index++
+
+		host := ""
+		// Iterate through group tags to find host tag since group by can be on multiple tags.
+		// We will use the value in host tag as the hostname for the metric. If host tag is
+		// not found, we will log an error and skip this series.
+		for _, groupTag := range *groupTagsPtr {
+			log.Debugf("%v\n", groupTag)
+			// ignore host:not-set since datadog returns this value when host tag is missing,
+			// and we want to avoid using not-set as a host identifier
+			if strings.HasPrefix(groupTag, "host:") {
+				parsedHost := getHostName(groupTag)
+				if parsedHost != "" && parsedHost != "not-set" {
+					host = parsedHost
+					break
+				}
+			}
+		}
+		// If host tag is not found, log an error and skip this series since we won't be able
+		// to identify which host this metric belongs to.
+		if host == "" {
+			log.Errorf(
+				"No valid host tag found. queryIndex=%d groupTags=%v",
+				*queryIndex,
+				*groupTagsPtr)
+			continue
 		}
-	}
 
-	if len(hosts) != len(*timeSeriesDataValuesPtr) {
-		errMsg := "Number of group tags does not match number of values in timeseries series."
-		log.Error(errMsg)
-		return metrics, errors.New(errMsg)
+		hosts[index] = host
+		isCPU[index] = (*queryIndex == 0)
 	}
-	// Find the average across returned values per 1 minute resolution
-	// Build a hostname map of array of metrics [CPU, memory]
+
 	hostIndex := 0
 	for _, timesSeriesDataValues := range *timeSeriesDataValuesPtr {
 		sum := 0.0
@@ -302,6 +325,16 @@ func getMetricsFromTimeSeriesResponse(resp datadogV2.TimeseriesFormulaQueryRespo
 				count += 1
 			}
 		}
+		// If count is 0, it means all values for this series are nil. This can happen
+		// when there is no data for the host during the time window. We will log an
+		// error and skip this host since we don't want to add a metric with value 0
+		// which can be misleading.
+		if count == 0 {
+			log.Errorf("No non-nil metric values for host %s", hosts[hostIndex])
+			hostIndex++
+			continue
+		}
+
 		fetchedMetric := watcher.Metric{Value: sum / count}
 		addDatadogMetadata(&fetchedMetric, isCPU[hostIndex])
 		metrics[hosts[hostIndex]] = append(metrics[hosts[hostIndex]], fetchedMetric)