From eb4106113808d8ebe4dba4396d4be0e32ea35631 Mon Sep 17 00:00:00 2001 From: lawwong Date: Wed, 10 Jun 2026 12:10:34 -0700 Subject: [PATCH 1/2] Make file change, dockerfile change to add PayPal root CA cert for docker build, fix for crash in dev52 when datadog returned 2 grouptags for a series, with [host: host:no-set] --- Dockerfile | 12 +++++ Makefile | 2 +- README.md | 5 ++ .../internal/metricsprovider/datadog.go | 53 ++++++++++++++----- 4 files changed, 57 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1eb10ed..5d537ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,24 @@ ARG BUILD_SPEC="build.amd64" +# Builder FROM golang:1.22 +COPY paypal-crypto-root-ca.crt /usr/local/share/ca-certificates/paypal-crypto-root-ca.crt + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates git \ + && update-ca-certificates + +ENV GOPROXY=direct + WORKDIR /go/src/github.com/paypal/load-watcher COPY . . RUN make ${BUILD_SPEC} +# Runtime FROM alpine:3.12 +RUN apk add --no-cache ca-certificates && update-ca-certificates + COPY --from=0 /go/src/github.com/paypal/load-watcher/bin/load-watcher /bin/load-watcher CMD ["/bin/load-watcher"] diff --git a/Makefile b/Makefile index d2fdec9..aaf149e 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ build: .PHONY: build.amd64 build.amd64: - GOOS=linux$(BUILDENVVAR) GOARCH=amd64 go build -o bin/load-watcher main.go + GOOS=linux $(BUILDENVVAR) GOARCH=amd64 go build -o bin/load-watcher main.go .PHONY: build.arm64 build.arm64: diff --git a/README.md b/README.md index 30d15a4..89b9ec2 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,11 @@ The default `main.go` is configured to watch Kubernetes Metrics Server. You can change this to any available metrics provider in `pkg/metricsprovider`. To build a client for new metrics provider, you will need to implement `FetcherClient` interface. +Instructions to export PayPal root CA from macOS Keychain (required for docker build) +1) cd to root directory +2) run the following: +security find-certificate -c "PayPal Crypto Mgt Corp Root CA" -p /Library/Keychains/System.keychain > paypal-crypto-root-ca.crt + From the root folder, run the following commands to build docker image of load watcher, tag it and push to your docker repository: ``` diff --git a/pkg/watcher/internal/metricsprovider/datadog.go b/pkg/watcher/internal/metricsprovider/datadog.go index 40338f2..21ec33a 100644 --- a/pkg/watcher/internal/metricsprovider/datadog.go +++ b/pkg/watcher/internal/metricsprovider/datadog.go @@ -254,11 +254,17 @@ func getMetricsFromTimeSeriesResponse(resp datadogV2.TimeseriesFormulaQueryRespo return metrics, errors.New("No values from timeseries attributes.") } + // The number of series should match with the number of values since each series corresponds to an array of values across time intervals. If they don't match, return error and empty metrics. + if len(*timeSeriesDataSeriesPtr) != len(*timeSeriesDataValuesPtr) { + errMsg := "Number of series does not match number of values in timeseries response." + log.Error(errMsg) + return metrics, errors.New(errMsg) + } + hosts := make([]string, len(*timeSeriesDataSeriesPtr)) isCPU := make([]bool, len(*timeSeriesDataSeriesPtr)) - index := 0 // Populate host array and corresponding query index - for _, timeSeriesDataSeries := range *timeSeriesDataSeriesPtr { + for index, timeSeriesDataSeries := range *timeSeriesDataSeriesPtr { queryIndex, ok := timeSeriesDataSeries.GetQueryIndexOk() if !ok { log.Error("Error when getting query index from timeseries series.") @@ -268,7 +274,6 @@ func getMetricsFromTimeSeriesResponse(resp datadogV2.TimeseriesFormulaQueryRespo log.Error("No query index from timeseries series.") continue } - isCPU[index] = (*queryIndex == 0) groupTagsPtr, ok := timeSeriesDataSeries.GetGroupTagsOk() if !ok { log.Error("Error when getting group tags from timeseries series.") @@ -278,20 +283,33 @@ func getMetricsFromTimeSeriesResponse(resp datadogV2.TimeseriesFormulaQueryRespo log.Error("No group tags from timeseries series.") continue } - for _, groupTags := range *groupTagsPtr { - log.Debugf("%v\n", groupTags) - hosts[index] = getHostName(groupTags) - index++ + + host := "" + // Iterate through group tags to find host tag since group by can be on multiple tags. We will use the value in host tag as the hostname for the metric. If host tag is not found, we will log an error and skip this series. + for _, groupTag := range *groupTagsPtr { + log.Debugf("%v\n", groupTag) + // ignore host:not-set since datadog returns this value when host tag is missing, and we want to avoid using not-set as a host identifier + if strings.HasPrefix(groupTag, "host:") { + parsedHost := getHostName(groupTag) + if parsedHost != "" && parsedHost != "not-set" { + host = parsedHost + break + } + } + } + // If host tag is not found, log an error and skip this series since we won't be able to identify which host this metric belongs to. + if host == "" { + log.Errorf( + "No valid host tag found. queryIndex=%d groupTags=%v", + *queryIndex, + *groupTagsPtr) + continue } - } - if len(hosts) != len(*timeSeriesDataValuesPtr) { - errMsg := "Number of group tags does not match number of values in timeseries series." - log.Error(errMsg) - return metrics, errors.New(errMsg) + hosts[index] = host + isCPU[index] = (*queryIndex == 0) } - // Find the average across returned values per 1 minute resolution - // Build a hostname map of array of metrics [CPU, memory] + hostIndex := 0 for _, timesSeriesDataValues := range *timeSeriesDataValuesPtr { sum := 0.0 @@ -302,6 +320,13 @@ func getMetricsFromTimeSeriesResponse(resp datadogV2.TimeseriesFormulaQueryRespo count += 1 } } + // If count is 0, it means all values for this series are nil. This can happen when there is no data for the host during the time window. We will log an error and skip this host since we don't want to add a metric with value 0 which can be misleading. + if count == 0 { + log.Errorf("No non-nil metric values for host %s", hosts[hostIndex]) + hostIndex++ + continue + } + fetchedMetric := watcher.Metric{Value: sum / count} addDatadogMetadata(&fetchedMetric, isCPU[hostIndex]) metrics[hosts[hostIndex]] = append(metrics[hosts[hostIndex]], fetchedMetric) From 4e46c05760a1b58fa0af8a25dbdcabc0a19d267e Mon Sep 17 00:00:00 2001 From: lawwong Date: Tue, 16 Jun 2026 20:50:25 -0700 Subject: [PATCH 2/2] Add Datadog support and corporate TLS build docs --- Dockerfile | 18 +++++---- README.md | 39 ++++++++++++++++--- .../internal/metricsprovider/datadog.go | 18 ++++++--- 3 files changed, 56 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5d537ac..46698c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,23 +2,25 @@ ARG BUILD_SPEC="build.amd64" # Builder FROM golang:1.22 -COPY paypal-crypto-root-ca.crt /usr/local/share/ca-certificates/paypal-crypto-root-ca.crt +# Optional corporate CA (BuildKit secret: --secret id=corp_ca,src=paypal-crypto-root-ca.crt) +RUN --mount=type=secret,id=corp_ca,required=false,target=/tmp/corp-ca.crt \ + if [ -f /tmp/corp-ca.crt ]; then \ + cp /tmp/corp-ca.crt /usr/local/share/ca-certificates/corp-ca.crt && update-ca-certificates; \ + fi -RUN apt-get update \ - && apt-get install -y --no-install-recommends ca-certificates git \ - && update-ca-certificates - -ENV GOPROXY=direct +ARG GOPROXY=direct +ARG GOSUMDB=off +ENV GOPROXY=${GOPROXY} +ENV GOSUMDB=${GOSUMDB} WORKDIR /go/src/github.com/paypal/load-watcher COPY . . + RUN make ${BUILD_SPEC} # Runtime FROM alpine:3.12 -RUN apk add --no-cache ca-certificates && update-ca-certificates - COPY --from=0 /go/src/github.com/paypal/load-watcher/bin/load-watcher /bin/load-watcher CMD ["/bin/load-watcher"] diff --git a/README.md b/README.md index 89b9ec2..42cdd75 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ The following metrics provider clients are currently supported: 1) SignalFx 2) Kubernetes Metrics Server 3) Prometheus +4) Datadog These clients fetch CPU usage currently, support for other resources will be added later as needed. @@ -19,15 +20,39 @@ The default `main.go` is configured to watch Kubernetes Metrics Server. You can change this to any available metrics provider in `pkg/metricsprovider`. To build a client for new metrics provider, you will need to implement `FetcherClient` interface. -Instructions to export PayPal root CA from macOS Keychain (required for docker build) -1) cd to root directory -2) run the following: -security find-certificate -c "PayPal Crypto Mgt Corp Root CA" -p /Library/Keychains/System.keychain > paypal-crypto-root-ca.crt +## Building behind a corporate TLS proxy -From the root folder, run the following commands to build docker image of load watcher, tag it and push to your docker repository: +Some corporate environments may fail during Go module downloads or base image pulls with errors such as: + +```text +x509: certificate signed by unknown authority + +Export the corporate root CA certificate: + +security find-certificate -c "PayPal Crypto Mgt Corp Root CA" -p \ + /Library/Keychains/System.keychain > paypal-crypto-root-ca.crt + +From the root folder, build the load-watcher image using Docker BuildKit: + +docker buildx build \ + --platform linux/amd64 \ + --secret id=corp_ca,src="$(pwd)/paypal-crypto-root-ca.crt" \ + --build-arg BUILD_SPEC=build.amd64 \ + --build-arg GOPROXY=direct \ + --build-arg GOSUMDB=off \ + --load \ + -t load-watcher: \ + . + +Notes: + +corp_ca is an optional BuildKit secret used to install the corporate root CA inside the builder container. +GOPROXY=direct and GOSUMDB=off can help in environments where access to proxy.golang.org is restricted. +linux/arm64 builds succeed locally. linux/amd64 builds on Apple Silicon/Rancher Desktop can intermittently hit Go compiler segmentation faults under emulation, so amd64 images should be validated on CI or an amd64 runner. + +Tag the docker image and push to your docker repository: ``` -docker build -t load-watcher: . docker tag load-watcher: : docker push ``` @@ -48,6 +73,8 @@ This will return metrics for all nodes. A query parameter to filter by host can - To use the SignalFx client, please configure environment variables `METRICS_PROVIDER_NAME`, `METRICS_PROVIDER_ADDRESS` and `METRICS_PROVIDER_TOKEN` to `SignalFx`, SignalFx address and auth token respectively. Default value of address set is `https://api.signalfx.com` for SignalFx client. +- To use the Datadog client, configure `METRICS_PROVIDER_NAME`, `METRICS_PROVIDER_ADDRESS`, `METRICS_PROVIDER_TOKEN`, `METRICS_PROVIDER_APP_KEY`, `DATADOG_ClUSTER_FILTER`, and `DATADOG_CLUSTER_NAME`. + ## Deploy `load-watcher` as a service To deploy `load-watcher` as a monitoring service in your Kubernetes cluster, you should replace the values in the `[]` with your own cluster monitoring stack and then you can run the following. ```bash diff --git a/pkg/watcher/internal/metricsprovider/datadog.go b/pkg/watcher/internal/metricsprovider/datadog.go index 21ec33a..8a534a3 100644 --- a/pkg/watcher/internal/metricsprovider/datadog.go +++ b/pkg/watcher/internal/metricsprovider/datadog.go @@ -254,7 +254,8 @@ func getMetricsFromTimeSeriesResponse(resp datadogV2.TimeseriesFormulaQueryRespo return metrics, errors.New("No values from timeseries attributes.") } - // The number of series should match with the number of values since each series corresponds to an array of values across time intervals. If they don't match, return error and empty metrics. + // The number of series should match with the number of values since each series corresponds to + // an array of values across time intervals. If they don't match, return error and empty metrics. if len(*timeSeriesDataSeriesPtr) != len(*timeSeriesDataValuesPtr) { errMsg := "Number of series does not match number of values in timeseries response." log.Error(errMsg) @@ -285,10 +286,13 @@ func getMetricsFromTimeSeriesResponse(resp datadogV2.TimeseriesFormulaQueryRespo } host := "" - // Iterate through group tags to find host tag since group by can be on multiple tags. We will use the value in host tag as the hostname for the metric. If host tag is not found, we will log an error and skip this series. + // Iterate through group tags to find host tag since group by can be on multiple tags. + // We will use the value in host tag as the hostname for the metric. If host tag is + // not found, we will log an error and skip this series. for _, groupTag := range *groupTagsPtr { log.Debugf("%v\n", groupTag) - // ignore host:not-set since datadog returns this value when host tag is missing, and we want to avoid using not-set as a host identifier + // ignore host:not-set since datadog returns this value when host tag is missing, + // and we want to avoid using not-set as a host identifier if strings.HasPrefix(groupTag, "host:") { parsedHost := getHostName(groupTag) if parsedHost != "" && parsedHost != "not-set" { @@ -297,7 +301,8 @@ func getMetricsFromTimeSeriesResponse(resp datadogV2.TimeseriesFormulaQueryRespo } } } - // If host tag is not found, log an error and skip this series since we won't be able to identify which host this metric belongs to. + // If host tag is not found, log an error and skip this series since we won't be able + // to identify which host this metric belongs to. if host == "" { log.Errorf( "No valid host tag found. queryIndex=%d groupTags=%v", @@ -320,7 +325,10 @@ func getMetricsFromTimeSeriesResponse(resp datadogV2.TimeseriesFormulaQueryRespo count += 1 } } - // If count is 0, it means all values for this series are nil. This can happen when there is no data for the host during the time window. We will log an error and skip this host since we don't want to add a metric with value 0 which can be misleading. + // If count is 0, it means all values for this series are nil. This can happen + // when there is no data for the host during the time window. We will log an + // error and skip this host since we don't want to add a metric with value 0 + // which can be misleading. if count == 0 { log.Errorf("No non-nil metric values for host %s", hosts[hostIndex]) hostIndex++