diff --git a/Dockerfile b/Dockerfile index 1eb10ed..46698c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,24 @@ ARG BUILD_SPEC="build.amd64" +# Builder FROM golang:1.22 +# Optional corporate CA (BuildKit secret: --secret id=corp_ca,src=paypal-crypto-root-ca.crt) +RUN --mount=type=secret,id=corp_ca,required=false,target=/tmp/corp-ca.crt \ + if [ -f /tmp/corp-ca.crt ]; then \ + cp /tmp/corp-ca.crt /usr/local/share/ca-certificates/corp-ca.crt && update-ca-certificates; \ + fi + +ARG GOPROXY=direct +ARG GOSUMDB=off +ENV GOPROXY=${GOPROXY} +ENV GOSUMDB=${GOSUMDB} + WORKDIR /go/src/github.com/paypal/load-watcher COPY . . + RUN make ${BUILD_SPEC} +# Runtime FROM alpine:3.12 COPY --from=0 /go/src/github.com/paypal/load-watcher/bin/load-watcher /bin/load-watcher diff --git a/Makefile b/Makefile index d2fdec9..aaf149e 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ build: .PHONY: build.amd64 build.amd64: - GOOS=linux$(BUILDENVVAR) GOARCH=amd64 go build -o bin/load-watcher main.go + GOOS=linux $(BUILDENVVAR) GOARCH=amd64 go build -o bin/load-watcher main.go .PHONY: build.arm64 build.arm64: diff --git a/README.md b/README.md index 30d15a4..42cdd75 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ The following metrics provider clients are currently supported: 1) SignalFx 2) Kubernetes Metrics Server 3) Prometheus +4) Datadog These clients fetch CPU usage currently, support for other resources will be added later as needed. @@ -19,10 +20,39 @@ The default `main.go` is configured to watch Kubernetes Metrics Server. You can change this to any available metrics provider in `pkg/metricsprovider`. To build a client for new metrics provider, you will need to implement `FetcherClient` interface. -From the root folder, run the following commands to build docker image of load watcher, tag it and push to your docker repository: +## Building behind a corporate TLS proxy + +Some corporate environments may fail during Go module downloads or base image pulls with errors such as: + +```text +x509: certificate signed by unknown authority + +Export the corporate root CA certificate: + +security find-certificate -c "PayPal Crypto Mgt Corp Root CA" -p \ + /Library/Keychains/System.keychain > paypal-crypto-root-ca.crt + +From the root folder, build the load-watcher image using Docker BuildKit: + +docker buildx build \ + --platform linux/amd64 \ + --secret id=corp_ca,src="$(pwd)/paypal-crypto-root-ca.crt" \ + --build-arg BUILD_SPEC=build.amd64 \ + --build-arg GOPROXY=direct \ + --build-arg GOSUMDB=off \ + --load \ + -t load-watcher: \ + . + +Notes: + +corp_ca is an optional BuildKit secret used to install the corporate root CA inside the builder container. +GOPROXY=direct and GOSUMDB=off can help in environments where access to proxy.golang.org is restricted. +linux/arm64 builds succeed locally. linux/amd64 builds on Apple Silicon/Rancher Desktop can intermittently hit Go compiler segmentation faults under emulation, so amd64 images should be validated on CI or an amd64 runner. + +Tag the docker image and push to your docker repository: ``` -docker build -t load-watcher: . docker tag load-watcher: : docker push ``` @@ -43,6 +73,8 @@ This will return metrics for all nodes. A query parameter to filter by host can - To use the SignalFx client, please configure environment variables `METRICS_PROVIDER_NAME`, `METRICS_PROVIDER_ADDRESS` and `METRICS_PROVIDER_TOKEN` to `SignalFx`, SignalFx address and auth token respectively. Default value of address set is `https://api.signalfx.com` for SignalFx client. +- To use the Datadog client, configure `METRICS_PROVIDER_NAME`, `METRICS_PROVIDER_ADDRESS`, `METRICS_PROVIDER_TOKEN`, `METRICS_PROVIDER_APP_KEY`, `DATADOG_ClUSTER_FILTER`, and `DATADOG_CLUSTER_NAME`. + ## Deploy `load-watcher` as a service To deploy `load-watcher` as a monitoring service in your Kubernetes cluster, you should replace the values in the `[]` with your own cluster monitoring stack and then you can run the following. ```bash diff --git a/pkg/watcher/internal/metricsprovider/datadog.go b/pkg/watcher/internal/metricsprovider/datadog.go index 40338f2..8a534a3 100644 --- a/pkg/watcher/internal/metricsprovider/datadog.go +++ b/pkg/watcher/internal/metricsprovider/datadog.go @@ -254,11 +254,18 @@ func getMetricsFromTimeSeriesResponse(resp datadogV2.TimeseriesFormulaQueryRespo return metrics, errors.New("No values from timeseries attributes.") } + // The number of series should match with the number of values since each series corresponds to + // an array of values across time intervals. If they don't match, return error and empty metrics. + if len(*timeSeriesDataSeriesPtr) != len(*timeSeriesDataValuesPtr) { + errMsg := "Number of series does not match number of values in timeseries response." + log.Error(errMsg) + return metrics, errors.New(errMsg) + } + hosts := make([]string, len(*timeSeriesDataSeriesPtr)) isCPU := make([]bool, len(*timeSeriesDataSeriesPtr)) - index := 0 // Populate host array and corresponding query index - for _, timeSeriesDataSeries := range *timeSeriesDataSeriesPtr { + for index, timeSeriesDataSeries := range *timeSeriesDataSeriesPtr { queryIndex, ok := timeSeriesDataSeries.GetQueryIndexOk() if !ok { log.Error("Error when getting query index from timeseries series.") @@ -268,7 +275,6 @@ func getMetricsFromTimeSeriesResponse(resp datadogV2.TimeseriesFormulaQueryRespo log.Error("No query index from timeseries series.") continue } - isCPU[index] = (*queryIndex == 0) groupTagsPtr, ok := timeSeriesDataSeries.GetGroupTagsOk() if !ok { log.Error("Error when getting group tags from timeseries series.") @@ -278,20 +284,37 @@ func getMetricsFromTimeSeriesResponse(resp datadogV2.TimeseriesFormulaQueryRespo log.Error("No group tags from timeseries series.") continue } - for _, groupTags := range *groupTagsPtr { - log.Debugf("%v\n", groupTags) - hosts[index] = getHostName(groupTags) - index++ + + host := "" + // Iterate through group tags to find host tag since group by can be on multiple tags. + // We will use the value in host tag as the hostname for the metric. If host tag is + // not found, we will log an error and skip this series. + for _, groupTag := range *groupTagsPtr { + log.Debugf("%v\n", groupTag) + // ignore host:not-set since datadog returns this value when host tag is missing, + // and we want to avoid using not-set as a host identifier + if strings.HasPrefix(groupTag, "host:") { + parsedHost := getHostName(groupTag) + if parsedHost != "" && parsedHost != "not-set" { + host = parsedHost + break + } + } + } + // If host tag is not found, log an error and skip this series since we won't be able + // to identify which host this metric belongs to. + if host == "" { + log.Errorf( + "No valid host tag found. queryIndex=%d groupTags=%v", + *queryIndex, + *groupTagsPtr) + continue } - } - if len(hosts) != len(*timeSeriesDataValuesPtr) { - errMsg := "Number of group tags does not match number of values in timeseries series." - log.Error(errMsg) - return metrics, errors.New(errMsg) + hosts[index] = host + isCPU[index] = (*queryIndex == 0) } - // Find the average across returned values per 1 minute resolution - // Build a hostname map of array of metrics [CPU, memory] + hostIndex := 0 for _, timesSeriesDataValues := range *timeSeriesDataValuesPtr { sum := 0.0 @@ -302,6 +325,16 @@ func getMetricsFromTimeSeriesResponse(resp datadogV2.TimeseriesFormulaQueryRespo count += 1 } } + // If count is 0, it means all values for this series are nil. This can happen + // when there is no data for the host during the time window. We will log an + // error and skip this host since we don't want to add a metric with value 0 + // which can be misleading. + if count == 0 { + log.Errorf("No non-nil metric values for host %s", hosts[hostIndex]) + hostIndex++ + continue + } + fetchedMetric := watcher.Metric{Value: sum / count} addDatadogMetadata(&fetchedMetric, isCPU[hostIndex]) metrics[hosts[hostIndex]] = append(metrics[hosts[hostIndex]], fetchedMetric)