Add a tool to measure the performance characteristics of providers

.github/workflows/ci.yml

@@ -75,6 +75,7 @@ jobs:
         uses: golangci/golangci-lint-action@v2
         with:
           version: ${{ env.GOLANGCI_VERSION }}
+          skip-go-installation: true
 
   check-diff:
     runs-on: ubuntu-18.04

go.mod

@@ -8,9 +8,16 @@ require (
 	github.com/google/addlicense v0.0.0-20210428195630-6d92264d7170
 	github.com/google/go-cmp v0.5.2
 	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v1.7.1
+	github.com/prometheus/common v0.10.0
+	github.com/sirupsen/logrus v1.6.0
+	github.com/spf13/cobra v1.1.1
+	github.com/spf13/pflag v1.0.5
+	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.20.2
 	k8s.io/apiextensions-apiserver v0.20.1
 	k8s.io/apimachinery v0.20.2
+	k8s.io/client-go v0.20.2
 	k8s.io/utils v0.0.0-20210111153108-fddb29f9d009
 	sigs.k8s.io/controller-runtime v0.8.3
 )

go.sum

@@ -345,6 +345,7 @@ github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ
 github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@@ -365,6 +366,7 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -496,6 +498,7 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
@@ -510,6 +513,7 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
+github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4=
 github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=

internal/providerScale/README.md

@@ -0,0 +1,34 @@
+# provider-scale
+
+This tool collects and reports some performance metrics. The details in the [one-pager].
+
+### Usage
+
+When this tool is executed, an end-to-end experiment will run. The inputs of this experiment can be manipulated by using
+the command-line options.
+
+```
+Flags:
+      --address string              Address of Prometheus service (default "http://localhost:9090")
+      --clean                       Delete deployed MRs (default true)
+  -h, --help                        help for provider-scale
+      --mrs stringToInt             Managed resource templates that will be deployed (default [])
+      --provider-namespace string   Namespace name of provider (default "crossplane-system")
+      --provider-pod string         Pod name of provider
+      --step-duration duration      Step duration between two data points (default 30s)
+```
+
+The `provider-pod` and `mrs` options are required.
+
+Example usage:
+
+```
+provider-scale --mrs ./internal/providerScale/manifests/virtualnetwork.yaml=2
+--mrs ./internal/providerScale/manifests/loadbalancer.yaml=2
+--provider-pod crossplane-provider-jet-azure
+--provider-namespace crossplane-system
+```
+
+With this input, two virtualnetwork & loadbalancer MRs will be deployed to the cluster.
+
+[one-pager]: https://github.com/crossplane/crossplane/pull/2983

internal/providerScale/cmd/common/common.go

@@ -0,0 +1,105 @@
+// Copyright 2022 The Crossplane Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/prometheus/common/model"
+
+	"github.com/prometheus/client_golang/api"
+	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
+)
+
+// Data represents a collected data
+type Data struct {
+	Timestamp time.Time
+	Value     float64
+}
+
+// Result represents all collected data for a metric
+type Result struct {
+	Data          []Data
+	Metric        string
+	MetricUnit    string
+	Peak, Average float64
+}
+
+// ConstructPrometheusClient creates a Prometheus API Client
+func ConstructPrometheusClient(address string) v1.API {
+	client, err := api.NewClient(api.Config{
+		Address: address,
+	})
+
+	if err != nil {
+		fmt.Printf("Error creating client: %v\n", err)
+		os.Exit(1)
+	}
+
+	return v1.NewAPI(client)
+}
+
+// ConstructTimeRange creates a Range object that consists the start time, end time and step duration
+func ConstructTimeRange(startTime, endTime time.Time, stepDuration time.Duration) v1.Range {
+	return v1.Range{
+		Start: startTime,
+		End:   endTime,
+		Step:  stepDuration,
+	}
+}
+
+// ConstructResult creates a Result object from collected data
+func ConstructResult(value model.Value, metric, unit string) (*Result, error) {
+	result := &Result{}
+	matrix := value.(model.Matrix)
+
+	for _, m := range matrix {
+		for _, v := range m.Values {
+			valueNum, err := strconv.ParseFloat(v.Value.String(), 64)
+			if err != nil {
+				return nil, err
+			}
+			result.Data = append(result.Data, Data{Timestamp: v.Timestamp.Time(), Value: valueNum})
+		}
+	}
+
+	result.Average, result.Peak = CalculateAverageAndPeak(result.Data)
+	result.Metric = metric
+	result.MetricUnit = unit
+	return result, nil
+}
+
+// CalculateAverageAndPeak calculates the average and peak values of related metric
+func CalculateAverageAndPeak(data []Data) (float64, float64) {
+	var sum, peak float64
+	for _, d := range data {
+		sum += d.Value
+
+		if d.Value > peak {
+			peak = d.Value
+		}
+	}
+	return sum / float64(len(data)), peak
+}
+
+func (r Result) String() {
+	log.Info(fmt.Sprintf("Average %s: %f %s \n", r.Metric, r.Average, r.MetricUnit))
+	log.Info(fmt.Sprintf("Peak %s: %f %s \n", r.Metric, r.Peak, r.MetricUnit))
+}