From cbb39bd200239169fed13836f2b69bb20a66e443 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergen=20Yal=C3=A7=C4=B1n?= <yalcinsergen97@gmail.com>
Date: Thu, 24 Mar 2022 09:55:41 +0300
Subject: [PATCH 1/6] Add a tool to measure the performance characteristics of
 providers
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Sergen Yalçın <yalcinsergen97@gmail.com>
---
 go.mod                                        |   4 +
 go.sum                                        |   2 +
 internal/providerScale/cmd/common/common.go   |  77 ++++++++++++
 .../cmd/managed/checkDeletion.sh              |  10 ++
 .../cmd/managed/checkReadiness.sh             |  10 ++
 .../cmd/managed/getCreationTime.sh            |  10 ++
 .../cmd/managed/getReadinessTime.sh           |  10 ++
 .../providerScale/cmd/managed/manage-mr.sh    |  13 ++
 internal/providerScale/cmd/managed/managed.go | 112 ++++++++++++++++++
 internal/providerScale/cmd/quantify.go        | 101 ++++++++++++++++
 internal/providerScale/main.go                |  16 +++
 .../providerScale/manifests/loadbalancer.yaml |  10 ++
 .../manifests/virtualnetwork.yaml             |  18 +++
 13 files changed, 393 insertions(+)
 create mode 100644 internal/providerScale/cmd/common/common.go
 create mode 100755 internal/providerScale/cmd/managed/checkDeletion.sh
 create mode 100755 internal/providerScale/cmd/managed/checkReadiness.sh
 create mode 100755 internal/providerScale/cmd/managed/getCreationTime.sh
 create mode 100755 internal/providerScale/cmd/managed/getReadinessTime.sh
 create mode 100755 internal/providerScale/cmd/managed/manage-mr.sh
 create mode 100644 internal/providerScale/cmd/managed/managed.go
 create mode 100644 internal/providerScale/cmd/quantify.go
 create mode 100644 internal/providerScale/main.go
 create mode 100644 internal/providerScale/manifests/loadbalancer.yaml
 create mode 100644 internal/providerScale/manifests/virtualnetwork.yaml

diff --git a/go.mod b/go.mod
index b988d3e..f64336f 100644
--- a/go.mod
+++ b/go.mod
@@ -8,6 +8,10 @@ require (
 	github.com/google/addlicense v0.0.0-20210428195630-6d92264d7170
 	github.com/google/go-cmp v0.5.2
 	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v1.7.1
+	github.com/prometheus/common v0.10.0
+	github.com/spf13/cobra v1.1.1
+	github.com/spf13/pflag v1.0.5
 	k8s.io/api v0.20.2
 	k8s.io/apiextensions-apiserver v0.20.1
 	k8s.io/apimachinery v0.20.2
diff --git a/go.sum b/go.sum
index 0cf036f..6c24cd7 100644
--- a/go.sum
+++ b/go.sum
@@ -345,6 +345,7 @@ github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ
 github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@@ -510,6 +511,7 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
+github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4=
 github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
diff --git a/internal/providerScale/cmd/common/common.go b/internal/providerScale/cmd/common/common.go
new file mode 100644
index 0000000..5d0c941
--- /dev/null
+++ b/internal/providerScale/cmd/common/common.go
@@ -0,0 +1,77 @@
+package common
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/prometheus/common/model"
+
+	"github.com/prometheus/client_golang/api"
+	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
+)
+
+type Data struct {
+	Timestamp time.Time
+	Value     float64
+}
+
+type Result struct {
+	Data          []Data
+	Metric        string
+	Peak, Average float64
+}
+
+func ConstructPrometheusClient(address string) v1.API {
+	client, err := api.NewClient(api.Config{
+		Address: address,
+	})
+
+	if err != nil {
+		fmt.Printf("Error creating client: %v\n", err)
+		os.Exit(1)
+	}
+
+	return v1.NewAPI(client)
+}
+
+func ConstructTimeRange(startTime, endTime time.Time, stepDuration time.Duration) v1.Range {
+	return v1.Range{
+		Start: startTime,
+		End:   endTime,
+		Step:  stepDuration,
+	}
+}
+
+func ConstructResult(value model.Value) (*Result, error) {
+	result := &Result{}
+	matrix := value.(model.Matrix)
+
+	for _, m := range matrix {
+		result.Metric = m.Metric.String()
+
+		for _, v := range m.Values {
+			valueNum, err := strconv.ParseFloat(v.Value.String(), 64)
+			if err != nil {
+				return nil, err
+			}
+			result.Data = append(result.Data, Data{Timestamp: v.Timestamp.Time(), Value: valueNum})
+		}
+	}
+
+	result.Average, result.Peak = CalculateAverageAndPeak(result.Data)
+	return result, nil
+}
+
+func CalculateAverageAndPeak(data []Data) (float64, float64) {
+	var sum, peak float64
+	for _, d := range data {
+		sum += d.Value
+
+		if d.Value > peak {
+			peak = d.Value
+		}
+	}
+	return sum / float64(len(data)), peak
+}
diff --git a/internal/providerScale/cmd/managed/checkDeletion.sh b/internal/providerScale/cmd/managed/checkDeletion.sh
new file mode 100755
index 0000000..ae6f076
--- /dev/null
+++ b/internal/providerScale/cmd/managed/checkDeletion.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env zsh
+
+# HO.
+
+set -euxo pipefail
+
+TEMPLATE="$1"
+NAME="$2"
+
+cat "${TEMPLATE}" | sed "s/{{SUFFIX}}/$NAME/g" | kubectl get --ignore-not-found -f -
\ No newline at end of file
diff --git a/internal/providerScale/cmd/managed/checkReadiness.sh b/internal/providerScale/cmd/managed/checkReadiness.sh
new file mode 100755
index 0000000..67ab091
--- /dev/null
+++ b/internal/providerScale/cmd/managed/checkReadiness.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env zsh
+
+# HO.
+
+set -euxo pipefail
+
+TEMPLATE="$1"
+NAME="$2"
+
+cat "${TEMPLATE}" | sed "s/{{SUFFIX}}/$NAME/g" | kubectl get -o json -f - | jq '.items[] | .status.conditions[] | select(.type=="Ready")' | jq .status
diff --git a/internal/providerScale/cmd/managed/getCreationTime.sh b/internal/providerScale/cmd/managed/getCreationTime.sh
new file mode 100755
index 0000000..1eb9807
--- /dev/null
+++ b/internal/providerScale/cmd/managed/getCreationTime.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env zsh
+
+# HO.
+
+set -euxo pipefail
+
+TEMPLATE="$1"
+NAME="$2"
+
+cat "${TEMPLATE}" | sed "s/{{SUFFIX}}/$NAME/g" | kubectl get -o json -f - | jq -r '.items[] | .metadata.creationTimestamp'
diff --git a/internal/providerScale/cmd/managed/getReadinessTime.sh b/internal/providerScale/cmd/managed/getReadinessTime.sh
new file mode 100755
index 0000000..8a59941
--- /dev/null
+++ b/internal/providerScale/cmd/managed/getReadinessTime.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env zsh
+
+# HO.
+
+set -euxo pipefail
+
+TEMPLATE="$1"
+NAME="$2"
+
+cat "${TEMPLATE}" | sed "s/{{SUFFIX}}/$NAME/g" | kubectl get -o json -f - | jq '.items[] | .status.conditions[] | select(.type=="Ready" and .status=="True" )' | jq .lastTransitionTime
diff --git a/internal/providerScale/cmd/managed/manage-mr.sh b/internal/providerScale/cmd/managed/manage-mr.sh
new file mode 100755
index 0000000..710e86f
--- /dev/null
+++ b/internal/providerScale/cmd/managed/manage-mr.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env zsh
+
+# HO.
+
+set -euxo pipefail
+
+# Example usage: ./manage-mr.sh create virtualnetwork.yaml 10
+# Example usage: ./manage-mr.sh delete virtualnetwork.yaml 10 | tee exp-virtualnetwork.log
+OPERATION="$1"
+TEMPLATE="$2"
+NAME="$3"
+
+cat "${TEMPLATE}" | sed "s/{{SUFFIX}}/$NAME/g" | kubectl --wait=false "${OPERATION}" -f -
diff --git a/internal/providerScale/cmd/managed/managed.go b/internal/providerScale/cmd/managed/managed.go
new file mode 100644
index 0000000..f588cb6
--- /dev/null
+++ b/internal/providerScale/cmd/managed/managed.go
@@ -0,0 +1,112 @@
+package managed
+
+import (
+	"fmt"
+	"github.com/crossplane/conformance/internal/providerScale/cmd/common"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func RunExperiment(mrTemplatePaths map[string]int, clean bool) ([]*common.Result, error) {
+	var timeToReadinessResults []*common.Result
+
+	for mrPath, count := range mrTemplatePaths {
+		for i := 1; i <= count; i++ {
+			cmd, err := exec.Command("./internal/providerScale/cmd/managed/manage-mr.sh", "apply", mrPath, strconv.Itoa(i)).Output()
+			fmt.Print(string(cmd))
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+	for mrPath, count := range mrTemplatePaths {
+		if err := checkReadiness(mrPath, count); err != nil {
+			return nil, err
+		}
+	}
+	for mrPath, count := range mrTemplatePaths {
+		timeToReadinessResult, err := calculateReadinessDuration(mrPath, count)
+		if err != nil {
+			return nil, err
+		}
+		timeToReadinessResults = append(timeToReadinessResults, timeToReadinessResult)
+	}
+	if clean {
+		for mrPath, count := range mrTemplatePaths {
+			fmt.Println("Deleting resources...")
+			for i := 1; i <= count; i++ {
+				cmd, err := exec.Command("./internal/providerScale/cmd/managed/manage-mr.sh", "delete", mrPath, strconv.Itoa(i)).Output()
+				fmt.Print(string(cmd))
+				if err != nil {
+					return nil, err
+				}
+			}
+		}
+		for mrPath, count := range mrTemplatePaths {
+			i := 1
+			for i <= count {
+				fmt.Println("Checking deletion of resources...")
+				cmd, err := exec.Command("./internal/providerScale/cmd/managed/checkDeletion.sh", mrPath, strconv.Itoa(i)).Output()
+				if err != nil {
+					return nil, err
+				}
+				if string(cmd) != "" {
+					time.Sleep(10 * time.Second)
+					continue
+				}
+				i++
+			}
+		}
+	}
+	return timeToReadinessResults, nil
+}
+
+func checkReadiness(mrPath string, count int) error {
+	i := 1
+	for i <= count {
+		fmt.Println("Checking readiness of resources...")
+		isReady, _ := exec.Command("./internal/providerScale/cmd/managed/checkReadiness.sh", mrPath, strconv.Itoa(i)).Output()
+		if !strings.Contains(string(isReady), "True") {
+			time.Sleep(10 * time.Second)
+			continue
+		}
+		i++
+	}
+
+	return nil
+}
+
+func calculateReadinessDuration(mrPath string, count int) (*common.Result, error) {
+	result := &common.Result{}
+	for i := 1; i <= count; i++ {
+		fmt.Println("Calculating readiness durations of resources...")
+		creationTimeByte, err := exec.Command("./internal/providerScale/cmd/managed/getCreationTime.sh", mrPath, strconv.Itoa(i)).Output()
+		if err != nil {
+			return nil, err
+		}
+		readinessTimeByte, err := exec.Command("./internal/providerScale/cmd/managed/getReadinessTime.sh", mrPath, strconv.Itoa(i)).Output()
+		if err != nil {
+			return nil, err
+		}
+		creationTimeString := string(creationTimeByte)
+		creationTimeString = creationTimeString[:strings.Index(creationTimeString, `Z`)+1]
+		creationTime, err := time.Parse(time.RFC3339, creationTimeString)
+		if err != nil {
+			return nil, err
+		}
+		readinessTimeString := string(readinessTimeByte)
+		readinessTimeString = readinessTimeString[strings.Index(readinessTimeString, `"`)+1 : strings.Index(readinessTimeString, `Z`)+1]
+		readinessTime, err := time.Parse(time.RFC3339, readinessTimeString)
+		if err != nil {
+			return nil, err
+		}
+		diff := readinessTime.Sub(creationTime)
+		result.Data = append(result.Data, common.Data{Value: diff.Seconds()})
+	}
+	result.Metric = mrPath
+	result.Average, result.Peak = common.CalculateAverageAndPeak(result.Data)
+	return result, nil
+}
+
diff --git a/internal/providerScale/cmd/quantify.go b/internal/providerScale/cmd/quantify.go
new file mode 100644
index 0000000..24deade
--- /dev/null
+++ b/internal/providerScale/cmd/quantify.go
@@ -0,0 +1,101 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"github.com/crossplane/conformance/internal/providerScale/cmd/common"
+	"github.com/crossplane/conformance/internal/providerScale/cmd/managed"
+	"time"
+
+	"github.com/prometheus/common/model"
+
+	"github.com/spf13/cobra"
+)
+
+type QuantifyOptions struct {
+	providerPod       string
+	providerNamespace string
+	mrPaths           map[string]int
+	cmd               *cobra.Command
+	address           string
+	startTime         time.Time
+	endTime           time.Time
+	stepDuration      time.Duration
+	clean             bool
+}
+
+func NewCmdQuantify() *cobra.Command {
+	o := QuantifyOptions{}
+	o.cmd = &cobra.Command{
+		Use:     fmt.Sprintf("Quantify the performance metrics"),
+		Short:   "quantify",
+		Example: fmt.Sprintf("Example"),
+		RunE:    o.Run,
+	}
+
+	o.cmd.Flags().StringVar(&o.providerPod, "provider-pod", "", "provider pod name...")
+	o.cmd.Flags().StringVar(&o.providerNamespace, "provider-namespace", "crossplane-system", "provider namespace...")
+	o.cmd.Flags().StringToIntVar(&o.mrPaths, "mrs", map[string]int{}, "mr paths...")
+	o.cmd.Flags().StringVar(&o.address, "address", "http://localhost:9090", "address...")
+	o.cmd.Flags().DurationVar(&o.stepDuration, "step-duration", 30*time.Second, "step duration...")
+	o.cmd.Flags().BoolVar(&o.clean, "clean", true, "clean cluster...")
+
+	return o.cmd
+}
+
+func (o *QuantifyOptions) Run(_ *cobra.Command, _ []string) error {
+	o.startTime = time.Now()
+	fmt.Printf("Experiment Started %v\n\n", o.startTime)
+	timeToReadinessResults, err := managed.RunExperiment(o.mrPaths, o.clean)
+	if err != nil {
+		return err
+	}
+	o.endTime = time.Now()
+	fmt.Printf("\nExperiment Ended %v\n\n", o.endTime)
+	fmt.Printf("Results\n------------------------------------------------------------\n")
+	fmt.Printf("Experiment Duration: %f seconds\n", o.endTime.Sub(o.startTime).Seconds())
+	queryResultCpu, err := o.CollectData(fmt.Sprintf(`sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{pod="%s", namespace="%s"})*100`,
+		o.providerPod, o.providerNamespace))
+	if err != nil {
+		return err
+	}
+	cpuResult, err := common.ConstructResult(queryResultCpu)
+	if err != nil {
+		return err
+	}
+	queryResultMemory, err := o.CollectData(fmt.Sprintf(`sum(node_namespace_pod_container:container_memory_working_set_bytes{pod="%s", namespace="%s"})`,
+		o.providerPod, o.providerNamespace))
+	if err != nil {
+		return err
+	}
+	memoryResult, err := common.ConstructResult(queryResultMemory)
+	if err != nil {
+		return err
+	}
+
+	for _, timeToReadinessResult := range timeToReadinessResults {
+		fmt.Printf("Average Time to Readiness of %s: %f \n", timeToReadinessResult.Metric, timeToReadinessResult.Average)
+		fmt.Printf("Peak Time to Readiness of %s: %f \n", timeToReadinessResult.Metric, timeToReadinessResult.Peak)
+	}
+	fmt.Printf("Average CPU: %f %% \n", cpuResult.Average)
+	fmt.Printf("Peak CPU: %f %% \n", cpuResult.Peak)
+	fmt.Printf("Average Memory: %f Bytes \n", memoryResult.Average)
+	fmt.Printf("Peak Memory: %f Bytes \n", memoryResult.Peak)
+	return nil
+}
+
+func (o *QuantifyOptions) CollectData(query string) (model.Value, error) {
+	client := common.ConstructPrometheusClient(o.address)
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	r := common.ConstructTimeRange(o.startTime, o.endTime, o.stepDuration)
+	result, warnings, err := client.QueryRange(ctx, query, r)
+	if err != nil {
+		return nil, err
+	}
+	if len(warnings) > 0 {
+		fmt.Printf("Warnings: %v\n", warnings)
+	}
+	return result, err
+}
diff --git a/internal/providerScale/main.go b/internal/providerScale/main.go
new file mode 100644
index 0000000..f4b91c9
--- /dev/null
+++ b/internal/providerScale/main.go
@@ -0,0 +1,16 @@
+package main
+
+import (
+	"github.com/crossplane/conformance/internal/providerScale/cmd"
+	"os"
+
+	"github.com/spf13/pflag"
+)
+
+func main() {
+	pflag.CommandLine = pflag.NewFlagSet("quantify", pflag.ExitOnError)
+	root := cmd.NewCmdQuantify()
+	if err := root.Execute(); err != nil {
+		os.Exit(1)
+	}
+}
diff --git a/internal/providerScale/manifests/loadbalancer.yaml b/internal/providerScale/manifests/loadbalancer.yaml
new file mode 100644
index 0000000..e4407d2
--- /dev/null
+++ b/internal/providerScale/manifests/loadbalancer.yaml
@@ -0,0 +1,10 @@
+apiVersion: network.azure.jet.crossplane.io/v1alpha2
+kind: LoadBalancer
+metadata:
+  name: test-{{SUFFIX}}
+spec:
+  forProvider:
+    location: East US
+    resourceGroupName: example
+  providerConfigRef:
+    name: example
diff --git a/internal/providerScale/manifests/virtualnetwork.yaml b/internal/providerScale/manifests/virtualnetwork.yaml
new file mode 100644
index 0000000..47bfb67
--- /dev/null
+++ b/internal/providerScale/manifests/virtualnetwork.yaml
@@ -0,0 +1,18 @@
+apiVersion: network.azure.jet.crossplane.io/v1alpha2
+kind: VirtualNetwork
+metadata:
+  name: test-{{SUFFIX}}
+spec:
+  forProvider:
+    addressSpace:
+    - 10.0.0.0/16
+    dnsServers:
+    - 10.0.0.1
+    - 10.0.0.2
+    - 10.0.0.3
+    location: East US
+    resourceGroupName: example
+    tags:
+      experiment: "2"
+  providerConfigRef:
+    name: example

From 01fbf6a86bbc7659ddaec2ec25eaa35c0468bcf7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergen=20Yal=C3=A7=C4=B1n?= <yalcinsergen97@gmail.com>
Date: Thu, 24 Mar 2022 14:56:45 +0300
Subject: [PATCH 2/6] Fix lint issues
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Sergen Yalçın <yalcinsergen97@gmail.com>
---
 internal/providerScale/cmd/common/common.go   |  6 ++++
 internal/providerScale/cmd/managed/managed.go | 33 +++++++++----------
 internal/providerScale/cmd/quantify.go        | 15 ++++++---
 internal/providerScale/main.go                |  3 +-
 4 files changed, 34 insertions(+), 23 deletions(-)

diff --git a/internal/providerScale/cmd/common/common.go b/internal/providerScale/cmd/common/common.go
index 5d0c941..886d02c 100644
--- a/internal/providerScale/cmd/common/common.go
+++ b/internal/providerScale/cmd/common/common.go
@@ -12,17 +12,20 @@ import (
 	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
 )
 
+// Data represents a collected data
 type Data struct {
 	Timestamp time.Time
 	Value     float64
 }
 
+// Result represents all collected data for a metric
 type Result struct {
 	Data          []Data
 	Metric        string
 	Peak, Average float64
 }
 
+// ConstructPrometheusClient creates a Prometheus API Client
 func ConstructPrometheusClient(address string) v1.API {
 	client, err := api.NewClient(api.Config{
 		Address: address,
@@ -36,6 +39,7 @@ func ConstructPrometheusClient(address string) v1.API {
 	return v1.NewAPI(client)
 }
 
+// ConstructTimeRange creates a Range object that consists the start time, end time and step duration
 func ConstructTimeRange(startTime, endTime time.Time, stepDuration time.Duration) v1.Range {
 	return v1.Range{
 		Start: startTime,
@@ -44,6 +48,7 @@ func ConstructTimeRange(startTime, endTime time.Time, stepDuration time.Duration
 	}
 }
 
+// ConstructResult creates a Result object from collected data
 func ConstructResult(value model.Value) (*Result, error) {
 	result := &Result{}
 	matrix := value.(model.Matrix)
@@ -64,6 +69,7 @@ func ConstructResult(value model.Value) (*Result, error) {
 	return result, nil
 }
 
+// CalculateAverageAndPeak calculates the average and peak values of related metric
 func CalculateAverageAndPeak(data []Data) (float64, float64) {
 	var sum, peak float64
 	for _, d := range data {
diff --git a/internal/providerScale/cmd/managed/managed.go b/internal/providerScale/cmd/managed/managed.go
index f588cb6..526ac56 100644
--- a/internal/providerScale/cmd/managed/managed.go
+++ b/internal/providerScale/cmd/managed/managed.go
@@ -2,19 +2,23 @@ package managed
 
 import (
 	"fmt"
-	"github.com/crossplane/conformance/internal/providerScale/cmd/common"
 	"os/exec"
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/crossplane/conformance/internal/providerScale/cmd/common"
 )
 
-func RunExperiment(mrTemplatePaths map[string]int, clean bool) ([]*common.Result, error) {
-	var timeToReadinessResults []*common.Result
+// RunExperiment runs the experiment according to command-line inputs.
+// Firstly the input manifests are deployed. After the all MRs are ready, time to readiness metrics are calculated.
+// Then, by default, all deployed MRs are deleted.
+func RunExperiment(mrTemplatePaths map[string]int, clean bool) ([]*common.Result, error) { //nolint:gocyclo
+	var timeToReadinessResults []*common.Result //nolint:prealloc
 
 	for mrPath, count := range mrTemplatePaths {
 		for i := 1; i <= count; i++ {
-			cmd, err := exec.Command("./internal/providerScale/cmd/managed/manage-mr.sh", "apply", mrPath, strconv.Itoa(i)).Output()
+			cmd, err := exec.Command("./internal/providerScale/cmd/managed/manage-mr.sh", "apply", mrPath, strconv.Itoa(i)).Output() //nolint:gosec
 			fmt.Print(string(cmd))
 			if err != nil {
 				return nil, err
@@ -22,9 +26,7 @@ func RunExperiment(mrTemplatePaths map[string]int, clean bool) ([]*common.Result
 		}
 	}
 	for mrPath, count := range mrTemplatePaths {
-		if err := checkReadiness(mrPath, count); err != nil {
-			return nil, err
-		}
+		checkReadiness(mrPath, count)
 	}
 	for mrPath, count := range mrTemplatePaths {
 		timeToReadinessResult, err := calculateReadinessDuration(mrPath, count)
@@ -37,7 +39,7 @@ func RunExperiment(mrTemplatePaths map[string]int, clean bool) ([]*common.Result
 		for mrPath, count := range mrTemplatePaths {
 			fmt.Println("Deleting resources...")
 			for i := 1; i <= count; i++ {
-				cmd, err := exec.Command("./internal/providerScale/cmd/managed/manage-mr.sh", "delete", mrPath, strconv.Itoa(i)).Output()
+				cmd, err := exec.Command("./internal/providerScale/cmd/managed/manage-mr.sh", "delete", mrPath, strconv.Itoa(i)).Output() //nolint:gosec
 				fmt.Print(string(cmd))
 				if err != nil {
 					return nil, err
@@ -48,11 +50,11 @@ func RunExperiment(mrTemplatePaths map[string]int, clean bool) ([]*common.Result
 			i := 1
 			for i <= count {
 				fmt.Println("Checking deletion of resources...")
-				cmd, err := exec.Command("./internal/providerScale/cmd/managed/checkDeletion.sh", mrPath, strconv.Itoa(i)).Output()
+				cmd, err := exec.Command("./internal/providerScale/cmd/managed/checkDeletion.sh", mrPath, strconv.Itoa(i)).Output() //nolint:gosec
 				if err != nil {
 					return nil, err
 				}
-				if string(cmd) != "" {
+				if len(cmd) != 0 {
 					time.Sleep(10 * time.Second)
 					continue
 				}
@@ -63,30 +65,28 @@ func RunExperiment(mrTemplatePaths map[string]int, clean bool) ([]*common.Result
 	return timeToReadinessResults, nil
 }
 
-func checkReadiness(mrPath string, count int) error {
+func checkReadiness(mrPath string, count int) {
 	i := 1
 	for i <= count {
 		fmt.Println("Checking readiness of resources...")
-		isReady, _ := exec.Command("./internal/providerScale/cmd/managed/checkReadiness.sh", mrPath, strconv.Itoa(i)).Output()
+		isReady, _ := exec.Command("./internal/providerScale/cmd/managed/checkReadiness.sh", mrPath, strconv.Itoa(i)).Output() //nolint:gosec
 		if !strings.Contains(string(isReady), "True") {
 			time.Sleep(10 * time.Second)
 			continue
 		}
 		i++
 	}
-
-	return nil
 }
 
 func calculateReadinessDuration(mrPath string, count int) (*common.Result, error) {
 	result := &common.Result{}
 	for i := 1; i <= count; i++ {
 		fmt.Println("Calculating readiness durations of resources...")
-		creationTimeByte, err := exec.Command("./internal/providerScale/cmd/managed/getCreationTime.sh", mrPath, strconv.Itoa(i)).Output()
+		creationTimeByte, err := exec.Command("./internal/providerScale/cmd/managed/getCreationTime.sh", mrPath, strconv.Itoa(i)).Output() //nolint:gosec
 		if err != nil {
 			return nil, err
 		}
-		readinessTimeByte, err := exec.Command("./internal/providerScale/cmd/managed/getReadinessTime.sh", mrPath, strconv.Itoa(i)).Output()
+		readinessTimeByte, err := exec.Command("./internal/providerScale/cmd/managed/getReadinessTime.sh", mrPath, strconv.Itoa(i)).Output() //nolint:gosec
 		if err != nil {
 			return nil, err
 		}
@@ -109,4 +109,3 @@ func calculateReadinessDuration(mrPath string, count int) (*common.Result, error
 	result.Average, result.Peak = common.CalculateAverageAndPeak(result.Data)
 	return result, nil
 }
-
diff --git a/internal/providerScale/cmd/quantify.go b/internal/providerScale/cmd/quantify.go
index 24deade..af4836b 100644
--- a/internal/providerScale/cmd/quantify.go
+++ b/internal/providerScale/cmd/quantify.go
@@ -3,15 +3,17 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"time"
+
 	"github.com/crossplane/conformance/internal/providerScale/cmd/common"
 	"github.com/crossplane/conformance/internal/providerScale/cmd/managed"
-	"time"
 
 	"github.com/prometheus/common/model"
 
 	"github.com/spf13/cobra"
 )
 
+// QuantifyOptions represents the options of quantify command
 type QuantifyOptions struct {
 	providerPod       string
 	providerNamespace string
@@ -24,12 +26,13 @@ type QuantifyOptions struct {
 	clean             bool
 }
 
+// NewCmdQuantify creates a cobra command
 func NewCmdQuantify() *cobra.Command {
 	o := QuantifyOptions{}
 	o.cmd = &cobra.Command{
-		Use:     fmt.Sprintf("Quantify the performance metrics"),
+		Use:     "Quantify the performance metrics",
 		Short:   "quantify",
-		Example: fmt.Sprintf("Example"),
+		Example: "Example",
 		RunE:    o.Run,
 	}
 
@@ -43,6 +46,7 @@ func NewCmdQuantify() *cobra.Command {
 	return o.cmd
 }
 
+// Run executes the quantify command's tasks.
 func (o *QuantifyOptions) Run(_ *cobra.Command, _ []string) error {
 	o.startTime = time.Now()
 	fmt.Printf("Experiment Started %v\n\n", o.startTime)
@@ -54,12 +58,12 @@ func (o *QuantifyOptions) Run(_ *cobra.Command, _ []string) error {
 	fmt.Printf("\nExperiment Ended %v\n\n", o.endTime)
 	fmt.Printf("Results\n------------------------------------------------------------\n")
 	fmt.Printf("Experiment Duration: %f seconds\n", o.endTime.Sub(o.startTime).Seconds())
-	queryResultCpu, err := o.CollectData(fmt.Sprintf(`sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{pod="%s", namespace="%s"})*100`,
+	queryResultCPU, err := o.CollectData(fmt.Sprintf(`sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{pod="%s", namespace="%s"})*100`,
 		o.providerPod, o.providerNamespace))
 	if err != nil {
 		return err
 	}
-	cpuResult, err := common.ConstructResult(queryResultCpu)
+	cpuResult, err := common.ConstructResult(queryResultCPU)
 	if err != nil {
 		return err
 	}
@@ -84,6 +88,7 @@ func (o *QuantifyOptions) Run(_ *cobra.Command, _ []string) error {
 	return nil
 }
 
+// CollectData sends query and collect data by using the prometheus client
 func (o *QuantifyOptions) CollectData(query string) (model.Value, error) {
 	client := common.ConstructPrometheusClient(o.address)
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
diff --git a/internal/providerScale/main.go b/internal/providerScale/main.go
index f4b91c9..2f59426 100644
--- a/internal/providerScale/main.go
+++ b/internal/providerScale/main.go
@@ -1,9 +1,10 @@
 package main
 
 import (
-	"github.com/crossplane/conformance/internal/providerScale/cmd"
 	"os"
 
+	"github.com/crossplane/conformance/internal/providerScale/cmd"
+
 	"github.com/spf13/pflag"
 )
 

From 02a875650b053f045b461ac25069905f3a6ea7e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergen=20Yal=C3=A7=C4=B1n?= <yalcinsergen97@gmail.com>
Date: Thu, 24 Mar 2022 15:02:46 +0300
Subject: [PATCH 3/6] Add copyright statements
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Sergen Yalçın <yalcinsergen97@gmail.com>
---
 internal/providerScale/cmd/common/common.go        | 14 ++++++++++++++
 .../providerScale/cmd/managed/checkDeletion.sh     | 14 ++++++++++++++
 .../providerScale/cmd/managed/checkReadiness.sh    | 14 ++++++++++++++
 .../providerScale/cmd/managed/getCreationTime.sh   | 14 ++++++++++++++
 .../providerScale/cmd/managed/getReadinessTime.sh  | 14 ++++++++++++++
 internal/providerScale/cmd/managed/manage-mr.sh    | 14 ++++++++++++++
 internal/providerScale/cmd/managed/managed.go      | 14 ++++++++++++++
 internal/providerScale/cmd/quantify.go             | 14 ++++++++++++++
 internal/providerScale/main.go                     | 14 ++++++++++++++
 internal/providerScale/manifests/loadbalancer.yaml | 14 ++++++++++++++
 .../providerScale/manifests/virtualnetwork.yaml    | 14 ++++++++++++++
 11 files changed, 154 insertions(+)

diff --git a/internal/providerScale/cmd/common/common.go b/internal/providerScale/cmd/common/common.go
index 886d02c..df7b4c4 100644
--- a/internal/providerScale/cmd/common/common.go
+++ b/internal/providerScale/cmd/common/common.go
@@ -1,3 +1,17 @@
+// Copyright 2022 The Crossplane Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package common
 
 import (
diff --git a/internal/providerScale/cmd/managed/checkDeletion.sh b/internal/providerScale/cmd/managed/checkDeletion.sh
index ae6f076..03f582b 100755
--- a/internal/providerScale/cmd/managed/checkDeletion.sh
+++ b/internal/providerScale/cmd/managed/checkDeletion.sh
@@ -1,4 +1,18 @@
 #!/usr/bin/env zsh
+# Copyright 2022 The Crossplane Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 # HO.
 
diff --git a/internal/providerScale/cmd/managed/checkReadiness.sh b/internal/providerScale/cmd/managed/checkReadiness.sh
index 67ab091..5c9e127 100755
--- a/internal/providerScale/cmd/managed/checkReadiness.sh
+++ b/internal/providerScale/cmd/managed/checkReadiness.sh
@@ -1,4 +1,18 @@
 #!/usr/bin/env zsh
+# Copyright 2022 The Crossplane Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 # HO.
 
diff --git a/internal/providerScale/cmd/managed/getCreationTime.sh b/internal/providerScale/cmd/managed/getCreationTime.sh
index 1eb9807..9724e4b 100755
--- a/internal/providerScale/cmd/managed/getCreationTime.sh
+++ b/internal/providerScale/cmd/managed/getCreationTime.sh
@@ -1,4 +1,18 @@
 #!/usr/bin/env zsh
+# Copyright 2022 The Crossplane Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 # HO.
 
diff --git a/internal/providerScale/cmd/managed/getReadinessTime.sh b/internal/providerScale/cmd/managed/getReadinessTime.sh
index 8a59941..29a340e 100755
--- a/internal/providerScale/cmd/managed/getReadinessTime.sh
+++ b/internal/providerScale/cmd/managed/getReadinessTime.sh
@@ -1,4 +1,18 @@
 #!/usr/bin/env zsh
+# Copyright 2022 The Crossplane Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 # HO.
 
diff --git a/internal/providerScale/cmd/managed/manage-mr.sh b/internal/providerScale/cmd/managed/manage-mr.sh
index 710e86f..a8ce079 100755
--- a/internal/providerScale/cmd/managed/manage-mr.sh
+++ b/internal/providerScale/cmd/managed/manage-mr.sh
@@ -1,4 +1,18 @@
 #!/usr/bin/env zsh
+# Copyright 2022 The Crossplane Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 # HO.
 
diff --git a/internal/providerScale/cmd/managed/managed.go b/internal/providerScale/cmd/managed/managed.go
index 526ac56..922c9ef 100644
--- a/internal/providerScale/cmd/managed/managed.go
+++ b/internal/providerScale/cmd/managed/managed.go
@@ -1,3 +1,17 @@
+// Copyright 2022 The Crossplane Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package managed
 
 import (
diff --git a/internal/providerScale/cmd/quantify.go b/internal/providerScale/cmd/quantify.go
index af4836b..db4a4ba 100644
--- a/internal/providerScale/cmd/quantify.go
+++ b/internal/providerScale/cmd/quantify.go
@@ -1,3 +1,17 @@
+// Copyright 2022 The Crossplane Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package cmd
 
 import (
diff --git a/internal/providerScale/main.go b/internal/providerScale/main.go
index 2f59426..730d99e 100644
--- a/internal/providerScale/main.go
+++ b/internal/providerScale/main.go
@@ -1,3 +1,17 @@
+// Copyright 2022 The Crossplane Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package main
 
 import (
diff --git a/internal/providerScale/manifests/loadbalancer.yaml b/internal/providerScale/manifests/loadbalancer.yaml
index e4407d2..b3ceae9 100644
--- a/internal/providerScale/manifests/loadbalancer.yaml
+++ b/internal/providerScale/manifests/loadbalancer.yaml
@@ -1,3 +1,17 @@
+# Copyright 2022 The Crossplane Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 apiVersion: network.azure.jet.crossplane.io/v1alpha2
 kind: LoadBalancer
 metadata:
diff --git a/internal/providerScale/manifests/virtualnetwork.yaml b/internal/providerScale/manifests/virtualnetwork.yaml
index 47bfb67..eb7917e 100644
--- a/internal/providerScale/manifests/virtualnetwork.yaml
+++ b/internal/providerScale/manifests/virtualnetwork.yaml
@@ -1,3 +1,17 @@
+# Copyright 2022 The Crossplane Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 apiVersion: network.azure.jet.crossplane.io/v1alpha2
 kind: VirtualNetwork
 metadata:

From edf9fffd2666fc691bc61a2355777d076112c23c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergen=20Yal=C3=A7=C4=B1n?= <yalcinsergen97@gmail.com>
Date: Thu, 24 Mar 2022 15:46:56 +0300
Subject: [PATCH 4/6] Add Readme
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Sergen Yalçın <yalcinsergen97@gmail.com>
---
 internal/providerScale/README.md       | 36 ++++++++++++++++++++++++++
 internal/providerScale/cmd/quantify.go | 36 +++++++++++++++++---------
 internal/providerScale/main.go         |  2 +-
 3 files changed, 61 insertions(+), 13 deletions(-)
 create mode 100644 internal/providerScale/README.md

diff --git a/internal/providerScale/README.md b/internal/providerScale/README.md
new file mode 100644
index 0000000..6c2e37b
--- /dev/null
+++ b/internal/providerScale/README.md
@@ -0,0 +1,36 @@
+# provider-scale
+
+This tool collects and reports some performance metrics. The details in the [one-pager].
+
+### Usage
+
+When this tool is executed, an end-to-end experiment will run. The inputs of this experiment can be manipulated by using
+the command-line options.
+
+```
+Flags:
+      --address string              Address of Prometheus service (default "http://localhost:9090")
+      --clean                       Delete deployed MRs (default true)
+  -h, --help                        help for provider-scale
+      --mrs stringToInt             Managed resource templates that will be deployed (default [])
+      --provider-namespace string   Namespace name of provider (default "crossplane-system")
+      --provider-pod string         Pod name of provider
+      --step-duration duration      Step duration between two data points (default 30s)
+```
+
+The `provider-pod` and `mrs` options are required.
+
+Example usage:
+
+```
+provider-scale --mrs ./internal/providerScale/manifests/virtualnetwork.yaml=2
+--mrs ./internal/providerScale/manifests/loadbalancer.yaml=2
+--provider-pod crossplane-provider-jet-azure
+--provider-namespace crossplane-system
+```
+
+With this input, two virtualnetwork & loadbalancer MRs will be deployed to the cluster.
+
+**_Note: In template manifests the name must be: `test-{{SUFFIX}}`_**
+
+[one-pager]: https://github.com/crossplane/crossplane/pull/2983
\ No newline at end of file
diff --git a/internal/providerScale/cmd/quantify.go b/internal/providerScale/cmd/quantify.go
index db4a4ba..33aa607 100644
--- a/internal/providerScale/cmd/quantify.go
+++ b/internal/providerScale/cmd/quantify.go
@@ -44,18 +44,30 @@ type QuantifyOptions struct {
 func NewCmdQuantify() *cobra.Command {
 	o := QuantifyOptions{}
 	o.cmd = &cobra.Command{
-		Use:     "Quantify the performance metrics",
-		Short:   "quantify",
-		Example: "Example",
-		RunE:    o.Run,
+		Use: "provider-scale [flags]",
+		Short: "This tool collects CPU & Memory Utilization and time to readiness of MRs metrics of providers and " +
+			"reports them. When you execute this tool an end-to-end experiment will run.",
+		Example: "provider-scale --mrs ./internal/providerScale/manifests/virtualnetwork.yaml=2 " +
+			"--mrs ./internal/providerScale/manifests/loadbalancer.yaml=2" +
+			"--provider-pod crossplane-provider-jet-azure " +
+			"--provider-namespace crossplane-system",
+		RunE: o.Run,
 	}
 
-	o.cmd.Flags().StringVar(&o.providerPod, "provider-pod", "", "provider pod name...")
-	o.cmd.Flags().StringVar(&o.providerNamespace, "provider-namespace", "crossplane-system", "provider namespace...")
-	o.cmd.Flags().StringToIntVar(&o.mrPaths, "mrs", map[string]int{}, "mr paths...")
-	o.cmd.Flags().StringVar(&o.address, "address", "http://localhost:9090", "address...")
-	o.cmd.Flags().DurationVar(&o.stepDuration, "step-duration", 30*time.Second, "step duration...")
-	o.cmd.Flags().BoolVar(&o.clean, "clean", true, "clean cluster...")
+	o.cmd.Flags().StringVar(&o.providerPod, "provider-pod", "", "Pod name of provider")
+	o.cmd.Flags().StringVar(&o.providerNamespace, "provider-namespace", "crossplane-system",
+		"Namespace name of provider")
+	o.cmd.Flags().StringToIntVar(&o.mrPaths, "mrs", nil, "Managed resource templates that will be deployed")
+	o.cmd.Flags().StringVar(&o.address, "address", "http://localhost:9090", "Address of Prometheus service")
+	o.cmd.Flags().DurationVar(&o.stepDuration, "step-duration", 30*time.Second, "Step duration between two data points")
+	o.cmd.Flags().BoolVar(&o.clean, "clean", true, "Delete deployed MRs")
+
+	if err := o.cmd.MarkFlagRequired("provider-pod"); err != nil {
+		panic(err)
+	}
+	if err := o.cmd.MarkFlagRequired("mrs"); err != nil {
+		panic(err)
+	}
 
 	return o.cmd
 }
@@ -95,8 +107,8 @@ func (o *QuantifyOptions) Run(_ *cobra.Command, _ []string) error {
 		fmt.Printf("Average Time to Readiness of %s: %f \n", timeToReadinessResult.Metric, timeToReadinessResult.Average)
 		fmt.Printf("Peak Time to Readiness of %s: %f \n", timeToReadinessResult.Metric, timeToReadinessResult.Peak)
 	}
-	fmt.Printf("Average CPU: %f %% \n", cpuResult.Average)
-	fmt.Printf("Peak CPU: %f %% \n", cpuResult.Peak)
+	fmt.Printf("Average CPU: %f seconds \n", cpuResult.Average)
+	fmt.Printf("Peak CPU: %f seconds \n", cpuResult.Peak)
 	fmt.Printf("Average Memory: %f Bytes \n", memoryResult.Average)
 	fmt.Printf("Peak Memory: %f Bytes \n", memoryResult.Peak)
 	return nil
diff --git a/internal/providerScale/main.go b/internal/providerScale/main.go
index 730d99e..fa69d1d 100644
--- a/internal/providerScale/main.go
+++ b/internal/providerScale/main.go
@@ -23,7 +23,7 @@ import (
 )
 
 func main() {
-	pflag.CommandLine = pflag.NewFlagSet("quantify", pflag.ExitOnError)
+	pflag.CommandLine = pflag.NewFlagSet("provider-scale", pflag.ExitOnError)
 	root := cmd.NewCmdQuantify()
 	if err := root.Execute(); err != nil {
 		os.Exit(1)

From e146ddac98d26f39a58a6ddfe37f36b1661484e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergen=20Yal=C3=A7=C4=B1n?= <yalcinsergen97@gmail.com>
Date: Mon, 4 Apr 2022 13:08:24 +0300
Subject: [PATCH 5/6] Remove bash scripts
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Sergen Yalçın <yalcinsergen97@gmail.com>
---
 .github/workflows/ci.yml                      |   1 +
 go.mod                                        |   3 +
 go.sum                                        |   2 +
 internal/providerScale/README.md              |   2 -
 internal/providerScale/cmd/common/common.go   |  14 +-
 .../cmd/managed/checkDeletion.sh              |  24 --
 .../cmd/managed/checkReadiness.sh             |  24 --
 .../cmd/managed/getCreationTime.sh            |  24 --
 .../cmd/managed/getReadinessTime.sh           |  24 --
 .../providerScale/cmd/managed/manage-mr.sh    |  27 --
 internal/providerScale/cmd/managed/managed.go | 269 +++++++++++++-----
 internal/providerScale/cmd/quantify.go        |  25 +-
 internal/providerScale/main.go                |   4 +-
 .../providerScale/manifests/loadbalancer.yaml |   2 +-
 .../manifests/virtualnetwork.yaml             |   2 +-
 15 files changed, 235 insertions(+), 212 deletions(-)
 delete mode 100755 internal/providerScale/cmd/managed/checkDeletion.sh
 delete mode 100755 internal/providerScale/cmd/managed/checkReadiness.sh
 delete mode 100755 internal/providerScale/cmd/managed/getCreationTime.sh
 delete mode 100755 internal/providerScale/cmd/managed/getReadinessTime.sh
 delete mode 100755 internal/providerScale/cmd/managed/manage-mr.sh

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f0cdb7a..dc792f8 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -75,6 +75,7 @@ jobs:
         uses: golangci/golangci-lint-action@v2
         with:
           version: ${{ env.GOLANGCI_VERSION }}
+          skip-go-installation: true
 
   check-diff:
     runs-on: ubuntu-18.04
diff --git a/go.mod b/go.mod
index f64336f..7e90ab1 100644
--- a/go.mod
+++ b/go.mod
@@ -10,11 +10,14 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.7.1
 	github.com/prometheus/common v0.10.0
+	github.com/sirupsen/logrus v1.6.0
 	github.com/spf13/cobra v1.1.1
 	github.com/spf13/pflag v1.0.5
+	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.20.2
 	k8s.io/apiextensions-apiserver v0.20.1
 	k8s.io/apimachinery v0.20.2
+	k8s.io/client-go v0.20.2
 	k8s.io/utils v0.0.0-20210111153108-fddb29f9d009
 	sigs.k8s.io/controller-runtime v0.8.3
 )
diff --git a/go.sum b/go.sum
index 6c24cd7..13cfa29 100644
--- a/go.sum
+++ b/go.sum
@@ -366,6 +366,7 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -497,6 +498,7 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
diff --git a/internal/providerScale/README.md b/internal/providerScale/README.md
index 6c2e37b..53728e7 100644
--- a/internal/providerScale/README.md
+++ b/internal/providerScale/README.md
@@ -31,6 +31,4 @@ provider-scale --mrs ./internal/providerScale/manifests/virtualnetwork.yaml=2
 
 With this input, two virtualnetwork & loadbalancer MRs will be deployed to the cluster.
 
-**_Note: In template manifests the name must be: `test-{{SUFFIX}}`_**
-
 [one-pager]: https://github.com/crossplane/crossplane/pull/2983
\ No newline at end of file
diff --git a/internal/providerScale/cmd/common/common.go b/internal/providerScale/cmd/common/common.go
index df7b4c4..47e2a9c 100644
--- a/internal/providerScale/cmd/common/common.go
+++ b/internal/providerScale/cmd/common/common.go
@@ -20,6 +20,8 @@ import (
 	"strconv"
 	"time"
 
+	log "github.com/sirupsen/logrus"
+
 	"github.com/prometheus/common/model"
 
 	"github.com/prometheus/client_golang/api"
@@ -36,6 +38,7 @@ type Data struct {
 type Result struct {
 	Data          []Data
 	Metric        string
+	MetricUnit    string
 	Peak, Average float64
 }
 
@@ -63,13 +66,11 @@ func ConstructTimeRange(startTime, endTime time.Time, stepDuration time.Duration
 }
 
 // ConstructResult creates a Result object from collected data
-func ConstructResult(value model.Value) (*Result, error) {
+func ConstructResult(value model.Value, metric, unit string) (*Result, error) {
 	result := &Result{}
 	matrix := value.(model.Matrix)
 
 	for _, m := range matrix {
-		result.Metric = m.Metric.String()
-
 		for _, v := range m.Values {
 			valueNum, err := strconv.ParseFloat(v.Value.String(), 64)
 			if err != nil {
@@ -80,6 +81,8 @@ func ConstructResult(value model.Value) (*Result, error) {
 	}
 
 	result.Average, result.Peak = CalculateAverageAndPeak(result.Data)
+	result.Metric = metric
+	result.MetricUnit = unit
 	return result, nil
 }
 
@@ -95,3 +98,8 @@ func CalculateAverageAndPeak(data []Data) (float64, float64) {
 	}
 	return sum / float64(len(data)), peak
 }
+
+func (r Result) String() {
+	log.Info(fmt.Sprintf("Average %s: %f %s \n", r.Metric, r.Average, r.MetricUnit))
+	log.Info(fmt.Sprintf("Peak %s: %f %s \n", r.Metric, r.Peak, r.MetricUnit))
+}
diff --git a/internal/providerScale/cmd/managed/checkDeletion.sh b/internal/providerScale/cmd/managed/checkDeletion.sh
deleted file mode 100755
index 03f582b..0000000
--- a/internal/providerScale/cmd/managed/checkDeletion.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env zsh
-# Copyright 2022 The Crossplane Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# HO.
-
-set -euxo pipefail
-
-TEMPLATE="$1"
-NAME="$2"
-
-cat "${TEMPLATE}" | sed "s/{{SUFFIX}}/$NAME/g" | kubectl get --ignore-not-found -f -
\ No newline at end of file
diff --git a/internal/providerScale/cmd/managed/checkReadiness.sh b/internal/providerScale/cmd/managed/checkReadiness.sh
deleted file mode 100755
index 5c9e127..0000000
--- a/internal/providerScale/cmd/managed/checkReadiness.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env zsh
-# Copyright 2022 The Crossplane Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# HO.
-
-set -euxo pipefail
-
-TEMPLATE="$1"
-NAME="$2"
-
-cat "${TEMPLATE}" | sed "s/{{SUFFIX}}/$NAME/g" | kubectl get -o json -f - | jq '.items[] | .status.conditions[] | select(.type=="Ready")' | jq .status
diff --git a/internal/providerScale/cmd/managed/getCreationTime.sh b/internal/providerScale/cmd/managed/getCreationTime.sh
deleted file mode 100755
index 9724e4b..0000000
--- a/internal/providerScale/cmd/managed/getCreationTime.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env zsh
-# Copyright 2022 The Crossplane Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# HO.
-
-set -euxo pipefail
-
-TEMPLATE="$1"
-NAME="$2"
-
-cat "${TEMPLATE}" | sed "s/{{SUFFIX}}/$NAME/g" | kubectl get -o json -f - | jq -r '.items[] | .metadata.creationTimestamp'
diff --git a/internal/providerScale/cmd/managed/getReadinessTime.sh b/internal/providerScale/cmd/managed/getReadinessTime.sh
deleted file mode 100755
index 29a340e..0000000
--- a/internal/providerScale/cmd/managed/getReadinessTime.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env zsh
-# Copyright 2022 The Crossplane Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# HO.
-
-set -euxo pipefail
-
-TEMPLATE="$1"
-NAME="$2"
-
-cat "${TEMPLATE}" | sed "s/{{SUFFIX}}/$NAME/g" | kubectl get -o json -f - | jq '.items[] | .status.conditions[] | select(.type=="Ready" and .status=="True" )' | jq .lastTransitionTime
diff --git a/internal/providerScale/cmd/managed/manage-mr.sh b/internal/providerScale/cmd/managed/manage-mr.sh
deleted file mode 100755
index a8ce079..0000000
--- a/internal/providerScale/cmd/managed/manage-mr.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/usr/bin/env zsh
-# Copyright 2022 The Crossplane Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# HO.
-
-set -euxo pipefail
-
-# Example usage: ./manage-mr.sh create virtualnetwork.yaml 10
-# Example usage: ./manage-mr.sh delete virtualnetwork.yaml 10 | tee exp-virtualnetwork.log
-OPERATION="$1"
-TEMPLATE="$2"
-NAME="$3"
-
-cat "${TEMPLATE}" | sed "s/{{SUFFIX}}/$NAME/g" | kubectl --wait=false "${OPERATION}" -f -
diff --git a/internal/providerScale/cmd/managed/managed.go b/internal/providerScale/cmd/managed/managed.go
index 922c9ef..713683c 100644
--- a/internal/providerScale/cmd/managed/managed.go
+++ b/internal/providerScale/cmd/managed/managed.go
@@ -15,111 +15,246 @@
 package managed
 
 import (
+	"context"
 	"fmt"
-	"os/exec"
-	"strconv"
+	"io/ioutil"
 	"strings"
 	"time"
 
+	log "github.com/sirupsen/logrus"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+
+	"gopkg.in/yaml.v2"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
+	"k8s.io/client-go/dynamic"
+
+	ctrl "sigs.k8s.io/controller-runtime"
+
 	"github.com/crossplane/conformance/internal/providerScale/cmd/common"
 )
 
 // RunExperiment runs the experiment according to command-line inputs.
 // Firstly the input manifests are deployed. After the all MRs are ready, time to readiness metrics are calculated.
 // Then, by default, all deployed MRs are deleted.
-func RunExperiment(mrTemplatePaths map[string]int, clean bool) ([]*common.Result, error) { //nolint:gocyclo
-	var timeToReadinessResults []*common.Result //nolint:prealloc
+func RunExperiment(mrTemplatePaths map[string]int, clean bool) ([]common.Result, error) { //nolint:gocyclo
+	var timeToReadinessResults []common.Result //nolint:prealloc
+
+	client := createDynamicClient()
+
+	if err := applyResources(client, mrTemplatePaths); err != nil {
+		return nil, err
+	}
+
+	if err := checkReadiness(client, mrTemplatePaths); err != nil {
+		return nil, err
+	}
+
+	timeToReadinessResults, err := calculateReadinessDuration(client, mrTemplatePaths)
+	if err != nil {
+		return nil, err
+	}
+
+	if clean {
+		log.Info("Deleting resources...")
+		if err := deleteResources(client, mrTemplatePaths); err != nil {
+			return nil, err
+		}
+		log.Info("Checking deletion of resources...")
+		if err := checkDeletion(client, mrTemplatePaths); err != nil {
+			return nil, err
+		}
+	}
+	return timeToReadinessResults, nil
+}
 
+func applyResources(client dynamic.Interface, mrTemplatePaths map[string]int) error {
 	for mrPath, count := range mrTemplatePaths {
+		m, err := readYamlFile(mrPath)
+		if err != nil {
+			return err
+		}
+		o := prepareUnstructuredObject(m)
+
 		for i := 1; i <= count; i++ {
-			cmd, err := exec.Command("./internal/providerScale/cmd/managed/manage-mr.sh", "apply", mrPath, strconv.Itoa(i)).Output() //nolint:gosec
-			fmt.Print(string(cmd))
+			o["metadata"].(map[string]interface{})["name"] = fmt.Sprintf("test-%d", i)
+			_, err := client.Resource(prepareGvk(m)).Create(context.TODO(), &unstructured.Unstructured{Object: o}, metav1.CreateOptions{})
 			if err != nil {
-				return nil, err
+				return err
 			}
+			log.Info(fmt.Sprintf("%s/%s was successfully created!\n", m["kind"], o["metadata"].(map[string]interface{})["name"]))
 		}
 	}
-	for mrPath, count := range mrTemplatePaths {
-		checkReadiness(mrPath, count)
-	}
-	for mrPath, count := range mrTemplatePaths {
-		timeToReadinessResult, err := calculateReadinessDuration(mrPath, count)
+	return nil
+}
+
+func deleteResources(client dynamic.Interface, mrTemplatePaths map[string]int) error {
+	for mrPath := range mrTemplatePaths {
+		m, err := readYamlFile(mrPath)
 		if err != nil {
-			return nil, err
+			return err
+		}
+
+		background := metav1.DeletePropagationBackground
+		if err := client.Resource(prepareGvk(m)).DeleteCollection(context.TODO(),
+			metav1.DeleteOptions{PropagationPolicy: &background}, metav1.ListOptions{}); err != nil {
+			return err
 		}
-		timeToReadinessResults = append(timeToReadinessResults, timeToReadinessResult)
 	}
-	if clean {
-		for mrPath, count := range mrTemplatePaths {
-			fmt.Println("Deleting resources...")
-			for i := 1; i <= count; i++ {
-				cmd, err := exec.Command("./internal/providerScale/cmd/managed/manage-mr.sh", "delete", mrPath, strconv.Itoa(i)).Output() //nolint:gosec
-				fmt.Print(string(cmd))
-				if err != nil {
-					return nil, err
-				}
-			}
+	return nil
+}
+
+func checkReadiness(client dynamic.Interface, mrTemplatePaths map[string]int) error {
+	for mrPath := range mrTemplatePaths {
+		m, err := readYamlFile(mrPath)
+		if err != nil {
+			return err
 		}
-		for mrPath, count := range mrTemplatePaths {
-			i := 1
-			for i <= count {
-				fmt.Println("Checking deletion of resources...")
-				cmd, err := exec.Command("./internal/providerScale/cmd/managed/checkDeletion.sh", mrPath, strconv.Itoa(i)).Output() //nolint:gosec
-				if err != nil {
-					return nil, err
-				}
-				if len(cmd) != 0 {
-					time.Sleep(10 * time.Second)
-					continue
-				}
-				i++
+
+		for {
+			log.Info("Checking readiness of resources...")
+			list, err := client.Resource(prepareGvk(m)).List(context.TODO(), metav1.ListOptions{})
+			if err != nil {
+				return err
 			}
+			if isReady(list) {
+				break
+			}
+			time.Sleep(10 * time.Second)
 		}
 	}
-	return timeToReadinessResults, nil
+	return nil
 }
 
-func checkReadiness(mrPath string, count int) {
-	i := 1
-	for i <= count {
-		fmt.Println("Checking readiness of resources...")
-		isReady, _ := exec.Command("./internal/providerScale/cmd/managed/checkReadiness.sh", mrPath, strconv.Itoa(i)).Output() //nolint:gosec
-		if !strings.Contains(string(isReady), "True") {
-			time.Sleep(10 * time.Second)
-			continue
+func isReady(list *unstructured.UnstructuredList) bool {
+	for _, l := range list.Items {
+		if l.Object["status"] == nil {
+			return false
+		}
+		conditions := l.Object["status"].(map[string]interface{})["conditions"].([]interface{})
+
+		status := ""
+		for _, condition := range conditions {
+			c := condition.(map[string]interface{})
+			if c["type"] == "Ready" {
+				status = c["status"].(string)
+			}
+		}
+
+		if status == "False" || status == "" {
+			return false
 		}
-		i++
 	}
+	return true
 }
 
-func calculateReadinessDuration(mrPath string, count int) (*common.Result, error) {
-	result := &common.Result{}
-	for i := 1; i <= count; i++ {
-		fmt.Println("Calculating readiness durations of resources...")
-		creationTimeByte, err := exec.Command("./internal/providerScale/cmd/managed/getCreationTime.sh", mrPath, strconv.Itoa(i)).Output() //nolint:gosec
+func calculateReadinessDuration(client dynamic.Interface, mrTemplatePaths map[string]int) ([]common.Result, error) {
+	var results []common.Result //nolint:prealloc
+	for mrPath := range mrTemplatePaths {
+		log.Info("Calculating readiness time of resources...")
+		var result common.Result
+
+		m, err := readYamlFile(mrPath)
 		if err != nil {
 			return nil, err
 		}
-		readinessTimeByte, err := exec.Command("./internal/providerScale/cmd/managed/getReadinessTime.sh", mrPath, strconv.Itoa(i)).Output() //nolint:gosec
+
+		list, err := client.Resource(prepareGvk(m)).List(context.TODO(), metav1.ListOptions{})
 		if err != nil {
 			return nil, err
 		}
-		creationTimeString := string(creationTimeByte)
-		creationTimeString = creationTimeString[:strings.Index(creationTimeString, `Z`)+1]
-		creationTime, err := time.Parse(time.RFC3339, creationTimeString)
-		if err != nil {
-			return nil, err
+		for _, l := range list.Items {
+			readinessTime := metav1.Time{}
+			creationTimestamp := l.GetCreationTimestamp()
+			conditions := l.Object["status"].(map[string]interface{})["conditions"].([]interface{})
+
+			for _, condition := range conditions {
+				c := condition.(map[string]interface{})
+
+				if c["type"] == "Ready" && c["status"] == "True" {
+					t, err := time.Parse(time.RFC3339, c["lastTransitionTime"].(string))
+					if err != nil {
+						return nil, err
+					}
+					readinessTime.Time = t
+				}
+
+				diff := readinessTime.Sub(creationTimestamp.Time)
+				result.Data = append(result.Data, common.Data{Value: diff.Seconds()})
+				break
+			}
 		}
-		readinessTimeString := string(readinessTimeByte)
-		readinessTimeString = readinessTimeString[strings.Index(readinessTimeString, `"`)+1 : strings.Index(readinessTimeString, `Z`)+1]
-		readinessTime, err := time.Parse(time.RFC3339, readinessTimeString)
+		result.Metric = fmt.Sprintf("Time to Readiness of %s", m["kind"])
+		result.MetricUnit = "seconds"
+		result.Average, result.Peak = common.CalculateAverageAndPeak(result.Data)
+		results = append(results, result)
+	}
+	return results, nil
+}
+
+func checkDeletion(client dynamic.Interface, mrTemplatePaths map[string]int) error {
+	for mrPath := range mrTemplatePaths {
+		m, err := readYamlFile(mrPath)
 		if err != nil {
-			return nil, err
+			return err
+		}
+
+		for {
+			log.Info("Checking deletion of resources...")
+			list, err := client.Resource(prepareGvk(m)).List(context.TODO(), metav1.ListOptions{})
+			if err != nil {
+				return err
+			}
+			if len(list.Items) == 0 {
+				break
+			}
+			time.Sleep(10 * time.Second)
 		}
-		diff := readinessTime.Sub(creationTime)
-		result.Data = append(result.Data, common.Data{Value: diff.Seconds()})
 	}
-	result.Metric = mrPath
-	result.Average, result.Peak = common.CalculateAverageAndPeak(result.Data)
-	return result, nil
+	return nil
+}
+
+func prepareGvk(m map[interface{}]interface{}) schema.GroupVersionResource {
+	apiVersion := strings.Split(m["apiVersion"].(string), "/")
+	kind := strings.ToLower(m["kind"].(string))
+	return schema.GroupVersionResource{
+		Group:    apiVersion[0],
+		Version:  apiVersion[1],
+		Resource: fmt.Sprintf("%ss", kind),
+	}
+}
+
+func prepareUnstructuredObject(m map[interface{}]interface{}) map[string]interface{} {
+	result := map[string]interface{}{}
+	for k, v := range m {
+		t, ok := v.(map[interface{}]interface{})
+		if ok {
+			result[k.(string)] = prepareUnstructuredObject(t)
+		} else {
+			result[k.(string)] = v
+		}
+	}
+	return result
+}
+
+func readYamlFile(fileName string) (map[interface{}]interface{}, error) {
+	yamlFile, err := ioutil.ReadFile(fileName) //nolint:gosec
+	if err != nil {
+		return nil, err
+	}
+
+	m := make(map[interface{}]interface{})
+	err = yaml.Unmarshal(yamlFile, m)
+	if err != nil {
+		return nil, err
+	}
+
+	return m, nil
+}
+
+func createDynamicClient() dynamic.Interface {
+	return dynamic.NewForConfigOrDie(ctrl.GetConfigOrDie())
 }
diff --git a/internal/providerScale/cmd/quantify.go b/internal/providerScale/cmd/quantify.go
index 33aa607..52134c0 100644
--- a/internal/providerScale/cmd/quantify.go
+++ b/internal/providerScale/cmd/quantify.go
@@ -24,6 +24,8 @@ import (
 
 	"github.com/prometheus/common/model"
 
+	log "github.com/sirupsen/logrus"
+
 	"github.com/spf13/cobra"
 )
 
@@ -75,21 +77,21 @@ func NewCmdQuantify() *cobra.Command {
 // Run executes the quantify command's tasks.
 func (o *QuantifyOptions) Run(_ *cobra.Command, _ []string) error {
 	o.startTime = time.Now()
-	fmt.Printf("Experiment Started %v\n\n", o.startTime)
+	log.Infof("Experiment Started %v\n\n", o.startTime)
 	timeToReadinessResults, err := managed.RunExperiment(o.mrPaths, o.clean)
 	if err != nil {
 		return err
 	}
 	o.endTime = time.Now()
-	fmt.Printf("\nExperiment Ended %v\n\n", o.endTime)
-	fmt.Printf("Results\n------------------------------------------------------------\n")
-	fmt.Printf("Experiment Duration: %f seconds\n", o.endTime.Sub(o.startTime).Seconds())
+	log.Infof("\nExperiment Ended %v\n\n", o.endTime)
+	log.Infof("Results\n------------------------------------------------------------\n")
+	log.Infof("Experiment Duration: %f seconds\n", o.endTime.Sub(o.startTime).Seconds())
 	queryResultCPU, err := o.CollectData(fmt.Sprintf(`sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{pod="%s", namespace="%s"})*100`,
 		o.providerPod, o.providerNamespace))
 	if err != nil {
 		return err
 	}
-	cpuResult, err := common.ConstructResult(queryResultCPU)
+	cpuResult, err := common.ConstructResult(queryResultCPU, "CPU", "seconds")
 	if err != nil {
 		return err
 	}
@@ -98,19 +100,16 @@ func (o *QuantifyOptions) Run(_ *cobra.Command, _ []string) error {
 	if err != nil {
 		return err
 	}
-	memoryResult, err := common.ConstructResult(queryResultMemory)
+	memoryResult, err := common.ConstructResult(queryResultMemory, "Memory", "Bytes")
 	if err != nil {
 		return err
 	}
 
 	for _, timeToReadinessResult := range timeToReadinessResults {
-		fmt.Printf("Average Time to Readiness of %s: %f \n", timeToReadinessResult.Metric, timeToReadinessResult.Average)
-		fmt.Printf("Peak Time to Readiness of %s: %f \n", timeToReadinessResult.Metric, timeToReadinessResult.Peak)
+		timeToReadinessResult.String()
 	}
-	fmt.Printf("Average CPU: %f seconds \n", cpuResult.Average)
-	fmt.Printf("Peak CPU: %f seconds \n", cpuResult.Peak)
-	fmt.Printf("Average Memory: %f Bytes \n", memoryResult.Average)
-	fmt.Printf("Peak Memory: %f Bytes \n", memoryResult.Peak)
+	cpuResult.String()
+	memoryResult.String()
 	return nil
 }
 
@@ -126,7 +125,7 @@ func (o *QuantifyOptions) CollectData(query string) (model.Value, error) {
 		return nil, err
 	}
 	if len(warnings) > 0 {
-		fmt.Printf("Warnings: %v\n", warnings)
+		log.Infof("Warnings: %v\n", warnings)
 	}
 	return result, err
 }
diff --git a/internal/providerScale/main.go b/internal/providerScale/main.go
index fa69d1d..838be20 100644
--- a/internal/providerScale/main.go
+++ b/internal/providerScale/main.go
@@ -17,9 +17,9 @@ package main
 import (
 	"os"
 
-	"github.com/crossplane/conformance/internal/providerScale/cmd"
-
 	"github.com/spf13/pflag"
+
+	"github.com/crossplane/conformance/internal/providerScale/cmd"
 )
 
 func main() {
diff --git a/internal/providerScale/manifests/loadbalancer.yaml b/internal/providerScale/manifests/loadbalancer.yaml
index b3ceae9..0159915 100644
--- a/internal/providerScale/manifests/loadbalancer.yaml
+++ b/internal/providerScale/manifests/loadbalancer.yaml
@@ -15,7 +15,7 @@
 apiVersion: network.azure.jet.crossplane.io/v1alpha2
 kind: LoadBalancer
 metadata:
-  name: test-{{SUFFIX}}
+  name: test
 spec:
   forProvider:
     location: East US
diff --git a/internal/providerScale/manifests/virtualnetwork.yaml b/internal/providerScale/manifests/virtualnetwork.yaml
index eb7917e..214755f 100644
--- a/internal/providerScale/manifests/virtualnetwork.yaml
+++ b/internal/providerScale/manifests/virtualnetwork.yaml
@@ -15,7 +15,7 @@
 apiVersion: network.azure.jet.crossplane.io/v1alpha2
 kind: VirtualNetwork
 metadata:
-  name: test-{{SUFFIX}}
+  name: test
 spec:
   forProvider:
     addressSpace:

From 23851c9e39bd913b4e90c6dd12094a871c585c11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergen=20Yal=C3=A7=C4=B1n?= <yalcinsergen97@gmail.com>
Date: Thu, 20 Jul 2023 18:46:22 +0200
Subject: [PATCH 6/6] Add performance metrics
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Sergen Yalçın <yalcinsergen97@gmail.com>
---
 internal/providerScale/cmd/managed/managed.go | 66 +++++++++++++++----
 internal/providerScale/cmd/quantify.go        | 19 +++---
 internal/providerScale/manifests/ecr.yaml     | 12 ++++
 3 files changed, 77 insertions(+), 20 deletions(-)
 create mode 100644 internal/providerScale/manifests/ecr.yaml

diff --git a/internal/providerScale/cmd/managed/managed.go b/internal/providerScale/cmd/managed/managed.go
index 713683c..a2bdcd8 100644
--- a/internal/providerScale/cmd/managed/managed.go
+++ b/internal/providerScale/cmd/managed/managed.go
@@ -15,23 +15,22 @@
 package managed
 
 import (
+	"bufio"
 	"context"
 	"fmt"
 	"io/ioutil"
+	"os"
+	"os/exec"
 	"strings"
 	"time"
 
+	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
-
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-
 	"gopkg.in/yaml.v2"
-
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-
 	"k8s.io/client-go/dynamic"
-
 	ctrl "sigs.k8s.io/controller-runtime"
 
 	"github.com/crossplane/conformance/internal/providerScale/cmd/common"
@@ -72,6 +71,14 @@ func RunExperiment(mrTemplatePaths map[string]int, clean bool) ([]common.Result,
 }
 
 func applyResources(client dynamic.Interface, mrTemplatePaths map[string]int) error {
+	file, err := os.Create(fmt.Sprintf("/tmp/test.yaml"))
+	if err != nil {
+		return err
+	}
+	if err := file.Close(); err != nil {
+		return err
+	}
+
 	for mrPath, count := range mrTemplatePaths {
 		m, err := readYamlFile(mrPath)
 		if err != nil {
@@ -79,14 +86,43 @@ func applyResources(client dynamic.Interface, mrTemplatePaths map[string]int) er
 		}
 		o := prepareUnstructuredObject(m)
 
+		f, err := os.OpenFile("/tmp/test.yaml", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
+		if err != nil {
+			return err
+		}
+
 		for i := 1; i <= count; i++ {
 			o["metadata"].(map[string]interface{})["name"] = fmt.Sprintf("test-%d", i)
-			_, err := client.Resource(prepareGvk(m)).Create(context.TODO(), &unstructured.Unstructured{Object: o}, metav1.CreateOptions{})
+
+			b, err := yaml.Marshal(o)
 			if err != nil {
 				return err
 			}
+
+			if _, err := f.Write(b); err != nil {
+				return err
+			}
+
+			if _, err := f.WriteString("\n---\n\n"); err != nil {
+				return err
+			}
+
 			log.Info(fmt.Sprintf("%s/%s was successfully created!\n", m["kind"], o["metadata"].(map[string]interface{})["name"]))
 		}
+
+		cmd := exec.Command("bash", "-c", fmt.Sprintf(`"kubectl" apply -f /tmp/test.yaml`)) // #nosec G204
+		stdout, _ := cmd.StdoutPipe()
+		if err := cmd.Start(); err != nil {
+			return errors.Wrap(err, "cannot start kubectl")
+		}
+		sc := bufio.NewScanner(stdout)
+		sc.Split(bufio.ScanLines)
+		for sc.Scan() {
+			fmt.Println(sc.Text())
+		}
+		if err := cmd.Wait(); err != nil {
+			return err
+		}
 	}
 	return nil
 }
@@ -180,11 +216,11 @@ func calculateReadinessDuration(client dynamic.Interface, mrTemplatePaths map[st
 						return nil, err
 					}
 					readinessTime.Time = t
-				}
 
-				diff := readinessTime.Sub(creationTimestamp.Time)
-				result.Data = append(result.Data, common.Data{Value: diff.Seconds()})
-				break
+					diff := readinessTime.Sub(creationTimestamp.Time)
+					result.Data = append(result.Data, common.Data{Value: diff.Seconds()})
+					break
+				}
 			}
 		}
 		result.Metric = fmt.Sprintf("Time to Readiness of %s", m["kind"])
@@ -218,12 +254,18 @@ func checkDeletion(client dynamic.Interface, mrTemplatePaths map[string]int) err
 }
 
 func prepareGvk(m map[interface{}]interface{}) schema.GroupVersionResource {
+	suffix := "s"
 	apiVersion := strings.Split(m["apiVersion"].(string), "/")
 	kind := strings.ToLower(m["kind"].(string))
+
+	if kind[len(kind)-1] == 'y' {
+		kind = kind[:len(kind)-1]
+		suffix = "ies"
+	}
 	return schema.GroupVersionResource{
 		Group:    apiVersion[0],
 		Version:  apiVersion[1],
-		Resource: fmt.Sprintf("%ss", kind),
+		Resource: fmt.Sprintf("%s%s", kind, suffix),
 	}
 }
 
diff --git a/internal/providerScale/cmd/quantify.go b/internal/providerScale/cmd/quantify.go
index 52134c0..543705e 100644
--- a/internal/providerScale/cmd/quantify.go
+++ b/internal/providerScale/cmd/quantify.go
@@ -40,6 +40,7 @@ type QuantifyOptions struct {
 	endTime           time.Time
 	stepDuration      time.Duration
 	clean             bool
+	nodeIP            string
 }
 
 // NewCmdQuantify creates a cobra command
@@ -63,6 +64,7 @@ func NewCmdQuantify() *cobra.Command {
 	o.cmd.Flags().StringVar(&o.address, "address", "http://localhost:9090", "Address of Prometheus service")
 	o.cmd.Flags().DurationVar(&o.stepDuration, "step-duration", 30*time.Second, "Step duration between two data points")
 	o.cmd.Flags().BoolVar(&o.clean, "clean", true, "Delete deployed MRs")
+	o.cmd.Flags().StringVar(&o.nodeIP, "node", "", "Node IP")
 
 	if err := o.cmd.MarkFlagRequired("provider-pod"); err != nil {
 		panic(err)
@@ -86,30 +88,29 @@ func (o *QuantifyOptions) Run(_ *cobra.Command, _ []string) error {
 	log.Infof("\nExperiment Ended %v\n\n", o.endTime)
 	log.Infof("Results\n------------------------------------------------------------\n")
 	log.Infof("Experiment Duration: %f seconds\n", o.endTime.Sub(o.startTime).Seconds())
-	queryResultCPU, err := o.CollectData(fmt.Sprintf(`sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{pod="%s", namespace="%s"})*100`,
+	time.Sleep(60 * time.Second)
+	queryResultMemory, err := o.CollectData(fmt.Sprintf(`sum(node_namespace_pod_container:container_memory_working_set_bytes{pod="%s", namespace="%s"})`,
 		o.providerPod, o.providerNamespace))
 	if err != nil {
 		return err
 	}
-	cpuResult, err := common.ConstructResult(queryResultCPU, "CPU", "seconds")
+	memoryResult, err := common.ConstructResult(queryResultMemory, "Memory Working Set", "Bytes")
 	if err != nil {
 		return err
 	}
-	queryResultMemory, err := o.CollectData(fmt.Sprintf(`sum(node_namespace_pod_container:container_memory_working_set_bytes{pod="%s", namespace="%s"})`,
-		o.providerPod, o.providerNamespace))
+	qureyResultCPURate, err := o.CollectData(fmt.Sprintf(`instance:node_cpu_utilisation:rate5m{instance="%s"} * 100`, o.nodeIP))
 	if err != nil {
 		return err
 	}
-	memoryResult, err := common.ConstructResult(queryResultMemory, "Memory", "Bytes")
+	cpuRateResult, err := common.ConstructResult(qureyResultCPURate, "CPU Utilisation", "Rate")
 	if err != nil {
 		return err
 	}
-
 	for _, timeToReadinessResult := range timeToReadinessResults {
 		timeToReadinessResult.String()
 	}
-	cpuResult.String()
 	memoryResult.String()
+	cpuRateResult.String()
 	return nil
 }
 
@@ -119,7 +120,9 @@ func (o *QuantifyOptions) CollectData(query string) (model.Value, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 
-	r := common.ConstructTimeRange(o.startTime, o.endTime, o.stepDuration)
+	// We continue to collect metrics after the tests were ended. Because
+	// the memory consumption sometimes increase after the tests completed.
+	r := common.ConstructTimeRange(o.startTime, o.endTime.Add(60*time.Second), o.stepDuration)
 	result, warnings, err := client.QueryRange(ctx, query, r)
 	if err != nil {
 		return nil, err
diff --git a/internal/providerScale/manifests/ecr.yaml b/internal/providerScale/manifests/ecr.yaml
new file mode 100644
index 0000000..621818b
--- /dev/null
+++ b/internal/providerScale/manifests/ecr.yaml
@@ -0,0 +1,12 @@
+apiVersion: ecr.aws.upbound.io/v1beta1
+kind: Repository
+metadata:
+  name: upbound-sample-repository
+spec:
+  forProvider:
+    region: us-east-1
+    imageScanningConfiguration:
+      - scanOnPush: true
+    imageTagMutability: "IMMUTABLE"
+    tags:
+      key1: value1
\ No newline at end of file