From 626d54e9e7e7fbe5a4e81162d0b76b3b1204c329 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Wed, 1 Dec 2021 09:43:24 +0200 Subject: [PATCH 1/7] add pr for scrips only Signed-off-by: matancarmeli7 --- Makefile | 8 ++-- build/ci/Dockerfile.unittest | 6 +++ hack/check-files-alignment.sh | 74 +++++++++++++++++++++++++++++++ hack/check-generated-manifests.sh | 26 ----------- hack/get_information_helper.sh | 39 ++++++++++++++++ 5 files changed, 123 insertions(+), 30 deletions(-) create mode 100755 hack/check-files-alignment.sh delete mode 100755 hack/check-generated-manifests.sh create mode 100755 hack/get_information_helper.sh diff --git a/Makefile b/Makefile index 3c2315785..2e4f7ba2b 100644 --- a/Makefile +++ b/Makefile @@ -76,16 +76,16 @@ run-unit-tests: $(run_unit_tests_image) make test .PHONY: test -test: check-generated-manifests update +test: check-files-alignment update ginkgo -r -v -skipPackage tests .PHONY: update update: kustomize hack/update-all.sh -.PHONY: check-generated-manifests -check-generated-manifests: - hack/check-generated-manifests.sh +.PHONY: check-files-alignment +check-files-alignment: + hack/check-files-alignment.sh .PHONY: update-generated-yamls update-generated-yamls: diff --git a/build/ci/Dockerfile.unittest b/build/ci/Dockerfile.unittest index 13e0cf262..09fb9509f 100644 --- a/build/ci/Dockerfile.unittest +++ b/build/ci/Dockerfile.unittest @@ -25,6 +25,12 @@ RUN go get github.com/onsi/ginkgo/ginkgo@v1.16.4 \ && make kustomize \ && make controller-gen +ARG YQ_VERSION=v4.13.5 +ARG ARCH=yq_linux_386 +RUN ARCH=$(if [ "$(uname -m)" = "x86_64" ]; then echo "amd64"; else echo "$(uname -m)" ; fi) \ + && wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${ARCH} -O /usr/bin/yq \ + && chmod +x /usr/bin/yq + COPY . . CMD ["make", "test"] diff --git a/hack/check-files-alignment.sh b/hack/check-files-alignment.sh new file mode 100755 index 000000000..25ed63713 --- /dev/null +++ b/hack/check-files-alignment.sh @@ -0,0 +1,74 @@ +#!/bin/bash -e + +# +# Copyright 2019 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +operator_yaml_path=deploy/installer/generated/ibm-block-csi-operator.yaml +roles_yaml_path=config/rbac/role.yaml +origin_crd_yaml_path=config/crd/bases/csi.ibm.com_ibmblockcsis.yaml + +check_generation (){ + echo "check generation" + project_dirname=ibm-block-csi-operator + cd .. + cp -r $project_dirname ./$project_dirname-expected + cd $project_dirname-expected/ + make update + cd .. + diff -qr --exclude=bin $project_dirname $project_dirname-expected/ + rm -rf $project_dirname-expected/ + cd $project_dirname +} + +verify_full_operator_yaml_is_align(){ + echo "check full operator yaml alignment" + declare -A orignial_yamls=( + ["config/rbac/role.yaml"]="ClusterRole" + ["config/crd/bases/csi.ibm.com_ibmblockcsis.yaml"]="CustomResourceDefinition" + ["config/rbac/service_account.yaml"]="ServiceAccount" + ["config/rbac/role_binding.yaml"]="ClusterRoleBinding" + ["config/manager/manager.yaml"]="Deployment" + ) + for orignial_yaml in ${!orignial_yamls[@]}; do + export resource_type=${orignial_yamls[${orignial_yaml}]} + diff <(yq e '... comments=""' $orignial_yaml) <(yq eval '(. | select(.kind == env(resource_type)))' $operator_yaml_path) + done +} + +verify_no_roles_diff (){ + echo "check roles alignment" + source hack/get_information_helper.sh + are_manifest_files_exsists_in_current_csi_version + csv_files=$(get_csv_files) + for csv_file in $csv_files; do + diff <(yq e .rules $roles_yaml_path) <(yq e .spec.install.spec.clusterPermissions[0].rules $csv_file) + done +} + +verify_no_crds_diff (){ + echo "check crds alignment" + source hack/get_information_helper.sh + are_manifest_files_exsists_in_current_csi_version + crd_files=$(get_bundle_crds) + for crd_file in $crd_files; do + diff $origin_crd_yaml_path $crd_file + done +} + +check_generation +verify_full_operator_yaml_is_align +verify_no_roles_diff +verify_no_crds_diff diff --git a/hack/check-generated-manifests.sh b/hack/check-generated-manifests.sh deleted file mode 100755 index 4fc00972b..000000000 --- a/hack/check-generated-manifests.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -e - -# -# Copyright 2019 IBM Corp. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -project_dirname=ibm-block-csi-operator -cd .. -cp -r $project_dirname ./$project_dirname-expected -cd $project_dirname-expected/ -hack/update-config-yamls.sh -cd .. -diff -qr --exclude=bin $project_dirname $project_dirname-expected/ -rm -rf $project_dirname-expected/ diff --git a/hack/get_information_helper.sh b/hack/get_information_helper.sh new file mode 100755 index 000000000..c0fb75c6f --- /dev/null +++ b/hack/get_information_helper.sh @@ -0,0 +1,39 @@ +#!/bin/bash -e + +# +# Copyright 2019 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +get_current_csi_version (){ + current_csi_version=$(cat version/version.go | grep -i driverversion | awk -F = '{print $2}') + echo ${current_csi_version//\"} +} + +are_manifest_files_exsists_in_current_csi_version (){ + current_csi_version=$(get_current_csi_version) + if ! compgen -G "${PWD}/deploy/olm-catalog/*/$current_csi_version" > /dev/null; then + exit 0 + fi +} + +get_csv_files (){ + current_csi_version=$(get_current_csi_version) + ls deploy/olm-catalog/*/$current_csi_version/manifests/ibm-block-csi-operator.v$current_csi_version.clusterserviceversion.yaml +} + +get_bundle_crds (){ + current_csi_version=$(get_current_csi_version) + ls deploy/olm-catalog/*/$current_csi_version/manifests/csi.ibm.com_ibmblockcsis.yaml +} From afcb83e53488bd038841e4ac2bc3302865dbce8e Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Wed, 1 Dec 2021 11:08:50 +0200 Subject: [PATCH 2/7] change yq download Signed-off-by: matancarmeli7 --- build/ci/Dockerfile.unittest | 7 +------ hack/check-files-alignment.sh | 14 ++++++-------- hack/get_information_helper.sh | 2 ++ hack/update-installer.sh | 2 +- 4 files changed, 10 insertions(+), 15 deletions(-) diff --git a/build/ci/Dockerfile.unittest b/build/ci/Dockerfile.unittest index 09fb9509f..1faeabfde 100644 --- a/build/ci/Dockerfile.unittest +++ b/build/ci/Dockerfile.unittest @@ -22,15 +22,10 @@ WORKDIR $WORKDIR COPY Makefile . RUN go get github.com/onsi/ginkgo/ginkgo@v1.16.4 \ + && go get github.com/mikefarah/yq/v4 \ && make kustomize \ && make controller-gen -ARG YQ_VERSION=v4.13.5 -ARG ARCH=yq_linux_386 -RUN ARCH=$(if [ "$(uname -m)" = "x86_64" ]; then echo "amd64"; else echo "$(uname -m)" ; fi) \ - && wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${ARCH} -O /usr/bin/yq \ - && chmod +x /usr/bin/yq - COPY . . CMD ["make", "test"] diff --git a/hack/check-files-alignment.sh b/hack/check-files-alignment.sh index 25ed63713..c4a23ca29 100755 --- a/hack/check-files-alignment.sh +++ b/hack/check-files-alignment.sh @@ -16,7 +16,7 @@ # limitations under the License. # -operator_yaml_path=deploy/installer/generated/ibm-block-csi-operator.yaml +source hack/get_information_helper.sh roles_yaml_path=config/rbac/role.yaml origin_crd_yaml_path=config/crd/bases/csi.ibm.com_ibmblockcsis.yaml @@ -33,24 +33,23 @@ check_generation (){ cd $project_dirname } -verify_full_operator_yaml_is_align(){ +verify_full_operator_yaml_is_aligned(){ echo "check full operator yaml alignment" - declare -A orignial_yamls=( + declare -A yaml_kinds_by_origin_yaml_path=( ["config/rbac/role.yaml"]="ClusterRole" ["config/crd/bases/csi.ibm.com_ibmblockcsis.yaml"]="CustomResourceDefinition" ["config/rbac/service_account.yaml"]="ServiceAccount" ["config/rbac/role_binding.yaml"]="ClusterRoleBinding" ["config/manager/manager.yaml"]="Deployment" ) - for orignial_yaml in ${!orignial_yamls[@]}; do - export resource_type=${orignial_yamls[${orignial_yaml}]} + for orignial_yaml in ${!yaml_kinds_by_origin_yaml_path[@]}; do + export resource_type=${yaml_kinds_by_origin_yaml_path[${orignial_yaml}]} diff <(yq e '... comments=""' $orignial_yaml) <(yq eval '(. | select(.kind == env(resource_type)))' $operator_yaml_path) done } verify_no_roles_diff (){ echo "check roles alignment" - source hack/get_information_helper.sh are_manifest_files_exsists_in_current_csi_version csv_files=$(get_csv_files) for csv_file in $csv_files; do @@ -60,7 +59,6 @@ verify_no_roles_diff (){ verify_no_crds_diff (){ echo "check crds alignment" - source hack/get_information_helper.sh are_manifest_files_exsists_in_current_csi_version crd_files=$(get_bundle_crds) for crd_file in $crd_files; do @@ -69,6 +67,6 @@ verify_no_crds_diff (){ } check_generation -verify_full_operator_yaml_is_align +verify_full_operator_yaml_is_aligned verify_no_roles_diff verify_no_crds_diff diff --git a/hack/get_information_helper.sh b/hack/get_information_helper.sh index c0fb75c6f..c6fada123 100755 --- a/hack/get_information_helper.sh +++ b/hack/get_information_helper.sh @@ -16,6 +16,8 @@ # limitations under the License. # +operator_yaml_path=deploy/installer/generated/ibm-block-csi-operator.yaml + get_current_csi_version (){ current_csi_version=$(cat version/version.go | grep -i driverversion | awk -F = '{print $2}') echo ${current_csi_version//\"} diff --git a/hack/update-installer.sh b/hack/update-installer.sh index f8971d2fc..ed37c3020 100755 --- a/hack/update-installer.sh +++ b/hack/update-installer.sh @@ -20,5 +20,5 @@ # folder into one file. yes | cp deploy/installer/kustomization.yaml . -kustomize build -o deploy/installer/generated/ibm-block-csi-operator.yaml +kustomize build -o $operator_yaml_path rm -f kustomization.yaml From 1dbf0673a9582fa150a058bebfbe38fb2f4c056f Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Wed, 1 Dec 2021 11:12:08 +0200 Subject: [PATCH 3/7] add source in update installer Signed-off-by: matancarmeli7 --- hack/update-installer.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/hack/update-installer.sh b/hack/update-installer.sh index ed37c3020..37605c776 100755 --- a/hack/update-installer.sh +++ b/hack/update-installer.sh @@ -19,6 +19,7 @@ # Combine all the operator related yamls in the deploy # folder into one file. +source hack/get_information_helper.sh yes | cp deploy/installer/kustomization.yaml . kustomize build -o $operator_yaml_path rm -f kustomization.yaml From 4842511d77dc79fd68f23a749c4305f2c29ae822 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Wed, 1 Dec 2021 14:31:14 +0200 Subject: [PATCH 4/7] change project_info.sh file name Signed-off-by: matancarmeli7 --- hack/check-files-alignment.sh | 2 +- hack/{get_information_helper.sh => project_info.sh} | 0 hack/update-installer.sh | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename hack/{get_information_helper.sh => project_info.sh} (100%) diff --git a/hack/check-files-alignment.sh b/hack/check-files-alignment.sh index c4a23ca29..32e1bb085 100755 --- a/hack/check-files-alignment.sh +++ b/hack/check-files-alignment.sh @@ -16,7 +16,7 @@ # limitations under the License. # -source hack/get_information_helper.sh +source hack/project_info.sh roles_yaml_path=config/rbac/role.yaml origin_crd_yaml_path=config/crd/bases/csi.ibm.com_ibmblockcsis.yaml diff --git a/hack/get_information_helper.sh b/hack/project_info.sh similarity index 100% rename from hack/get_information_helper.sh rename to hack/project_info.sh diff --git a/hack/update-installer.sh b/hack/update-installer.sh index 37605c776..4efe2f81c 100755 --- a/hack/update-installer.sh +++ b/hack/update-installer.sh @@ -19,7 +19,7 @@ # Combine all the operator related yamls in the deploy # folder into one file. -source hack/get_information_helper.sh +source hack/project_info.sh yes | cp deploy/installer/kustomization.yaml . kustomize build -o $operator_yaml_path rm -f kustomization.yaml From 9a306b84b5198c2518846c7bd024db57bb03a143 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Thu, 2 Dec 2021 09:49:34 +0200 Subject: [PATCH 5/7] add variables to array Signed-off-by: matancarmeli7 --- hack/check-files-alignment.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/check-files-alignment.sh b/hack/check-files-alignment.sh index 32e1bb085..2a14709c4 100755 --- a/hack/check-files-alignment.sh +++ b/hack/check-files-alignment.sh @@ -36,8 +36,8 @@ check_generation (){ verify_full_operator_yaml_is_aligned(){ echo "check full operator yaml alignment" declare -A yaml_kinds_by_origin_yaml_path=( - ["config/rbac/role.yaml"]="ClusterRole" - ["config/crd/bases/csi.ibm.com_ibmblockcsis.yaml"]="CustomResourceDefinition" + [$roles_yaml_path]="ClusterRole" + [$origin_crd_yaml_path]="CustomResourceDefinition" ["config/rbac/service_account.yaml"]="ServiceAccount" ["config/rbac/role_binding.yaml"]="ClusterRoleBinding" ["config/manager/manager.yaml"]="Deployment" From a511e92a9905e8c3b54d6ad41238fd5342536cd0 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Thu, 2 Dec 2021 13:50:35 +0200 Subject: [PATCH 6/7] change origin_crd_yaml_path variable Signed-off-by: matancarmeli7 --- hack/check-files-alignment.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hack/check-files-alignment.sh b/hack/check-files-alignment.sh index 2a14709c4..7dc362a5d 100755 --- a/hack/check-files-alignment.sh +++ b/hack/check-files-alignment.sh @@ -18,7 +18,7 @@ source hack/project_info.sh roles_yaml_path=config/rbac/role.yaml -origin_crd_yaml_path=config/crd/bases/csi.ibm.com_ibmblockcsis.yaml +crd_yaml_path=config/crd/bases/csi.ibm.com_ibmblockcsis.yaml check_generation (){ echo "check generation" @@ -37,7 +37,7 @@ verify_full_operator_yaml_is_aligned(){ echo "check full operator yaml alignment" declare -A yaml_kinds_by_origin_yaml_path=( [$roles_yaml_path]="ClusterRole" - [$origin_crd_yaml_path]="CustomResourceDefinition" + [$crd_yaml_path]="CustomResourceDefinition" ["config/rbac/service_account.yaml"]="ServiceAccount" ["config/rbac/role_binding.yaml"]="ClusterRoleBinding" ["config/manager/manager.yaml"]="Deployment" @@ -62,7 +62,7 @@ verify_no_crds_diff (){ are_manifest_files_exsists_in_current_csi_version crd_files=$(get_bundle_crds) for crd_file in $crd_files; do - diff $origin_crd_yaml_path $crd_file + diff $crd_yaml_path $crd_file done } From 5f6371694ab49490297168fe2b3765156072b9dc Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Wed, 22 Dec 2021 09:48:58 +0200 Subject: [PATCH 7/7] fix PR comments Signed-off-by: matancarmeli7 --- hack/check-files-alignment.sh | 18 ++++++++++++------ hack/project_info.sh | 13 +++++++------ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/hack/check-files-alignment.sh b/hack/check-files-alignment.sh index 7dc362a5d..fb9d88202 100755 --- a/hack/check-files-alignment.sh +++ b/hack/check-files-alignment.sh @@ -33,6 +33,15 @@ check_generation (){ cd $project_dirname } +verify_yaml_doc_in_multidoc_yaml(){ + yamls_dictionary=${1#*=} + eval "declare -A yaml_kinds_by_origin_yaml_path="${yamls_dictionary} + for orignial_yaml in ${!yaml_kinds_by_origin_yaml_path[@]}; do + export resource_type=${yaml_kinds_by_origin_yaml_path[${orignial_yaml}]} + diff <(yq e '... comments=""' $orignial_yaml) <(yq eval '(. | select(.kind == env(resource_type)))' $operator_yaml_path) + done +} + verify_full_operator_yaml_is_aligned(){ echo "check full operator yaml alignment" declare -A yaml_kinds_by_origin_yaml_path=( @@ -42,15 +51,12 @@ verify_full_operator_yaml_is_aligned(){ ["config/rbac/role_binding.yaml"]="ClusterRoleBinding" ["config/manager/manager.yaml"]="Deployment" ) - for orignial_yaml in ${!yaml_kinds_by_origin_yaml_path[@]}; do - export resource_type=${yaml_kinds_by_origin_yaml_path[${orignial_yaml}]} - diff <(yq e '... comments=""' $orignial_yaml) <(yq eval '(. | select(.kind == env(resource_type)))' $operator_yaml_path) - done + verify_yaml_doc_in_multidoc_yaml "$(declare -p yaml_kinds_by_origin_yaml_path)" } verify_no_roles_diff (){ echo "check roles alignment" - are_manifest_files_exsists_in_current_csi_version + are_manifest_files_exist_in_current_csi_version csv_files=$(get_csv_files) for csv_file in $csv_files; do diff <(yq e .rules $roles_yaml_path) <(yq e .spec.install.spec.clusterPermissions[0].rules $csv_file) @@ -59,7 +65,7 @@ verify_no_roles_diff (){ verify_no_crds_diff (){ echo "check crds alignment" - are_manifest_files_exsists_in_current_csi_version + are_manifest_files_exist_in_current_csi_version crd_files=$(get_bundle_crds) for crd_file in $crd_files; do diff $crd_yaml_path $crd_file diff --git a/hack/project_info.sh b/hack/project_info.sh index c6fada123..8eb70d672 100755 --- a/hack/project_info.sh +++ b/hack/project_info.sh @@ -23,19 +23,20 @@ get_current_csi_version (){ echo ${current_csi_version//\"} } -are_manifest_files_exsists_in_current_csi_version (){ - current_csi_version=$(get_current_csi_version) +current_csi_version=$(get_current_csi_version) + +are_manifest_files_exist_in_current_csi_version (){ if ! compgen -G "${PWD}/deploy/olm-catalog/*/$current_csi_version" > /dev/null; then exit 0 fi } get_csv_files (){ - current_csi_version=$(get_current_csi_version) - ls deploy/olm-catalog/*/$current_csi_version/manifests/ibm-block-csi-operator.v$current_csi_version.clusterserviceversion.yaml + ls deploy/olm-catalog/ibm-block-csi-operator-community/$current_csi_version/manifests/ibm-block-csi-operator.v$current_csi_version.clusterserviceversion.yaml + ls deploy/olm-catalog/ibm-block-csi-operator/$current_csi_version/manifests/ibm-block-csi-operator.clusterserviceversion.yaml } get_bundle_crds (){ - current_csi_version=$(get_current_csi_version) - ls deploy/olm-catalog/*/$current_csi_version/manifests/csi.ibm.com_ibmblockcsis.yaml + ls deploy/olm-catalog/ibm-block-csi-operator-community/$current_csi_version/manifests/csi.ibm.com_ibmblockcsis.yaml + ls deploy/olm-catalog/ibm-block-csi-operator/$current_csi_version/manifests/csi.ibm.com_ibmblockcsis.yaml }