From 34337ec7421a94dea19a77a77e1cfbbaa3f1db47 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Thu, 21 Apr 2022 21:26:25 -0400 Subject: [PATCH 01/14] Longer wait on rke --- roles/rke/tasks/common.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/rke/tasks/common.yaml b/roles/rke/tasks/common.yaml index abd3b61..b2c3540 100644 --- a/roles/rke/tasks/common.yaml +++ b/roles/rke/tasks/common.yaml @@ -101,5 +101,5 @@ wait_for: path: /etc/rancher/node/password state: present - timeout: 180 + timeout: 1800 From 310abd0e0a1d58cf8c6461680be3b7cc441494b4 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Tue, 21 Jun 2022 14:54:08 -0400 Subject: [PATCH 02/14] Try latest k8s --- roles/rke/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/rke/defaults/main.yml b/roles/rke/defaults/main.yml index 2cf72df..80bed84 100644 --- a/roles/rke/defaults/main.yml +++ b/roles/rke/defaults/main.yml @@ -15,8 +15,8 @@ random_pwd: "{{ lookup('password', '/dev/null length=12 chars=ascii_letters,digi # Used for generating client secrets random_client_secret: "{{ lookup('password', '/dev/null length=8 chars=ascii_letters,digits') | lower }}-{{ lookup('password', '/dev/null length=4 chars=ascii_letters,digits') | lower }}-{{ lookup('password', '/dev/null length=4 chars=ascii_letters,digits') | lower }}-{{ lookup('password', '/dev/null length=4 chars=ascii_letters,digits') | lower }}-{{ lookup('password', '/dev/null length=12 chars=ascii_letters,digits') | lower }}" -rke_version: "v1.19.9+rke2r1" -rke_container_image: "rancher/rke2-runtime:v1.19.9-rke2r1" +rke_version: "v1.24.1+rke2r2" +rke_container_image: "rancher/rke2-runtime:v1.24.1+rke2r2" rke_in_docker: "{{ lookup('env', 'RKE_IN_DOCKER') | default(false, true) }}" rke_registration_server: "{{ hostvars[groups['controllers'][0]]['ansible_fqdn'] | default(groups['controllers'][0]) }}" is_rke_registration_server: "{{ rke_registration_server == ansible_fqdn }}" From 7ab3579d3aa05568c96fc80db66987b05dca1e86 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Tue, 21 Jun 2022 15:09:16 -0400 Subject: [PATCH 03/14] false intree provider --- roles/rke/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/rke/defaults/main.yml b/roles/rke/defaults/main.yml index 80bed84..7bfe3ce 100644 --- a/roles/rke/defaults/main.yml +++ b/roles/rke/defaults/main.yml @@ -4,7 +4,7 @@ kube_cloud_provider: "{{ lookup('env', 'KUBE_CLOUD_PROVIDER') }}" kube_cloud_conf: "{{ lookup('env', 'KUBE_CLOUD_CONF') | b64decode }}" # Whether to use the in-tree cloud provider or external cloud-provider -kube_in_tree_provider: true +kube_in_tree_provider: false # cluster settings cluster_hostname: "{{ lookup('env', 'CLUSTER_HOSTNAME') | default(inventory_hostname, true) }}" From a4a83d7d69ca68875ab2fff6818f782d906f3f83 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Tue, 21 Jun 2022 15:17:37 -0400 Subject: [PATCH 04/14] hostname for js2 --- roles/rke/templates/rke2_config.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/rke/templates/rke2_config.j2 b/roles/rke/templates/rke2_config.j2 index e3ddb9a..c2fd6e7 100644 --- a/roles/rke/templates/rke2_config.j2 +++ b/roles/rke/templates/rke2_config.j2 @@ -1,4 +1,4 @@ -node-name: "{{ ansible_hostname if 'jetstream' in ansible_fqdn else ansible_fqdn }}" +node-name: "{{ ansible_hostname if 'jetstream' in ansible_fqdn or 'js2local' in ansible_fqdn else ansible_fqdn }}" token: {{ rke_registration_token }} {% if 'controllers' in group_names %} From 59de0f4c774d687b5810d558f1c759e006a0c91f Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Tue, 21 Jun 2022 15:27:18 -0400 Subject: [PATCH 05/14] Update links for openstack manifests --- roles/rke/tasks/registration.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/rke/tasks/registration.yaml b/roles/rke/tasks/registration.yaml index b8dcf47..ac9e353 100644 --- a/roles/rke/tasks/registration.yaml +++ b/roles/rke/tasks/registration.yaml @@ -26,8 +26,8 @@ - name: Install OpenStack Cloud Controller Manager shell: kubectl apply -f {{ item }} with_items: - - https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/cluster/addons/rbac/cloud-controller-manager-roles.yaml - - https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/cluster/addons/rbac/cloud-controller-manager-role-bindings.yaml + - https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/manifests/controller-manager/cloud-controller-manager-roles.yaml + - https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/manifests/controller-manager/cloud-controller-manager-role-bindings.yaml - https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/manifests/controller-manager/openstack-cloud-controller-manager-ds.yaml when: not kube_in_tree_provider and kube_cloud_provider == "openstack" From c36ccb119805b8585a5347050609716f160af5cc Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Tue, 21 Jun 2022 15:58:22 -0400 Subject: [PATCH 06/14] Update daemon patch xref https://github.com/kubernetes/cloud-provider-openstack/commit/babbb064dbc8fc45ab9f55243df198226a0c4dc4 --- roles/rke/tasks/registration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/rke/tasks/registration.yaml b/roles/rke/tasks/registration.yaml index ac9e353..ebd13a1 100644 --- a/roles/rke/tasks/registration.yaml +++ b/roles/rke/tasks/registration.yaml @@ -33,7 +33,7 @@ - name: Patch node-selector on OpenStack because of https://github.com/rancher/k3s/issues/1869 shell: | - kubectl patch daemonset -n kube-system openstack-cloud-controller-manager -p '{"spec": {"template": {"spec": {"nodeSelector": {"node-role.kubernetes.io/master": "true"}}}}}' + kubectl patch daemonset -n kube-system openstack-cloud-controller-manager -p '{"spec": {"template": {"spec": {"nodeSelector": {"node-role.kubernetes.io/control-plane": "true"}}}}}' ignore_errors: true when: not kube_in_tree_provider and kube_cloud_provider == "openstack" From 37c68aea12f98968d8ead36038a007fe01b24913 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Tue, 21 Jun 2022 16:16:08 -0400 Subject: [PATCH 07/14] Revert "Try latest k8s" This reverts commit 310abd0e0a1d58cf8c6461680be3b7cc441494b4. --- roles/rke/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/rke/defaults/main.yml b/roles/rke/defaults/main.yml index 7bfe3ce..cbb2ad9 100644 --- a/roles/rke/defaults/main.yml +++ b/roles/rke/defaults/main.yml @@ -15,8 +15,8 @@ random_pwd: "{{ lookup('password', '/dev/null length=12 chars=ascii_letters,digi # Used for generating client secrets random_client_secret: "{{ lookup('password', '/dev/null length=8 chars=ascii_letters,digits') | lower }}-{{ lookup('password', '/dev/null length=4 chars=ascii_letters,digits') | lower }}-{{ lookup('password', '/dev/null length=4 chars=ascii_letters,digits') | lower }}-{{ lookup('password', '/dev/null length=4 chars=ascii_letters,digits') | lower }}-{{ lookup('password', '/dev/null length=12 chars=ascii_letters,digits') | lower }}" -rke_version: "v1.24.1+rke2r2" -rke_container_image: "rancher/rke2-runtime:v1.24.1+rke2r2" +rke_version: "v1.19.9+rke2r1" +rke_container_image: "rancher/rke2-runtime:v1.19.9-rke2r1" rke_in_docker: "{{ lookup('env', 'RKE_IN_DOCKER') | default(false, true) }}" rke_registration_server: "{{ hostvars[groups['controllers'][0]]['ansible_fqdn'] | default(groups['controllers'][0]) }}" is_rke_registration_server: "{{ rke_registration_server == ansible_fqdn }}" From a3e6f4e3d517cac9956836a0a1577037073ac09d Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Tue, 21 Jun 2022 16:27:04 -0400 Subject: [PATCH 08/14] Revert "Revert "Try latest k8s"" This reverts commit 37c68aea12f98968d8ead36038a007fe01b24913. --- roles/rke/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/rke/defaults/main.yml b/roles/rke/defaults/main.yml index cbb2ad9..7bfe3ce 100644 --- a/roles/rke/defaults/main.yml +++ b/roles/rke/defaults/main.yml @@ -15,8 +15,8 @@ random_pwd: "{{ lookup('password', '/dev/null length=12 chars=ascii_letters,digi # Used for generating client secrets random_client_secret: "{{ lookup('password', '/dev/null length=8 chars=ascii_letters,digits') | lower }}-{{ lookup('password', '/dev/null length=4 chars=ascii_letters,digits') | lower }}-{{ lookup('password', '/dev/null length=4 chars=ascii_letters,digits') | lower }}-{{ lookup('password', '/dev/null length=4 chars=ascii_letters,digits') | lower }}-{{ lookup('password', '/dev/null length=12 chars=ascii_letters,digits') | lower }}" -rke_version: "v1.19.9+rke2r1" -rke_container_image: "rancher/rke2-runtime:v1.19.9-rke2r1" +rke_version: "v1.24.1+rke2r2" +rke_container_image: "rancher/rke2-runtime:v1.24.1+rke2r2" rke_in_docker: "{{ lookup('env', 'RKE_IN_DOCKER') | default(false, true) }}" rke_registration_server: "{{ hostvars[groups['controllers'][0]]['ansible_fqdn'] | default(groups['controllers'][0]) }}" is_rke_registration_server: "{{ rke_registration_server == ansible_fqdn }}" From 60233b8679990b481831cc2d6e97f7b554aeebd3 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Tue, 21 Jun 2022 16:27:38 -0400 Subject: [PATCH 09/14] Bump nginx --- roles/cloudman-boot/tasks/ingress.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/cloudman-boot/tasks/ingress.yaml b/roles/cloudman-boot/tasks/ingress.yaml index c3b3f53..974bc10 100644 --- a/roles/cloudman-boot/tasks/ingress.yaml +++ b/roles/cloudman-boot/tasks/ingress.yaml @@ -13,7 +13,7 @@ - name: Helm install nginx ingress controller command: > /usr/local/bin/helm upgrade --install ingress-nginx ingress-nginx/ingress-nginx - --version 3.34.0 + --version 4.1.4 --namespace ingress-nginx --set controller.kind="DaemonSet" --set controller.hostNetwork=true From 392cf8ca3bccc706274adb3045f5fe1675b4d6c2 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Tue, 21 Jun 2022 16:28:21 -0400 Subject: [PATCH 10/14] Remove cvmfs --- roles/cloudman-boot/tasks/storage.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/roles/cloudman-boot/tasks/storage.yaml b/roles/cloudman-boot/tasks/storage.yaml index 0f4fba5..ee2df37 100644 --- a/roles/cloudman-boot/tasks/storage.yaml +++ b/roles/cloudman-boot/tasks/storage.yaml @@ -48,10 +48,10 @@ when: kube_cloud_provider == "openstack" ignore_errors: true -- name: Helm install galaxy-cvmfs-csi - command: > - /usr/local/bin/helm upgrade --install gxy-cvmfs galaxyproject/galaxy-cvmfs-csi - --version {{ cm_cvmfs_csi_version }} - {{ cm_cvmfs_csi_extra_params }} - --namespace csi-drivers - ignore_errors: true +# - name: Helm install galaxy-cvmfs-csi +# command: > +# /usr/local/bin/helm upgrade --install gxy-cvmfs galaxyproject/galaxy-cvmfs-csi +# --version {{ cm_cvmfs_csi_version }} +# {{ cm_cvmfs_csi_extra_params }} +# --namespace csi-drivers +# ignore_errors: true From dbc947ac527bcca936412b31df1f4e711578171f Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Tue, 21 Jun 2022 16:48:03 -0400 Subject: [PATCH 11/14] Revert "Remove cvmfs" This reverts commit 392cf8ca3bccc706274adb3045f5fe1675b4d6c2. Revert "Bump nginx" This reverts commit 60233b8679990b481831cc2d6e97f7b554aeebd3. Revert "Revert "Revert "Try latest k8s""" This reverts commit a3e6f4e3d517cac9956836a0a1577037073ac09d. Revert "Revert "Try latest k8s"" This reverts commit 37c68aea12f98968d8ead36038a007fe01b24913. Revert "Update daemon patch" This reverts commit c36ccb119805b8585a5347050609716f160af5cc. --- roles/cloudman-boot/tasks/ingress.yaml | 2 +- roles/cloudman-boot/tasks/storage.yaml | 14 +++++++------- roles/rke/tasks/registration.yaml | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/roles/cloudman-boot/tasks/ingress.yaml b/roles/cloudman-boot/tasks/ingress.yaml index 974bc10..c3b3f53 100644 --- a/roles/cloudman-boot/tasks/ingress.yaml +++ b/roles/cloudman-boot/tasks/ingress.yaml @@ -13,7 +13,7 @@ - name: Helm install nginx ingress controller command: > /usr/local/bin/helm upgrade --install ingress-nginx ingress-nginx/ingress-nginx - --version 4.1.4 + --version 3.34.0 --namespace ingress-nginx --set controller.kind="DaemonSet" --set controller.hostNetwork=true diff --git a/roles/cloudman-boot/tasks/storage.yaml b/roles/cloudman-boot/tasks/storage.yaml index ee2df37..0f4fba5 100644 --- a/roles/cloudman-boot/tasks/storage.yaml +++ b/roles/cloudman-boot/tasks/storage.yaml @@ -48,10 +48,10 @@ when: kube_cloud_provider == "openstack" ignore_errors: true -# - name: Helm install galaxy-cvmfs-csi -# command: > -# /usr/local/bin/helm upgrade --install gxy-cvmfs galaxyproject/galaxy-cvmfs-csi -# --version {{ cm_cvmfs_csi_version }} -# {{ cm_cvmfs_csi_extra_params }} -# --namespace csi-drivers -# ignore_errors: true +- name: Helm install galaxy-cvmfs-csi + command: > + /usr/local/bin/helm upgrade --install gxy-cvmfs galaxyproject/galaxy-cvmfs-csi + --version {{ cm_cvmfs_csi_version }} + {{ cm_cvmfs_csi_extra_params }} + --namespace csi-drivers + ignore_errors: true diff --git a/roles/rke/tasks/registration.yaml b/roles/rke/tasks/registration.yaml index ebd13a1..ac9e353 100644 --- a/roles/rke/tasks/registration.yaml +++ b/roles/rke/tasks/registration.yaml @@ -33,7 +33,7 @@ - name: Patch node-selector on OpenStack because of https://github.com/rancher/k3s/issues/1869 shell: | - kubectl patch daemonset -n kube-system openstack-cloud-controller-manager -p '{"spec": {"template": {"spec": {"nodeSelector": {"node-role.kubernetes.io/control-plane": "true"}}}}}' + kubectl patch daemonset -n kube-system openstack-cloud-controller-manager -p '{"spec": {"template": {"spec": {"nodeSelector": {"node-role.kubernetes.io/master": "true"}}}}}' ignore_errors: true when: not kube_in_tree_provider and kube_cloud_provider == "openstack" From 7d4c1d8b9abe4389c9632b08dba1a3bf8d9d6e7a Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Tue, 21 Jun 2022 16:51:56 -0400 Subject: [PATCH 12/14] Modified gist for opentack-manager daemonset --- roles/rke/tasks/registration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/rke/tasks/registration.yaml b/roles/rke/tasks/registration.yaml index ac9e353..6e09a3f 100644 --- a/roles/rke/tasks/registration.yaml +++ b/roles/rke/tasks/registration.yaml @@ -28,7 +28,7 @@ with_items: - https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/manifests/controller-manager/cloud-controller-manager-roles.yaml - https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/manifests/controller-manager/cloud-controller-manager-role-bindings.yaml - - https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/manifests/controller-manager/openstack-cloud-controller-manager-ds.yaml + - https://gist.githubusercontent.com/almahmoud/b02942eb2b18ea14123e163e069a79e2/raw/b59139b17d697c9075ce3870ad7e88c567851d8a/openstack-ds.yaml when: not kube_in_tree_provider and kube_cloud_provider == "openstack" - name: Patch node-selector on OpenStack because of https://github.com/rancher/k3s/issues/1869 From 67e6e1e267dbfa158d0c9869df31f8958d9ba539 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Tue, 21 Jun 2022 17:00:15 -0400 Subject: [PATCH 13/14] Revert "Revert "Revert "Try latest k8s""" This reverts commit a3e6f4e3d517cac9956836a0a1577037073ac09d. --- roles/rke/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/rke/defaults/main.yml b/roles/rke/defaults/main.yml index 7bfe3ce..cbb2ad9 100644 --- a/roles/rke/defaults/main.yml +++ b/roles/rke/defaults/main.yml @@ -15,8 +15,8 @@ random_pwd: "{{ lookup('password', '/dev/null length=12 chars=ascii_letters,digi # Used for generating client secrets random_client_secret: "{{ lookup('password', '/dev/null length=8 chars=ascii_letters,digits') | lower }}-{{ lookup('password', '/dev/null length=4 chars=ascii_letters,digits') | lower }}-{{ lookup('password', '/dev/null length=4 chars=ascii_letters,digits') | lower }}-{{ lookup('password', '/dev/null length=4 chars=ascii_letters,digits') | lower }}-{{ lookup('password', '/dev/null length=12 chars=ascii_letters,digits') | lower }}" -rke_version: "v1.24.1+rke2r2" -rke_container_image: "rancher/rke2-runtime:v1.24.1+rke2r2" +rke_version: "v1.19.9+rke2r1" +rke_container_image: "rancher/rke2-runtime:v1.19.9-rke2r1" rke_in_docker: "{{ lookup('env', 'RKE_IN_DOCKER') | default(false, true) }}" rke_registration_server: "{{ hostvars[groups['controllers'][0]]['ansible_fqdn'] | default(groups['controllers'][0]) }}" is_rke_registration_server: "{{ rke_registration_server == ansible_fqdn }}" From d3e67cb84c7c2ae2dbbcdee08d2b23ac91755ca6 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Tue, 21 Jun 2022 17:28:22 -0400 Subject: [PATCH 14/14] change volume az --- roles/rke/templates/ebs_storage_class.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/rke/templates/ebs_storage_class.j2 b/roles/rke/templates/ebs_storage_class.j2 index 1076585..dff8934 100644 --- a/roles/rke/templates/ebs_storage_class.j2 +++ b/roles/rke/templates/ebs_storage_class.j2 @@ -39,7 +39,7 @@ provisioner: kubernetes.io/cinder provisioner: cinder.csi.openstack.org {% endif %} parameters: - availability: melbourne-qh2 + availability: nova {% else %} provisioner: rancher.io/local-path {% endif %}