From ed622cd3a7a5a2bc4b135651f8dd96998fc0111d Mon Sep 17 00:00:00 2001 From: Swapnil Kulkarni Date: Tue, 12 Jul 2022 15:09:15 +0530 Subject: [PATCH 1/3] Updated scripts for Kubernets version 1.24.2 --- .env | 2 +- base.yaml | 43 +++++++++++++++++++++++++++++------------- containerd-config.yaml | 5 +++++ kubeadm-config.yaml | 2 +- 4 files changed, 37 insertions(+), 15 deletions(-) create mode 100644 containerd-config.yaml diff --git a/.env b/.env index 99300d3..89792e9 100644 --- a/.env +++ b/.env @@ -3,7 +3,7 @@ PROVIDER="virtualbox" NODE_MEMORY="2048" NODE_CPU="1" HOSTNAME="k8s-node" -KUBE_VERSION="1.21.1" +KUBE_VERSION="1.24.2" diff --git a/base.yaml b/base.yaml index d785509..ac20eba 100644 --- a/base.yaml +++ b/base.yaml @@ -7,6 +7,8 @@ service: name=docker state=started - name: kubelet status service: name=kubelet state=started + - name: containerd status + service: name=containerd state=started tasks: - name: Remove existing Docker Kubernetes packages apt: @@ -21,6 +23,10 @@ - kubectl - kubernetes-cni - docker-ce + - docker + - docker-engine + - containerd + - runc - name: Recursively remove directory ansible.builtin.file: @@ -65,6 +71,13 @@ state: present update_cache: true + - name: Recursively remove directory + ansible.builtin.file: + path: "{{ item }}" + state: directory + recurse: yes + with_items: + - /etc/apt/keyrings - name: Add an apt signing key for Docker apt_key: @@ -102,6 +115,7 @@ - apparmor-utils - docker-ce - docker-ce-cli + - containerd.io - "kubelet={{ kube_version }}-00" - "kubeadm={{ kube_version }}-00" - "kubectl={{ kube_version }}-00" @@ -143,18 +157,6 @@ name: docker enabled: yes - - name: Restart kubelet and docker daemon - service: - name: "{{ item }}" - daemon_reload: yes - state: restarted - with_items: - - docker - - kubelet - notify: - - docker status - - kubelet status - - name: Remove useless packages from the cache apt: autoclean: yes @@ -185,7 +187,22 @@ - { src: /vagrant/approve-nodes.sh, dest: /etc/k8s-init-scripts/approve-nodes.sh } - { src: /vagrant/kubeadm-config.yaml, dest: /etc/k8s-init-scripts/kubeadm-config.yaml } - { src: /vagrant/metallb-config.yaml, dest: /etc/k8s-init-scripts/metallb-config.yaml } - - { src: /vagrant/enable-olm.sh, dest: /etc/k8s-init-scripts/enable-olm.sh } + - { src: /vagrant/enable-olm.sh, dest: /etc/k8s-init-scripts/enable-olm.sh } + - { src: /vagrant/containerd-config.yaml, dest: /etc/containerd/config.toml } + - name: Restart kubelet and docker daemon + service: + name: "{{ item }}" + daemon_reload: yes + state: restarted + with_items: + - docker + - kubelet + - containerd + notify: + - docker status + - kubelet status + - containerd status + - name: Pull Kubernetes images shell: kubeadm config images pull \ No newline at end of file diff --git a/containerd-config.yaml b/containerd-config.yaml new file mode 100644 index 0000000..3f2c51f --- /dev/null +++ b/containerd-config.yaml @@ -0,0 +1,5 @@ + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + + [plugins."io.containerd.grpc.v1.cri"] + sandbox_image = "k8s.gcr.io/pause:3.2" \ No newline at end of file diff --git a/kubeadm-config.yaml b/kubeadm-config.yaml index 56350a7..d3b1267 100644 --- a/kubeadm-config.yaml +++ b/kubeadm-config.yaml @@ -34,7 +34,7 @@ networking: serviceSubnet: "10.96.0.0/12" podSubnet: "PODSUBNET" dnsDomain: "cluster.local" -kubernetesVersion: "v1.21.1" +kubernetesVersion: "v1.24.2" apiServer: extraArgs: authorization-mode: "Node,RBAC" From a54f108f31b736bfc4e2c69a3e22511d135017dd Mon Sep 17 00:00:00 2001 From: Swapnil Kulkarni Date: Tue, 12 Jul 2022 15:17:58 +0530 Subject: [PATCH 2/3] Created new directory structure --- .gitignore | 1 + Vagrantfile | 4 ++-- base.yaml => base/base.yaml | 22 ++++++++----------- .../containerd-config.yaml | 0 daemon.json => base/daemon.json | 0 install-ansible.sh => base/install-ansible.sh | 2 +- set-custom-rc.sh => base/set-custom-rc.sh | 0 approve-nodes.sh => k8s/approve-nodes.sh | 0 .../auto-approve-csr.yaml | 0 enable-olm.sh => k8s/enable-olm.sh | 0 .../kubeadm-config.yaml | 0 master.yaml => k8s/master.yaml | 0 .../metallb-config.yaml | 0 worker.yaml => k8s/worker.yaml | 0 14 files changed, 13 insertions(+), 16 deletions(-) rename base.yaml => base/base.yaml (85%) rename containerd-config.yaml => base/containerd-config.yaml (100%) rename daemon.json => base/daemon.json (100%) rename install-ansible.sh => base/install-ansible.sh (72%) rename set-custom-rc.sh => base/set-custom-rc.sh (100%) rename approve-nodes.sh => k8s/approve-nodes.sh (100%) rename auto-approve-csr.yaml => k8s/auto-approve-csr.yaml (100%) rename enable-olm.sh => k8s/enable-olm.sh (100%) rename kubeadm-config.yaml => k8s/kubeadm-config.yaml (100%) rename master.yaml => k8s/master.yaml (100%) rename metallb-config.yaml => k8s/metallb-config.yaml (100%) rename worker.yaml => k8s/worker.yaml (100%) diff --git a/.gitignore b/.gitignore index c9a7393..40b8a36 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .vagrant .ssh +*.box* diff --git a/Vagrantfile b/Vagrantfile index 9ffa8a4..c2057d6 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -12,12 +12,12 @@ Vagrant.configure("2") do |config| config.hostmanager.enabled = true config.hostmanager.manage_guest = true config.vm.synced_folder ".", "/vagrant" - config.vm.provision :shell, :path => "install-ansible.sh" + config.vm.provision :shell, :path => "base/install-ansible.sh" config.vm.define ENV["HOSTNAME"] do |master| master.vm.hostname = ENV["HOSTNAME"] master.vm.provision "ansible_local" do |ansible| - ansible.playbook = "base.yaml" + ansible.playbook = "base/base.yaml" ansible.extra_vars = { node_ip: "127.0.0.1", kube_version: ENV["KUBE_VERSION"] diff --git a/base.yaml b/base/base.yaml similarity index 85% rename from base.yaml rename to base/base.yaml index ac20eba..cf6c02c 100644 --- a/base.yaml +++ b/base/base.yaml @@ -147,11 +147,6 @@ command: swapoff -a when: ansible_swaptotal_mb > 0 - - name: Deploy Docker daemon.json. - copy: - src: daemon.json - dest: /etc/docker/daemon.json - - name: Enable service docker ansible.builtin.systemd: name: docker @@ -181,14 +176,15 @@ dest: "{{ item.dest }}" mode: '0755' with_items: - - { src: /vagrant/master.yaml, dest: /etc/k8s-scripts/master.yaml } - - { src: /vagrant/worker.yaml, dest: /etc/k8s-scripts/worker.yaml } - - { src: /vagrant/set-custom-rc.sh, dest: /etc/k8s-init-scripts/set-custom-rc.sh } - - { src: /vagrant/approve-nodes.sh, dest: /etc/k8s-init-scripts/approve-nodes.sh } - - { src: /vagrant/kubeadm-config.yaml, dest: /etc/k8s-init-scripts/kubeadm-config.yaml } - - { src: /vagrant/metallb-config.yaml, dest: /etc/k8s-init-scripts/metallb-config.yaml } - - { src: /vagrant/enable-olm.sh, dest: /etc/k8s-init-scripts/enable-olm.sh } - - { src: /vagrant/containerd-config.yaml, dest: /etc/containerd/config.toml } + - { src: /vagrant/base/daemon.json, dest: /etc/docker/daemon.json } + - { src: /vagrant/k8s/master.yaml, dest: /etc/k8s-scripts/master.yaml } + - { src: /vagrant/k8s/worker.yaml, dest: /etc/k8s-scripts/worker.yaml } + - { src: /vagrant/base/set-custom-rc.sh, dest: /etc/k8s-init-scripts/set-custom-rc.sh } + - { src: /vagrant/k8s/approve-nodes.sh, dest: /etc/k8s-init-scripts/approve-nodes.sh } + - { src: /vagrant/k8s/kubeadm-config.yaml, dest: /etc/k8s-init-scripts/kubeadm-config.yaml } + - { src: /vagrant/k8s/metallb-config.yaml, dest: /etc/k8s-init-scripts/metallb-config.yaml } + - { src: /vagrant/k8s/enable-olm.sh, dest: /etc/k8s-init-scripts/enable-olm.sh } + - { src: /vagrant/base/containerd-config.yaml, dest: /etc/containerd/config.toml } - name: Restart kubelet and docker daemon service: diff --git a/containerd-config.yaml b/base/containerd-config.yaml similarity index 100% rename from containerd-config.yaml rename to base/containerd-config.yaml diff --git a/daemon.json b/base/daemon.json similarity index 100% rename from daemon.json rename to base/daemon.json diff --git a/install-ansible.sh b/base/install-ansible.sh similarity index 72% rename from install-ansible.sh rename to base/install-ansible.sh index 4a49143..d6474fc 100644 --- a/install-ansible.sh +++ b/base/install-ansible.sh @@ -4,4 +4,4 @@ apt-add-repository ppa:ansible/ansible apt update apt install ansible -y -sh /vagrant/set-custom-rc.sh \ No newline at end of file +sh /vagrant/base/set-custom-rc.sh \ No newline at end of file diff --git a/set-custom-rc.sh b/base/set-custom-rc.sh similarity index 100% rename from set-custom-rc.sh rename to base/set-custom-rc.sh diff --git a/approve-nodes.sh b/k8s/approve-nodes.sh similarity index 100% rename from approve-nodes.sh rename to k8s/approve-nodes.sh diff --git a/auto-approve-csr.yaml b/k8s/auto-approve-csr.yaml similarity index 100% rename from auto-approve-csr.yaml rename to k8s/auto-approve-csr.yaml diff --git a/enable-olm.sh b/k8s/enable-olm.sh similarity index 100% rename from enable-olm.sh rename to k8s/enable-olm.sh diff --git a/kubeadm-config.yaml b/k8s/kubeadm-config.yaml similarity index 100% rename from kubeadm-config.yaml rename to k8s/kubeadm-config.yaml diff --git a/master.yaml b/k8s/master.yaml similarity index 100% rename from master.yaml rename to k8s/master.yaml diff --git a/metallb-config.yaml b/k8s/metallb-config.yaml similarity index 100% rename from metallb-config.yaml rename to k8s/metallb-config.yaml diff --git a/worker.yaml b/k8s/worker.yaml similarity index 100% rename from worker.yaml rename to k8s/worker.yaml From cf41ff0d0d540095e46c147da11f2806cc7f05a8 Mon Sep 17 00:00:00 2001 From: Swapnil Kulkarni Date: Tue, 20 Dec 2022 11:58:34 +0530 Subject: [PATCH 3/3] Initial changes for 1.25.5 --- .env | 4 +-- Vagrantfile | 2 +- base/base.yaml | 5 ++-- k8s/kubeadm-config.yaml | 59 ----------------------------------------- k8s/master.yaml | 50 ++++++++++++++++++++-------------- k8s/setup-metallb.sh | 5 ++++ k8s/worker.yaml | 26 ++++++++++++++++-- 7 files changed, 65 insertions(+), 86 deletions(-) delete mode 100644 k8s/kubeadm-config.yaml create mode 100644 k8s/setup-metallb.sh diff --git a/.env b/.env index 89792e9..bd7ab5f 100644 --- a/.env +++ b/.env @@ -1,9 +1,9 @@ -BOX_IMAGE="generic/ubuntu1804" +BOX_IMAGE="ubuntu/focal64" PROVIDER="virtualbox" NODE_MEMORY="2048" NODE_CPU="1" HOSTNAME="k8s-node" -KUBE_VERSION="1.24.2" +KUBE_VERSION="1.25.5" diff --git a/Vagrantfile b/Vagrantfile index c2057d6..2915569 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -3,7 +3,7 @@ Vagrant.configure("2") do |config| config.vm.box = ENV["BOX_IMAGE"] config.vm.box_version = ENV["BOX_VERSION"] config.vm.box_check_update = false - + config.ssh.forward_agent = true config.vm.provider ENV["PROVIDER"] do |l| l.cpus = ENV["NODE_CPU"] l.memory = ENV["NODE_MEMORY"] diff --git a/base/base.yaml b/base/base.yaml index cf6c02c..744cd58 100644 --- a/base/base.yaml +++ b/base/base.yaml @@ -119,7 +119,7 @@ - "kubelet={{ kube_version }}-00" - "kubeadm={{ kube_version }}-00" - "kubectl={{ kube_version }}-00" - - kubernetes-cni=0.8.7-00 + - kubernetes-cni notify: - docker status @@ -181,11 +181,12 @@ - { src: /vagrant/k8s/worker.yaml, dest: /etc/k8s-scripts/worker.yaml } - { src: /vagrant/base/set-custom-rc.sh, dest: /etc/k8s-init-scripts/set-custom-rc.sh } - { src: /vagrant/k8s/approve-nodes.sh, dest: /etc/k8s-init-scripts/approve-nodes.sh } - - { src: /vagrant/k8s/kubeadm-config.yaml, dest: /etc/k8s-init-scripts/kubeadm-config.yaml } - { src: /vagrant/k8s/metallb-config.yaml, dest: /etc/k8s-init-scripts/metallb-config.yaml } - { src: /vagrant/k8s/enable-olm.sh, dest: /etc/k8s-init-scripts/enable-olm.sh } + - { src: /vagrant/k8s/setup-metallb.sh, dest: /etc/k8s-init-scripts/setup-metallb.sh } - { src: /vagrant/base/containerd-config.yaml, dest: /etc/containerd/config.toml } + - name: Restart kubelet and docker daemon service: name: "{{ item }}" diff --git a/k8s/kubeadm-config.yaml b/k8s/kubeadm-config.yaml deleted file mode 100644 index d3b1267..0000000 --- a/k8s/kubeadm-config.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: kubeadm.k8s.io/v1beta2 -kind: InitConfiguration -bootstrapTokens: -- token: "9a08jv.c0izixklcxtmnze7" - description: "kubeadm bootstrap token" - ttl: "24h" -- token: "783bde.3f89s0fje9f38fhf" - description: "another bootstrap token" - usages: - - authentication - - signing - groups: - - system:bootstrappers:kubeadm:default-node-token -nodeRegistration: - name: "HOSTNAME" - criSocket: "/var/run/dockershim.sock" -localAPIEndpoint: - advertiseAddress: "IPADDRESS" - bindPort: 6443 ---- -apiVersion: kubeadm.k8s.io/v1beta2 -kind: ClusterConfiguration -etcd: - # one of local or external - local: - dataDir: "/var/lib/etcd" - extraArgs: - serverCertSANs: - - "IPADDRESS" - - "HOSTNAME" - peerCertSANs: - - "IPADDRESS" -networking: - serviceSubnet: "10.96.0.0/12" - podSubnet: "PODSUBNET" - dnsDomain: "cluster.local" -kubernetesVersion: "v1.24.2" -apiServer: - extraArgs: - authorization-mode: "Node,RBAC" - certSANs: - - "IPADDRESS" - - "HOSTNAME" - timeoutForControlPlane: 4m0s -controllerManager: {} -scheduler: {} -certificatesDir: "/etc/kubernetes/pki" -imageRepository: "k8s.gcr.io" -clusterName: "kubernetes" ---- -apiVersion: kubelet.config.k8s.io/v1beta1 -kind: KubeletConfiguration -serverTLSBootstrap: true ---- -apiVersion: kubeproxy.config.k8s.io/v1alpha1 -kind: KubeProxyConfiguration -mode: "ipvs" -ipvs: - strictARP: true \ No newline at end of file diff --git a/k8s/master.yaml b/k8s/master.yaml index ae6ecf4..9d692f1 100644 --- a/k8s/master.yaml +++ b/k8s/master.yaml @@ -2,12 +2,18 @@ - hosts: all become: true gather_facts: yes + handlers: + - name: docker status + service: name=docker state=started + - name: kubelet status + service: name=kubelet state=started + - name: containerd status + service: name=containerd state=started tasks: - name: Check war files in release local_action: stat path="/vagrant/.ssh/id_rsa.pub" register: key_file - - name: Set authorized key taken from file authorized_key: user: vagrant @@ -46,22 +52,26 @@ flush: yes - name: Configure default IP route for service network - command: "ip route add 10.96.0.0/16 dev eth1 src {{ ansible_eth1.ipv4.address }}" - - - name: Create custom configuration - replace: - path: /etc/k8s-init-scripts/kubeadm-config.yaml - regexp: "{{ item.regexp }}" - replace: "{{ item.replace }}" + command: "ip route add 10.96.0.0/16 dev enp0s8 src {{ ansible_enp0s8.ipv4.address }}" + + - name: Restart kubelet and docker daemon + service: + name: "{{ item }}" + daemon_reload: yes + state: restarted with_items: - - { regexp: "IPADDRESS", replace: "{{ ansible_eth1.ipv4.address }}" } - - { regexp: "HOSTNAME", replace: "{{ ansible_hostname }}" } - - { regexp: "PODSUBNET", replace: "{{ pod_network_cidr }}" } + - docker + - kubelet + - containerd + notify: + - docker status + - kubelet status + - containerd status - name: Initialize the Kubernetes cluster using kubeadm - command: kubeadm init --config /etc/k8s-init-scripts/kubeadm-config.yaml + command: "kubeadm init --control-plane-endpoint {{ ansible_enp0s8.ipv4.address }} --apiserver-advertise-address={{ ansible_enp0s8.ipv4.address }} --pod-network-cidr={{ pod_network_cidr }}" - - name: Create new directory for Kubenetes config + - name: Create new directory for Kubernetes config ansible.builtin.file: path: "{{ item }}" state: directory @@ -93,7 +103,7 @@ group: vagrant - name: Untaint the master nodes - command: kubectl taint nodes --all node-role.kubernetes.io/master- + command: kubectl taint nodes --all node-role.kubernetes.io/control-plane- - name: Install calico pod network become: false @@ -130,18 +140,17 @@ replace: "{{ item.replace }}" with_items: - { regexp: "LB_IPPOOL", replace: "{{ lb_ippool }}" } + when: enable_metallb == 'yes' - name: Enable MetalLB become: false - command: "{{ item}}" - with_items: - - "kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.10.2/manifests/namespace.yaml" - - "kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.10.2/manifests/metallb.yaml" - - "kubectl apply -f /etc/k8s-init-scripts/metallb-config.yaml" + command: "sh /etc/k8s-init-scripts/setup-metallb.sh" + when: enable_metallb == 'yes' - name: Enable Ingress become: false command: "kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v0.47.0/deploy/static/provider/cloud/deploy.yaml" + when: enable_ingress == 'yes' - name: Install Helm become: true @@ -172,4 +181,5 @@ ansible.builtin.cron: name: "approve_node_csrs" minute: "*/1" - job: "sh /etc/k8s-init-scripts/approve-nodes.sh" \ No newline at end of file + job: "sh /etc/k8s-init-scripts/approve-nodes.sh" + diff --git a/k8s/setup-metallb.sh b/k8s/setup-metallb.sh new file mode 100644 index 0000000..5928f39 --- /dev/null +++ b/k8s/setup-metallb.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +kubectl get configmap kube-proxy -n kube-system -o yaml | sed -e "s/strictARP: false/strictARP: true/" | kubectl apply -f - -n kube-system +kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.13.7/config/manifests/metallb-native.yam +kubectl apply -f /etc/k8s-init-scripts/metallb-config.yaml \ No newline at end of file diff --git a/k8s/worker.yaml b/k8s/worker.yaml index 836259f..7dbb102 100644 --- a/k8s/worker.yaml +++ b/k8s/worker.yaml @@ -2,6 +2,13 @@ - hosts: all become: true gather_facts: yes + handlers: + - name: docker status + service: name=docker state=started + - name: kubelet status + service: name=kubelet state=started + - name: containerd status + service: name=containerd state=started tasks: - name: Check war files in release local_action: stat path="/vagrant/.ssh/id_rsa.pub" @@ -41,7 +48,22 @@ flush: yes - name: Configure default IP route for service network - command: "ip route add 10.96.0.0/16 dev eth1 src {{ ansible_eth1.ipv4.address }}" + command: "ip route add 10.96.0.0/16 dev enp0s8 src {{ ansible_enp0s8.ipv4.address }}" + - name: Restart kubelet and docker daemon + service: + name: "{{ item }}" + daemon_reload: yes + state: restarted + with_items: + - docker + - kubelet + - containerd + notify: + - docker status + - kubelet status + - containerd status + - name: Run the join command - shell: $(cat /etc/.vagrantdata/kubeadm-join) \ No newline at end of file + shell: $(cat /etc/.vagrantdata/kubeadm-join) +