./hack/update-expected.sh

Signed-off-by: Ciprian Hacman <ciprian@hakman.dev>

Author: hakman

tests/integration/update_cluster/gossip-aws/data/aws_iam_role_policy_nodes.gossip.k8s.local_policy

@@ -1,27 +1,5 @@
 {
   "Statement": [
-    {
-      "Action": [
-        "s3:Get*"
-      ],
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws-test:s3:::placeholder-read-bucket/tests/gossip.k8s.local/cluster-completed.spec",
-        "arn:aws-test:s3:::placeholder-read-bucket/tests/gossip.k8s.local/igconfig/node/*"
-      ]
-    },
-    {
-      "Action": [
-        "s3:GetBucketLocation",
-        "s3:GetEncryptionConfiguration",
-        "s3:ListBucket",
-        "s3:ListBucketVersions"
-      ],
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws-test:s3:::placeholder-read-bucket"
-      ]
-    },
     {
       "Action": [
         "ec2:DescribeInstanceTypes",

tests/integration/update_cluster/gossip-aws/data/aws_launch_template_nodes.gossip.k8s.local_user_data

@@ -123,10 +123,33 @@ ensure-install-dir
 cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
 CloudProvider: aws
 ClusterName: gossip.k8s.local
-ConfigBase: memfs://tests/gossip.k8s.local
+ConfigServer:
+  CACertificates: |
+    -----BEGIN CERTIFICATE-----
+    MIIBbjCCARigAwIBAgIMFpANqBD8NSD82AUSMA0GCSqGSIb3DQEBCwUAMBgxFjAU
+    BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwODAwWhcNMzEwNzA3MDcw
+    ODAwWjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
+    SwAwSAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0yIg1
+    jt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
+    MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNG3zVjTcLlJwDsJ4/K9DV7KohUA
+    MA0GCSqGSIb3DQEBCwUAA0EAB8d03fY2w7WKpfO29qI295pu2C4ca9AiVGOpgSc8
+    tmQsq6rcxt3T+rb589PVtz0mw/cKTxOk6gH2CCC+yHfy2w==
+    -----END CERTIFICATE-----
+    -----BEGIN CERTIFICATE-----
+    MIIBbjCCARigAwIBAgIMFpANvmSa0OAlYmXKMA0GCSqGSIb3DQEBCwUAMBgxFjAU
+    BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwOTM2WhcNMzEwNzA3MDcw
+    OTM2WjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
+    SwAwSAJBAMF6F4aZdpe0RUpyykaBpWwZCnwbffhYGOw+fs6RdLuUq7QCNmJm/Eq7
+    WWOziMYDiI9SbclpD+6QiJ0N3EqppVUCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
+    MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLImp6ARjPDAH6nhI+scWVt3Q9bn
+    MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA
+    9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw==
+    -----END CERTIFICATE-----
+  servers:
+  - https://kops-controller.internal.gossip.k8s.local:3988/
 InstanceGroupName: nodes
 InstanceGroupRole: Node
-NodeupConfigHash: n/rKtAyAAO6tknrf2LGSDgoD/GNY2gJUDvx8CHp2i7E=
+NodeupConfigHash: 4pK9aRKK+peUykRrpjE5uI4ms+BHch2SFkbvBIRGRE4=
 
 __EOF_KUBE_ENV
 

tests/integration/update_cluster/gossip-aws/data/aws_s3_object_nodeupconfig-nodes_content

@@ -17,28 +17,7 @@ Assets:
   - d6dcab36d1b6af1b72c7f0662e5fcf446a291271ba6006532b95c4144e19d428@https://github.com/opencontainers/runc/releases/download/v1.3.4/runc.arm64
   - 25b57b0555fad42e5762246334681bf1c943794fcecdb680a79e482be5c08815@https://artifacts.k8s.io/binaries/kops/1.34.0-beta.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.34.0-beta.1/protokube-linux-arm64
   - 04470f8313796032fce85b974da4fc26420f36931e574fff6d117d21caf22770@https://artifacts.k8s.io/binaries/kops/1.34.0-beta.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.34.0-beta.1/channels-linux-arm64
-CAs:
-  kubernetes-ca: |
-    -----BEGIN CERTIFICATE-----
-    MIIBbjCCARigAwIBAgIMFpANqBD8NSD82AUSMA0GCSqGSIb3DQEBCwUAMBgxFjAU
-    BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwODAwWhcNMzEwNzA3MDcw
-    ODAwWjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
-    SwAwSAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0yIg1
-    jt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
-    MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNG3zVjTcLlJwDsJ4/K9DV7KohUA
-    MA0GCSqGSIb3DQEBCwUAA0EAB8d03fY2w7WKpfO29qI295pu2C4ca9AiVGOpgSc8
-    tmQsq6rcxt3T+rb589PVtz0mw/cKTxOk6gH2CCC+yHfy2w==
-    -----END CERTIFICATE-----
-    -----BEGIN CERTIFICATE-----
-    MIIBbjCCARigAwIBAgIMFpANvmSa0OAlYmXKMA0GCSqGSIb3DQEBCwUAMBgxFjAU
-    BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwOTM2WhcNMzEwNzA3MDcw
-    OTM2WjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
-    SwAwSAJBAMF6F4aZdpe0RUpyykaBpWwZCnwbffhYGOw+fs6RdLuUq7QCNmJm/Eq7
-    WWOziMYDiI9SbclpD+6QiJ0N3EqppVUCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
-    MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLImp6ARjPDAH6nhI+scWVt3Q9bn
-    MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA
-    9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw==
-    -----END CERTIFICATE-----
+CAs: {}
 ClusterName: gossip.k8s.local
 Hooks:
 - null
@@ -75,9 +54,6 @@ Networking:
 UpdatePolicy: automatic
 channels:
 - memfs://tests/gossip.k8s.local/addons/bootstrap-channel.yaml
-configStore:
-  keypairs: memfs://tests/gossip.k8s.local/pki
-  secrets: memfs://tests/gossip.k8s.local/secrets
 containerdConfig:
   logLevel: info
   runc:

tests/integration/update_cluster/gossip-aws/kubernetes.tf

@@ -157,7 +157,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-gossip-k8s-local" {
     propagate_at_launch = true
     value               = "owned"
   }
-  target_group_arns   = [aws_lb_target_group.tcp-gossip-k8s-local-k2q54l.id]
+  target_group_arns   = [aws_lb_target_group.kops-controller-gossip-k8-53agdi.id, aws_lb_target_group.tcp-gossip-k8s-local-k2q54l.id]
   vpc_zone_identifier = [aws_subnet.us-test-1a-gossip-k8s-local.id]
 }
 
@@ -579,7 +579,7 @@ resource "aws_launch_template" "nodes-gossip-k8s-local" {
 }
 
 resource "aws_lb" "api-gossip-k8s-local" {
-  enable_cross_zone_load_balancing = false
+  enable_cross_zone_load_balancing = true
   internal                         = false
   load_balancer_type               = "network"
   name                             = "api-gossip-k8s-local-t08tb7"
@@ -594,6 +594,16 @@ resource "aws_lb" "api-gossip-k8s-local" {
   }
 }
 
+resource "aws_lb_listener" "api-gossip-k8s-local-3988" {
+  default_action {
+    target_group_arn = aws_lb_target_group.kops-controller-gossip-k8-53agdi.id
+    type             = "forward"
+  }
+  load_balancer_arn = aws_lb.api-gossip-k8s-local.id
+  port              = 3988
+  protocol          = "TCP"
+}
+
 resource "aws_lb_listener" "api-gossip-k8s-local-443" {
   default_action {
     target_group_arn = aws_lb_target_group.tcp-gossip-k8s-local-k2q54l.id
@@ -604,6 +614,27 @@ resource "aws_lb_listener" "api-gossip-k8s-local-443" {
   protocol          = "TCP"
 }
 
+resource "aws_lb_target_group" "kops-controller-gossip-k8-53agdi" {
+  connection_termination = "true"
+  deregistration_delay   = "30"
+  health_check {
+    healthy_threshold   = 2
+    interval            = 10
+    path                = "/healthz"
+    protocol            = "HTTPS"
+    unhealthy_threshold = 2
+  }
+  name     = "kops-controller-gossip-k8-53agdi"
+  port     = 3988
+  protocol = "TCP"
+  tags = {
+    "KubernetesCluster"                      = "gossip.k8s.local"
+    "Name"                                   = "kops-controller-gossip-k8-53agdi"
+    "kubernetes.io/cluster/gossip.k8s.local" = "owned"
+  }
+  vpc_id = aws_vpc.gossip-k8s-local.id
+}
+
 resource "aws_lb_target_group" "tcp-gossip-k8s-local-k2q54l" {
   connection_termination = "true"
   deregistration_delay   = "30"
@@ -1052,6 +1083,24 @@ resource "aws_security_group_rule" "icmpv6-pmtu-api-elb-__--0" {
   type              = "ingress"
 }
 
+resource "aws_security_group_rule" "kops-controller-elb-to-cp" {
+  from_port                = 3988
+  protocol                 = "tcp"
+  security_group_id        = aws_security_group.masters-gossip-k8s-local.id
+  source_security_group_id = aws_security_group.api-elb-gossip-k8s-local.id
+  to_port                  = 3988
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "node-to-elb" {
+  from_port                = 0
+  protocol                 = "-1"
+  security_group_id        = aws_security_group.api-elb-gossip-k8s-local.id
+  source_security_group_id = aws_security_group.nodes-gossip-k8s-local.id
+  to_port                  = 0
+  type                     = "ingress"
+}
+
 resource "aws_sqs_queue" "gossip-k8s-local-nth" {
   message_retention_seconds = 300
   name                      = "gossip-k8s-local-nth"

tests/integration/update_cluster/gossip-azure/data/azurerm_linux_virtual_machine_scale_set_nodes-eastus-1.gossip.k8s.local_user_data

@@ -123,10 +123,33 @@ ensure-install-dir
 cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
 CloudProvider: azure
 ClusterName: gossip.k8s.local
-ConfigBase: memfs://tests/gossip.k8s.local
+ConfigServer:
+  CACertificates: |
+    -----BEGIN CERTIFICATE-----
+    MIIBbjCCARigAwIBAgIMFpANqBD8NSD82AUSMA0GCSqGSIb3DQEBCwUAMBgxFjAU
+    BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwODAwWhcNMzEwNzA3MDcw
+    ODAwWjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
+    SwAwSAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0yIg1
+    jt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
+    MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNG3zVjTcLlJwDsJ4/K9DV7KohUA
+    MA0GCSqGSIb3DQEBCwUAA0EAB8d03fY2w7WKpfO29qI295pu2C4ca9AiVGOpgSc8
+    tmQsq6rcxt3T+rb589PVtz0mw/cKTxOk6gH2CCC+yHfy2w==
+    -----END CERTIFICATE-----
+    -----BEGIN CERTIFICATE-----
+    MIIBbjCCARigAwIBAgIMFpANvmSa0OAlYmXKMA0GCSqGSIb3DQEBCwUAMBgxFjAU
+    BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwOTM2WhcNMzEwNzA3MDcw
+    OTM2WjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
+    SwAwSAJBAMF6F4aZdpe0RUpyykaBpWwZCnwbffhYGOw+fs6RdLuUq7QCNmJm/Eq7
+    WWOziMYDiI9SbclpD+6QiJ0N3EqppVUCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
+    MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLImp6ARjPDAH6nhI+scWVt3Q9bn
+    MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA
+    9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw==
+    -----END CERTIFICATE-----
+  servers:
+  - https://kops-controller.internal.gossip.k8s.local:3988/
 InstanceGroupName: nodes-eastus-1
 InstanceGroupRole: Node
-NodeupConfigHash: fcsX2QneMy3wdlG7nuf95wCwFlL9I+4hPQqV+ot9TMI=
+NodeupConfigHash: HcigG4YIjLOFKCVOEHaO/cJEdqe2qmaqoVkBKXR13Qg=
 
 __EOF_KUBE_ENV
 

tests/integration/update_cluster/gossip-azure/data/azurerm_storage_blob_nodeupconfig-nodes-eastus-1_source

@@ -22,28 +22,7 @@ AzureRouteTableName: gossip.k8s.local
 AzureSecurityGroupName: gossip.k8s.local
 AzureSubnetName: gossip.k8s.local
 AzureSubscriptionID: sub-123
-CAs:
-  kubernetes-ca: |
-    -----BEGIN CERTIFICATE-----
-    MIIBbjCCARigAwIBAgIMFpANqBD8NSD82AUSMA0GCSqGSIb3DQEBCwUAMBgxFjAU
-    BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwODAwWhcNMzEwNzA3MDcw
-    ODAwWjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
-    SwAwSAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0yIg1
-    jt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
-    MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNG3zVjTcLlJwDsJ4/K9DV7KohUA
-    MA0GCSqGSIb3DQEBCwUAA0EAB8d03fY2w7WKpfO29qI295pu2C4ca9AiVGOpgSc8
-    tmQsq6rcxt3T+rb589PVtz0mw/cKTxOk6gH2CCC+yHfy2w==
-    -----END CERTIFICATE-----
-    -----BEGIN CERTIFICATE-----
-    MIIBbjCCARigAwIBAgIMFpANvmSa0OAlYmXKMA0GCSqGSIb3DQEBCwUAMBgxFjAU
-    BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwOTM2WhcNMzEwNzA3MDcw
-    OTM2WjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
-    SwAwSAJBAMF6F4aZdpe0RUpyykaBpWwZCnwbffhYGOw+fs6RdLuUq7QCNmJm/Eq7
-    WWOziMYDiI9SbclpD+6QiJ0N3EqppVUCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
-    MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLImp6ARjPDAH6nhI+scWVt3Q9bn
-    MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA
-    9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw==
-    -----END CERTIFICATE-----
+CAs: {}
 ClusterName: gossip.k8s.local
 Hooks:
 - null
@@ -80,9 +59,6 @@ Networking:
 UpdatePolicy: automatic
 channels:
 - memfs://tests/gossip.k8s.local/addons/bootstrap-channel.yaml
-configStore:
-  keypairs: memfs://tests/gossip.k8s.local/pki
-  secrets: memfs://tests/gossip.k8s.local/secrets
 containerdConfig:
   logLevel: info
   runc:

tests/integration/update_cluster/gossip-azure/kubernetes.tf

@@ -377,6 +377,28 @@ resource "azurerm_network_security_group" "gossip-k8s-local" {
     source_application_security_group_ids      = [azurerm_application_security_group.nodes-gossip-k8s-local.id]
     source_port_range                          = "*"
   }
+  security_rule {
+    access                                     = "Allow"
+    destination_application_security_group_ids = [azurerm_application_security_group.control-plane-gossip-k8s-local.id]
+    destination_port_range                     = "443"
+    direction                                  = "Inbound"
+    name                                       = "AllowNodesToKubernetesAPI"
+    priority                                   = 2000
+    protocol                                   = "Tcp"
+    source_address_prefix                      = "*"
+    source_port_range                          = "*"
+  }
+  security_rule {
+    access                                     = "Allow"
+    destination_application_security_group_ids = [azurerm_application_security_group.control-plane-gossip-k8s-local.id]
+    destination_port_range                     = "3988"
+    direction                                  = "Inbound"
+    name                                       = "AllowNodesToKopsController"
+    priority                                   = 2001
+    protocol                                   = "Tcp"
+    source_address_prefix                      = "*"
+    source_port_range                          = "*"
+  }
   security_rule {
     access                     = "Allow"
     destination_address_prefix = "VirtualNetwork"

tests/integration/update_cluster/gossip-gce/kubernetes.tf

@@ -229,6 +229,30 @@ resource "google_compute_firewall" "https-api-ipv6-gossip-k8s-local" {
   target_tags   = ["gossip-k8s-local-k8s-io-role-control-plane"]
 }
 
+resource "google_compute_firewall" "kops-controller-gossip-k8s-local" {
+  allow {
+    ports    = ["3988"]
+    protocol = "tcp"
+  }
+  disabled      = false
+  name          = "kops-controller-gossip-k8s-local"
+  network       = google_compute_network.gossip-k8s-local.name
+  source_ranges = ["0.0.0.0/0"]
+  target_tags   = ["gossip-k8s-local-k8s-io-role-control-plane"]
+}
+
+resource "google_compute_firewall" "kops-controller-ipv6-gossip-k8s-local" {
+  allow {
+    ports    = ["3988"]
+    protocol = "tcp"
+  }
+  disabled      = false
+  name          = "kops-controller-ipv6-gossip-k8s-local"
+  network       = google_compute_network.gossip-k8s-local.name
+  source_ranges = ["::/0"]
+  target_tags   = ["gossip-k8s-local-k8s-io-role-control-plane"]
+}
+
 resource "google_compute_firewall" "lb-health-checks-gossip-k8s-local" {
   allow {
     protocol = "tcp"
@@ -462,6 +486,21 @@ resource "google_compute_forwarding_rule" "api-us-test1-gossip-k8s-local" {
   subnetwork            = google_compute_subnetwork.us-test1-gossip-k8s-local.name
 }
 
+resource "google_compute_forwarding_rule" "kops-controller-us-test1-gossip-k8s-local" {
+  backend_service = google_compute_region_backend_service.api-gossip-k8s-local.id
+  ip_address      = google_compute_address.api-us-test1-gossip-k8s-local.address
+  ip_protocol     = "TCP"
+  labels = {
+    "k8s-io-cluster-name" = "gossip-k8s-local"
+    "name"                = "kops-controller-us-test1"
+  }
+  load_balancing_scheme = "INTERNAL"
+  name                  = "kops-controller-us-test1-gossip-k8s-local"
+  network               = google_compute_network.gossip-k8s-local.name
+  ports                 = ["3988"]
+  subnetwork            = google_compute_subnetwork.us-test1-gossip-k8s-local.name
+}
+
 resource "google_compute_http_health_check" "api-gossip-k8s-local" {
   name         = "api-gossip-k8s-local"
   port         = 3990