From cde084aba4385a926c21e781fa27aded36a1448e Mon Sep 17 00:00:00 2001 From: es3n1n Date: Wed, 4 Mar 2026 00:16:05 +0900 Subject: [PATCH] feat: add per-port domain override --- dist/resources/install.yaml | 8 + kctf-operator/api/v1/challenge_types.go | 7 + .../config/crd/bases/kctf.dev_challenges.yaml | 8 + .../controllers/service/functions.go | 317 +++++++++++------- kctf-operator/controllers/service/service.go | 225 ++++++++----- kctf-operator/resources/external-dns.go | 2 +- 6 files changed, 368 insertions(+), 199 deletions(-) diff --git a/dist/resources/install.yaml b/dist/resources/install.yaml index 5fbe03ef..fb6de77c 100644 --- a/dist/resources/install.yaml +++ b/dist/resources/install.yaml @@ -121,6 +121,14 @@ spec: description: By default, one port is set with default values items: properties: + domain: + description: |- + Domain overrides the subdomain prefix used for DNS. + For HTTPS it replaces "{name}-web", for TCP it replaces "{name}". + When empty the default kCTF naming is used. + maxLength: 63 + pattern: ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ + type: string domains: description: Extra domains for managed certificates. Only used for type HTTPS. diff --git a/kctf-operator/api/v1/challenge_types.go b/kctf-operator/api/v1/challenge_types.go index 6a2fad28..a3890c4d 100644 --- a/kctf-operator/api/v1/challenge_types.go +++ b/kctf-operator/api/v1/challenge_types.go @@ -37,6 +37,13 @@ type PortSpec struct { // +kubebuilder:validation:Required Protocol corev1.Protocol `json:"protocol"` + // Domain overrides the subdomain prefix used for DNS. + // For HTTPS it replaces "{name}-web", for TCP it replaces "{name}". + // When empty the default kCTF naming is used. + // +kubebuilder:validation:MaxLength=63 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([a-z0-9-]*[a-z0-9])?$` + Domain string `json:"domain,omitempty"` + // Extra domains for managed certificates. Only used for type HTTPS. Domains []string `json:"domains,omitempty"` } diff --git a/kctf-operator/config/crd/bases/kctf.dev_challenges.yaml b/kctf-operator/config/crd/bases/kctf.dev_challenges.yaml index 405ea88f..e35df704 100644 --- a/kctf-operator/config/crd/bases/kctf.dev_challenges.yaml +++ b/kctf-operator/config/crd/bases/kctf.dev_challenges.yaml @@ -115,6 +115,14 @@ spec: description: By default, one port is set with default values items: properties: + domain: + description: |- + Domain overrides the subdomain prefix used for DNS. + For HTTPS it replaces "{name}-web", for TCP it replaces "{name}". + When empty the default kCTF naming is used. + maxLength: 63 + pattern: ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ + type: string domains: description: Extra domains for managed certificates. Only used for type HTTPS. diff --git a/kctf-operator/controllers/service/functions.go b/kctf-operator/controllers/service/functions.go index 52229f9c..60daeba0 100644 --- a/kctf-operator/controllers/service/functions.go +++ b/kctf-operator/controllers/service/functions.go @@ -14,6 +14,7 @@ import ( corev1 "k8s.io/api/core/v1" netv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" backendv1 "k8s.io/ingress-gce/pkg/apis/backendconfig/v1" @@ -21,11 +22,21 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) +func annotationEqual(a, b map[string]string, key string) bool { + return a[key] == b[key] +} + +// isServiceEqual compares two services for equality. +// Used for both NodePort (internal) and LoadBalancer services — the annotation +// check is a no-op for NodePort since neither side sets annExternalDNSHostname. func isServiceEqual(serviceFound *corev1.Service, serv *corev1.Service) bool { if !equalPorts(serviceFound.Spec.Ports, serv.Spec.Ports) { return false } - return reflect.DeepEqual(serviceFound.Spec.LoadBalancerSourceRanges, serv.Spec.LoadBalancerSourceRanges) + if !reflect.DeepEqual(serviceFound.Spec.LoadBalancerSourceRanges, serv.Spec.LoadBalancerSourceRanges) { + return false + } + return annotationEqual(serviceFound.Annotations, serv.Annotations, annExternalDNSHostname) } func isCertEqual(existingCert *gkenetv1.ManagedCertificate, newCert *gkenetv1.ManagedCertificate) bool { @@ -33,7 +44,10 @@ func isCertEqual(existingCert *gkenetv1.ManagedCertificate, newCert *gkenetv1.Ma } func isIngressEqual(ingressFound *netv1.Ingress, ingress *netv1.Ingress) bool { - return reflect.DeepEqual(ingressFound.Spec, ingress.Spec) + if !reflect.DeepEqual(ingressFound.Spec, ingress.Spec) { + return false + } + return annotationEqual(ingressFound.Annotations, ingress.Annotations, annManagedCertificates) } // Check if the arrays of ports are the same @@ -130,158 +144,223 @@ func updateBackendConfig(challenge *kctfv1.Challenge, client client.Client, sche return true, err } -func updateManagedCertificate(challenge *kctfv1.Challenge, client client.Client, scheme *runtime.Scheme, +func updateManagedCertificates(challenge *kctfv1.Challenge, c client.Client, scheme *runtime.Scheme, log logr.Logger, ctx context.Context) (bool, error) { - existingCert := &gkenetv1.ManagedCertificate{} - err := client.Get(ctx, types.NamespacedName{Name: challenge.Name, Namespace: challenge.Namespace}, existingCert) + desiredCerts := generateManagedCertificates(challenge) - if err != nil && !errors.IsNotFound(err) { - return false, err - } - certExists := err == nil - - port := findHTTPSPort(challenge) - if port == nil || port.Domains == nil { - if certExists { - err := client.Delete(ctx, existingCert) - return true, err - } - return false, nil - } + changed := false + desiredNames := make(map[string]bool) - newCert := generateManagedCertificate(challenge, port.Domains) + for _, newCert := range desiredCerts { + desiredNames[newCert.Name] = true - if certExists { - if isCertEqual(existingCert, newCert) { - return false, nil + existingCert := &gkenetv1.ManagedCertificate{} + err := c.Get(ctx, types.NamespacedName{Name: newCert.Name, Namespace: newCert.Namespace}, existingCert) + if err != nil && !errors.IsNotFound(err) { + return false, err } - existingCert.Spec.Domains = newCert.Spec.Domains - - err := client.Update(ctx, existingCert) - - return true, err + if err == nil { + if isCertEqual(existingCert, newCert) { + continue + } + existingCert.Spec.Domains = newCert.Spec.Domains + if err := c.Update(ctx, existingCert); err != nil { + log.Error(err, "Failed to update managed certificate", "name", newCert.Name) + return changed, err + } + log.Info("Updated managed certificate", "name", newCert.Name) + changed = true + } else { + controllerutil.SetControllerReference(challenge, newCert, scheme) + if err := c.Create(ctx, newCert); err != nil { + return changed, err + } + log.Info("Created managed certificate", "name", newCert.Name) + changed = true + } } - controllerutil.SetControllerReference(challenge, newCert, scheme) + // Clean up stale certs + existingCerts := &gkenetv1.ManagedCertificateList{} + if err := c.List(ctx, existingCerts, + client.InNamespace(challenge.Namespace), + client.MatchingLabels{"app": challenge.Name}); err != nil { + return changed, err + } - err = client.Create(ctx, newCert) + for i := range existingCerts.Items { + cert := &existingCerts.Items[i] + if desiredNames[cert.Name] { + continue + } + if !metav1.IsControlledBy(cert, challenge) { + continue + } + if err := c.Delete(ctx, cert); err != nil { + log.Error(err, "Failed to delete stale managed certificate", "name", cert.Name) + return changed, err + } + log.Info("Deleted stale managed certificate", "name", cert.Name) + changed = true + } - return true, err + return changed, nil } -func updateIngress(challenge *kctfv1.Challenge, client client.Client, scheme *runtime.Scheme, +func updateIngresses(challenge *kctfv1.Challenge, c client.Client, scheme *runtime.Scheme, log logr.Logger, ctx context.Context) (bool, error) { - existingIngress := &netv1.Ingress{} - err := client.Get(ctx, types.NamespacedName{Name: challenge.Name, Namespace: challenge.Namespace}, existingIngress) - - if err != nil && !errors.IsNotFound(err) { - return false, err - } - ingressExists := err == nil - port := findHTTPSPort(challenge) - // Only one https port is supported at the moment. - // To support more, we will need a field to specify the domain name per ingress. + domainName := utils.GetDomainName(challenge, c, log, ctx) - if port == nil { - if ingressExists { - err := client.Delete(ctx, existingIngress) - return true, err - } - return false, nil + var desiredIngresses []*netv1.Ingress + if challenge.Spec.Network.Public { + desiredIngresses = generateIngresses(domainName, challenge) } - domainName := utils.GetDomainName(challenge, client, log, ctx) - newIngress := generateIngress(domainName, challenge, port) + changed := false + desiredNames := make(map[string]bool) - if ingressExists { - if isIngressEqual(existingIngress, newIngress) { - return false, nil - } + for _, newIngress := range desiredIngresses { + desiredNames[newIngress.Name] = true - existingIngress.Spec.DefaultBackend = newIngress.Spec.DefaultBackend - existingIngress.ObjectMeta.Annotations = newIngress.ObjectMeta.Annotations - err := client.Update(ctx, existingIngress) + existingIngress := &netv1.Ingress{} + err := c.Get(ctx, types.NamespacedName{Name: newIngress.Name, Namespace: newIngress.Namespace}, existingIngress) + if err != nil && !errors.IsNotFound(err) { + return false, err + } - return true, err + if err == nil { + if isIngressEqual(existingIngress, newIngress) { + continue + } + existingIngress.Spec = newIngress.Spec + existingIngress.ObjectMeta.Annotations = newIngress.ObjectMeta.Annotations + if err := c.Update(ctx, existingIngress); err != nil { + log.Error(err, "Failed to update ingress", " Name: ", newIngress.Name) + return changed, err + } + log.Info("Updated ingress", " Name: ", newIngress.Name) + changed = true + } else { + controllerutil.SetControllerReference(challenge, newIngress, scheme) + if err := c.Create(ctx, newIngress); err != nil { + return changed, err + } + log.Info("Created ingress", " Name: ", newIngress.Name) + changed = true + } } - if newIngress.Spec.DefaultBackend == nil || challenge.Spec.Network.Public == false { - return false, nil + // Clean up stale ingresses + existingIngresses := &netv1.IngressList{} + if err := c.List(ctx, existingIngresses, + client.InNamespace(challenge.Namespace), + client.MatchingLabels{"app": challenge.Name}); err != nil { + return changed, err } - controllerutil.SetControllerReference(challenge, newIngress, scheme) - - err = client.Create(ctx, newIngress) + for i := range existingIngresses.Items { + ing := &existingIngresses.Items[i] + if desiredNames[ing.Name] { + continue + } + if !metav1.IsControlledBy(ing, challenge) { + continue + } + if err := c.Delete(ctx, ing); err != nil { + log.Error(err, "Failed to delete stale ingress", " Name: ", ing.Name) + return changed, err + } + log.Info("Deleted stale ingress", " Name: ", ing.Name) + changed = true + } - return true, err + return changed, nil } -func updateLoadBalancerService(challenge *kctfv1.Challenge, client client.Client, scheme *runtime.Scheme, +func updateLoadBalancerServices(challenge *kctfv1.Challenge, c client.Client, scheme *runtime.Scheme, log logr.Logger, ctx context.Context) (bool, error) { - // Service is created in challenge_controller and here we just ensure that everything is alright - // Creates the service if it doesn't exist - // Check existence of the service: - existingService := &corev1.Service{} - err := client.Get(ctx, types.NamespacedName{Name: challenge.Name + "-lb-service", - Namespace: challenge.Namespace}, existingService) - if err != nil && !errors.IsNotFound(err) { - return false, err + domainName := utils.GetDomainName(challenge, c, log, ctx) + + var desiredServices []*corev1.Service + if challenge.Spec.Network.Public { + desiredServices = generateLoadBalancerServices(domainName, challenge) } - serviceExists := err == nil - // Get the domainName - domainName := utils.GetDomainName(challenge, client, log, ctx) - newService := generateLoadBalancerService(domainName, challenge) + changed := false + desiredNames := make(map[string]bool) - if serviceExists { - if len(newService.Spec.Ports) == 0 || challenge.Spec.Network.Public == false { - err := client.Delete(ctx, existingService) - return true, err - } + // Create or update desired services + for _, newService := range desiredServices { + desiredNames[newService.Name] = true - if isServiceEqual(existingService, newService) { - return false, nil + existingService := &corev1.Service{} + err := c.Get(ctx, types.NamespacedName{Name: newService.Name, Namespace: newService.Namespace}, existingService) + if err != nil && !errors.IsNotFound(err) { + return false, err } - copyPorts(existingService, newService) - existingService.ObjectMeta.Annotations = newService.ObjectMeta.Annotations - copyLoadBalancerSourceRanges(existingService, newService) - - err := client.Update(ctx, existingService) - if err == nil { - log.Info("Updated load balancer service", " Name: ", newService.Name, " with namespace ", newService.Namespace) + if isServiceEqual(existingService, newService) { + continue + } + copyPorts(existingService, newService) + existingService.ObjectMeta.Annotations = newService.ObjectMeta.Annotations + copyLoadBalancerSourceRanges(existingService, newService) + if err := c.Update(ctx, existingService); err != nil { + log.Error(err, "Failed to update LB service", " Name: ", newService.Name) + return changed, err + } + log.Info("Updated LB service", " Name: ", newService.Name) + changed = true } else { - log.Error(err, "Failed to update load balancer service", " Name: ", newService.Name, " with namespace ", newService.Namespace) + controllerutil.SetControllerReference(challenge, newService, scheme) + if err := c.Create(ctx, newService); err != nil { + return changed, err + } + log.Info("Created LB service", " Name: ", newService.Name) + changed = true } - - return true, err } - if len(newService.Spec.Ports) == 0 || challenge.Spec.Network.Public == false { - return false, nil + // Clean up stale LB services + existingServices := &corev1.ServiceList{} + if err := c.List(ctx, existingServices, + client.InNamespace(challenge.Namespace), + client.MatchingLabels{"app": challenge.Name}); err != nil { + return changed, err } - controllerutil.SetControllerReference(challenge, newService, scheme) - - err = client.Create(ctx, newService) + for i := range existingServices.Items { + svc := &existingServices.Items[i] + if svc.Spec.Type != corev1.ServiceTypeLoadBalancer { + continue + } + if desiredNames[svc.Name] { + continue + } + if !metav1.IsControlledBy(svc, challenge) { + continue + } + if err := c.Delete(ctx, svc); err != nil { + log.Error(err, "Failed to delete stale LB service", " Name: ", svc.Name) + return changed, err + } + log.Info("Deleted stale LB service", " Name: ", svc.Name) + changed = true + } - return true, err + return changed, nil } func checkPortsValid(challenge *kctfv1.Challenge) error { - seenHTTPSPort := false ports := make(map[int32]int32) + httpsDomains := make(map[string]bool) + for _, port := range challenge.Spec.Network.Ports { - if port.Protocol == "HTTPS" { - if seenHTTPSPort { - return fmt.Errorf("only one https port supported") - } - } externalPort := port.Port targetPort := port.TargetPort.IntVal if externalPort == 0 { @@ -292,6 +371,16 @@ func checkPortsValid(challenge *kctfv1.Challenge) error { return fmt.Errorf("conflicting port mapping %v->%v and %v->%v", externalPort, existingPort, externalPort, targetPort) } ports[externalPort] = targetPort + + if port.Protocol == "HTTPS" { + if httpsDomains[port.Domain] { + if port.Domain == "" { + return fmt.Errorf("only one HTTPS port allowed without an explicit domain") + } + return fmt.Errorf("duplicate domain %q for HTTPS ports", port.Domain) + } + httpsDomains[port.Domain] = true + } } return nil } @@ -317,13 +406,13 @@ func Update(challenge *kctfv1.Challenge, client client.Client, scheme *runtime.S } changed = changed || internalServiceChanged - loadBalancerServiceChanged, err := updateLoadBalancerService(challenge, client, scheme, log, ctx) + loadBalancerServicesChanged, err := updateLoadBalancerServices(challenge, client, scheme, log, ctx) if err != nil { - log.Error(err, "Error updating load balancer service", " Name: ", + log.Error(err, "Error updating load balancer services", " Name: ", challenge.Name, " with namespace ", challenge.Namespace) return false, err } - changed = changed || loadBalancerServiceChanged + changed = changed || loadBalancerServicesChanged backendConfigChanged, err := updateBackendConfig(challenge, client, scheme, log, ctx) if err != nil { @@ -333,21 +422,21 @@ func Update(challenge *kctfv1.Challenge, client client.Client, scheme *runtime.S } changed = changed || backendConfigChanged - managedCertificateChanged, err := updateManagedCertificate(challenge, client, scheme, log, ctx) + managedCertsChanged, err := updateManagedCertificates(challenge, client, scheme, log, ctx) if err != nil { - log.Error(err, "Error updating ManagedCertificate", " Name: ", + log.Error(err, "Error updating managed certificates", " Name: ", challenge.Name, " with namespace ", challenge.Namespace) return false, err } - changed = changed || managedCertificateChanged + changed = changed || managedCertsChanged - ingressChanged, err := updateIngress(challenge, client, scheme, log, ctx) + ingressesChanged, err := updateIngresses(challenge, client, scheme, log, ctx) if err != nil { - log.Error(err, "Error updating ingress", " Name: ", + log.Error(err, "Error updating ingresses", " Name: ", challenge.Name, " with namespace ", challenge.Namespace) return false, err } - changed = changed || ingressChanged + changed = changed || ingressesChanged return changed, nil } diff --git a/kctf-operator/controllers/service/service.go b/kctf-operator/controllers/service/service.go index fca32fd7..b1d074d6 100644 --- a/kctf-operator/controllers/service/service.go +++ b/kctf-operator/controllers/service/service.go @@ -14,6 +14,18 @@ import ( backendv1 "k8s.io/ingress-gce/pkg/apis/backendconfig/v1" ) +const ( + annExternalDNSHostname = "external-dns.alpha.kubernetes.io/hostname" + annManagedCertificates = "networking.gke.io/managed-certificates" +) + +func portHost(challengeName string, port *kctfv1.PortSpec, domainName string, defaultSuffix string) string { + if port.Domain != "" { + return port.Domain + "." + domainName + } + return challengeName + defaultSuffix + "." + domainName +} + func generateNodePortService(challenge *kctfv1.Challenge) *corev1.Service { service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -32,11 +44,6 @@ func generateNodePortService(challenge *kctfv1.Challenge) *corev1.Service { portsSeen := make(map[int32]bool) for i, port := range challenge.Spec.Network.Ports { - if portsSeen[port.Port] { - continue - } - portsSeen[port.Port] = true - protocol := corev1.ProtocolTCP switch port.Protocol { case corev1.ProtocolSCTP, corev1.ProtocolTCP, corev1.ProtocolUDP: @@ -47,6 +54,10 @@ func generateNodePortService(challenge *kctfv1.Challenge) *corev1.Service { if servicePort == 0 { servicePort = port.TargetPort.IntVal } + if portsSeen[servicePort] { + continue + } + portsSeen[servicePort] = true portName := port.Name if portName == "" { @@ -80,117 +91,163 @@ func generateBackendConfig(challenge *kctfv1.Challenge) *backendv1.BackendConfig return config } -func findHTTPSPort(challenge *kctfv1.Challenge) *kctfv1.PortSpec { +func ingressName(challengeName string, domain string) string { + if domain == "" { + return challengeName + } + return challengeName + "-ingress-" + domain +} + +func certName(challengeName string, domain string) string { + if domain == "" { + return challengeName + } + return challengeName + "-cert-" + domain +} + +func generateIngresses(domainName string, challenge *kctfv1.Challenge) []*netv1.Ingress { + var ingresses []*netv1.Ingress + for _, port := range challenge.Spec.Network.Ports { - // non-HTTPS is handled by generateLoadBalancerService if port.Protocol != "HTTPS" { continue } - return &port - } - return nil -} -func generateManagedCertificate(challenge *kctfv1.Challenge, domains []string) *gkenetv1.ManagedCertificate { - cert := &gkenetv1.ManagedCertificate{ - ObjectMeta: metav1.ObjectMeta{ - Name: challenge.Name, - Namespace: challenge.Namespace, - Labels: map[string]string{"app": challenge.Name}, - }, - Spec: gkenetv1.ManagedCertificateSpec{ - Domains: domains, - }, - Status: gkenetv1.ManagedCertificateStatus{ - DomainStatus: []gkenetv1.DomainStatus{}, - }, + servicePort := port.Port + if servicePort == 0 { + servicePort = port.TargetPort.IntVal + } + + ingress := &netv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: ingressName(challenge.Name, port.Domain), + Namespace: challenge.Namespace, + Labels: map[string]string{"app": challenge.Name}, + Annotations: map[string]string{}, + }, + Spec: netv1.IngressSpec{ + TLS: []netv1.IngressTLS{{ + SecretName: "tls-cert", + }}, + Rules: []netv1.IngressRule{{ + Host: portHost(challenge.Name, &port, domainName, "-web"), + }}, + DefaultBackend: &netv1.IngressBackend{ + Service: &netv1.IngressServiceBackend{ + Name: challenge.Name, + Port: netv1.ServiceBackendPort{ + Number: int32(servicePort), + }, + }, + }, + }, + } + + if port.Domains != nil { + ingress.Annotations[annManagedCertificates] = certName(challenge.Name, port.Domain) + } + + ingresses = append(ingresses, ingress) } - return cert + + return ingresses } -func generateIngress(domainName string, challenge *kctfv1.Challenge, port *kctfv1.PortSpec) *netv1.Ingress { - // Ingress object - ingress := &netv1.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: challenge.Name, - Namespace: challenge.Namespace, - Labels: map[string]string{"app": challenge.Name}, - Annotations: map[string]string{}, - }, - Spec: netv1.IngressSpec{ - TLS: []netv1.IngressTLS{{ - SecretName: "tls-cert", - }}, - Rules: []netv1.IngressRule{{ - Host: challenge.Name + "-web." + domainName, - }}, - }, - } +func generateManagedCertificates(challenge *kctfv1.Challenge) []*gkenetv1.ManagedCertificate { + var certs []*gkenetv1.ManagedCertificate - servicePort := port.Port - if servicePort == 0 { - servicePort = port.TargetPort.IntVal - } + for _, port := range challenge.Spec.Network.Ports { + if port.Protocol != "HTTPS" || port.Domains == nil { + continue + } - ingress.Spec.DefaultBackend = &netv1.IngressBackend{ - Service: &netv1.IngressServiceBackend{ - Name: challenge.Name, - Port: netv1.ServiceBackendPort{ - Number: int32(servicePort), + certs = append(certs, &gkenetv1.ManagedCertificate{ + ObjectMeta: metav1.ObjectMeta{ + Name: certName(challenge.Name, port.Domain), + Namespace: challenge.Namespace, + Labels: map[string]string{"app": challenge.Name}, }, - }, + Spec: gkenetv1.ManagedCertificateSpec{ + Domains: port.Domains, + }, + Status: gkenetv1.ManagedCertificateStatus{ + DomainStatus: []gkenetv1.DomainStatus{}, + }, + }) } - if port.Domains != nil { - ingress.Annotations["networking.gke.io/managed-certificates"] = challenge.Name - } + return certs +} - return ingress +func lbServiceName(challengeName string, domain string) string { + if domain == "" { + return challengeName + "-lb-service" + } + return challengeName + "-lb-" + domain } -func generateLoadBalancerService(domainName string, challenge *kctfv1.Challenge) *corev1.Service { - // Service object - service := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: challenge.Name + "-lb-service", - Namespace: challenge.Namespace, - Labels: map[string]string{"app": challenge.Name}, - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{"app": challenge.Name}, - Type: "LoadBalancer", - LoadBalancerSourceRanges: strings.Split(os.Getenv("ALLOWED_IPS"), ","), - }, +func generateLoadBalancerServices(domainName string, challenge *kctfv1.Challenge) []*corev1.Service { + type lbGroup struct { + hostname string + ports []corev1.ServicePort } + groups := make(map[string]*lbGroup) + var groupOrder []string + for i, port := range challenge.Spec.Network.Ports { - // HTTPS is handled by generateIngress if port.Protocol == "HTTPS" { continue } + key := port.Domain + group, ok := groups[key] + if !ok { + group = &lbGroup{ + hostname: portHost(challenge.Name, &port, domainName, ""), + } + groups[key] = group + groupOrder = append(groupOrder, key) + } + servicePortNumber := port.Port if servicePortNumber == 0 { servicePortNumber = port.TargetPort.IntVal } - servicePort := corev1.ServicePort{ + portName := port.Name + if portName == "" { + portName = "port-" + strconv.Itoa(i) + } + + group.ports = append(group.ports, corev1.ServicePort{ Port: servicePortNumber, TargetPort: port.TargetPort, Protocol: port.Protocol, - } - - if port.Name != "" { - servicePort.Name = port.Name - } else { - servicePort.Name = "port-" + strconv.Itoa(i) - } - - service.Spec.Ports = append(service.Spec.Ports, servicePort) + Name: portName, + }) } - service.ObjectMeta.Annotations = - map[string]string{"external-dns.alpha.kubernetes.io/hostname": challenge.Name + "." + domainName} + var services []*corev1.Service + for _, key := range groupOrder { + group := groups[key] + services = append(services, &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: lbServiceName(challenge.Name, key), + Namespace: challenge.Namespace, + Labels: map[string]string{"app": challenge.Name}, + Annotations: map[string]string{ + annExternalDNSHostname: group.hostname, + }, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{"app": challenge.Name}, + Type: "LoadBalancer", + LoadBalancerSourceRanges: strings.Split(os.Getenv("ALLOWED_IPS"), ","), + Ports: group.ports, + }, + }) + } - return service + return services } diff --git a/kctf-operator/resources/external-dns.go b/kctf-operator/resources/external-dns.go index 230fc68e..a47f1834 100644 --- a/kctf-operator/resources/external-dns.go +++ b/kctf-operator/resources/external-dns.go @@ -96,7 +96,7 @@ func NewExternalDnsDeployment() client.Object { Key: "CLUSTER_NAME", }, }, - }, }, + }}, Args: []string{"--log-level=debug", "--source=service", "--source=ingress", "--provider=google", "--domain-filter=$(DOMAIN_NAME)", "--registry=txt", "--txt-owner-id=kctf-cloud-dns-$(CLUSTER_NAME)"},