diff --git a/internal/cmd/manager/manager.go b/internal/cmd/manager/manager.go index 11dd37b3..aee44905 100644 --- a/internal/cmd/manager/manager.go +++ b/internal/cmd/manager/manager.go @@ -84,521 +84,536 @@ type BuildInfo struct { BuildDate string } +type managerConfig struct { + build BuildInfo + serverConfigFile string + probeAddr string + enableLeaderElection bool + leaderElectionNamespace string + enableClusterSharding bool + clusterShardingLeaseNamespace string + clusterShardingLeasePrefix string + clusterShardingPeerWeight uint + singletonControllersLeaderElection bool + singletonControllersLeaderElectionID string + opts zap.Options +} + // NewCommand builds the "manager" subcommand, which runs the // network-services-operator controller manager. func NewCommand(build BuildInfo) *cobra.Command { - var enableLeaderElection bool - var leaderElectionNamespace string - var probeAddr string - var enableClusterSharding bool - var clusterShardingLeaseNamespace string - var clusterShardingLeasePrefix string - var clusterShardingPeerWeight uint - var singletonControllersLeaderElection bool - var singletonControllersLeaderElectionID string - - var serverConfigFile string + cfg := managerConfig{ + build: build, + opts: zap.Options{Development: true}, + } fs := flag.NewFlagSet("manager", flag.ContinueOnError) - fs.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") - fs.BoolVar(&enableLeaderElection, "leader-elect", false, + fs.StringVar(&cfg.probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + fs.BoolVar(&cfg.enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") - fs.StringVar(&leaderElectionNamespace, "leader-elect-namespace", "", "The namespace to use for leader election.") + fs.StringVar(&cfg.leaderElectionNamespace, "leader-elect-namespace", "", "The namespace to use for leader election.") fs.BoolVar( - &enableClusterSharding, + &cfg.enableClusterSharding, "cluster-sharding-enabled", false, "Enable multicluster controller sharding via per-cluster coordination leases.", ) fs.StringVar( - &clusterShardingLeaseNamespace, + &cfg.clusterShardingLeaseNamespace, "cluster-sharding-lease-namespace", "kube-system", "Namespace for controller cluster sharding leases.", ) fs.StringVar( - &clusterShardingLeasePrefix, + &cfg.clusterShardingLeasePrefix, "cluster-sharding-lease-prefix", "mcr-shard", "Lease name prefix for controller cluster sharding.", ) fs.UintVar( - &clusterShardingPeerWeight, + &cfg.clusterShardingPeerWeight, "cluster-sharding-peer-weight", 1, "Relative shard weight for this controller instance.", ) fs.BoolVar( - &singletonControllersLeaderElection, + &cfg.singletonControllersLeaderElection, "singleton-controllers-leader-elect", true, "Enable leader election for singleton downstream controllers (Challenge and GatewayDownstreamCertificateSolver).", ) fs.StringVar( - &singletonControllersLeaderElectionID, + &cfg.singletonControllersLeaderElectionID, "singleton-controllers-leader-election-id", "6a7d51cc.datumapis.com-singleton", "Leader election ID for singleton downstream controllers.", ) - opts := zap.Options{ - Development: true, - } - - fs.StringVar(&serverConfigFile, "server-config", "", "path to the server config file") + fs.StringVar(&cfg.serverConfigFile, "server-config", "", "path to the server config file") - opts.BindFlags(fs) + cfg.opts.BindFlags(fs) cmd := &cobra.Command{ Use: "manager", Short: "Run the network-services-operator controller manager", Args: cobra.NoArgs, - // nolint:gocyclo RunE: func(_ *cobra.Command, _ []string) error { - ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + return runManager(cfg) + }, + } - setupLog.Info("starting network-services-operator", - "version", build.Version, - "gitCommit", build.GitCommit, - "gitTreeState", build.GitTreeState, - "buildDate", build.BuildDate, - ) + cmd.Flags().AddGoFlagSet(fs) + return cmd +} + +//nolint:gocyclo +func runManager(mcfg managerConfig) error { + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&mcfg.opts))) + + setupLog.Info("starting network-services-operator", + "version", mcfg.build.Version, + "gitCommit", mcfg.build.GitCommit, + "gitTreeState", mcfg.build.GitTreeState, + "buildDate", mcfg.build.BuildDate, + ) + + var serverConfig config.NetworkServicesOperator + var configData []byte + if len(mcfg.serverConfigFile) > 0 { + var err error + configData, err = os.ReadFile(mcfg.serverConfigFile) + if err != nil { + setupLog.Error(fmt.Errorf("unable to read server config from %q", mcfg.serverConfigFile), "") + os.Exit(1) + } + } + + if err := runtime.DecodeInto(codecs.UniversalDecoder(), configData, &serverConfig); err != nil { + setupLog.Error(err, "unable to decode server config") + os.Exit(1) + } + + // Allow overriding Redis URL at runtime via env var. + if redisURL := strings.TrimSpace(os.Getenv("REDIS_URL")); redisURL != "" { + serverConfig.Redis.URL = redisURL + setupLog.Info("overriding redis.url from REDIS_URL") + } + + setupLog.Info("server config", "config", serverConfig) + + if err := serverConfig.Validate(); err != nil { + setupLog.Error(err, "invalid server config") + os.Exit(1) + } + + restCfg := ctrl.GetConfigOrDie() + serverConfig.ControlPlaneClient.ApplyTo(restCfg) + + deploymentCluster, err := cluster.New(restCfg, func(o *cluster.Options) { + o.Scheme = scheme + }) + if err != nil { + setupLog.Error(err, "failed creating local cluster") + os.Exit(1) + } + + runnables, provider, err := initializeClusterDiscovery(serverConfig, deploymentCluster, scheme) + if err != nil { + setupLog.Error(err, "unable to initialize cluster discovery") + os.Exit(1) + } + + setupLog.Info("cluster discovery mode", "mode", serverConfig.Discovery.Mode) + + ctx := ctrl.SetupSignalHandler() + + deploymentClusterClient := deploymentCluster.GetClient() + + metricsServerOptions := serverConfig.MetricsServer.Options(ctx, deploymentClusterClient) + + webhookServer := webhook.NewServer( + serverConfig.WebhookServer.Options(ctx, deploymentClusterClient), + ) - var serverConfig config.NetworkServicesOperator - var configData []byte - if len(serverConfigFile) > 0 { - var err error - configData, err = os.ReadFile(serverConfigFile) - if err != nil { - setupLog.Error(fmt.Errorf("unable to read server config from %q", serverConfigFile), "") - os.Exit(1) - } - } - - if err := runtime.DecodeInto(codecs.UniversalDecoder(), configData, &serverConfig); err != nil { - setupLog.Error(err, "unable to decode server config") - os.Exit(1) - } - - // Allow overriding Redis URL at runtime via env var. - if redisURL := strings.TrimSpace(os.Getenv("REDIS_URL")); redisURL != "" { - serverConfig.Redis.URL = redisURL - setupLog.Info("overriding redis.url from REDIS_URL") - } - - setupLog.Info("server config", "config", serverConfig) - - if err := serverConfig.Validate(); err != nil { - setupLog.Error(err, "invalid server config") - os.Exit(1) - } - - cfg := ctrl.GetConfigOrDie() - serverConfig.ControlPlaneClient.ApplyTo(cfg) - - deploymentCluster, err := cluster.New(cfg, func(o *cluster.Options) { - o.Scheme = scheme - }) - if err != nil { - setupLog.Error(err, "failed creating local cluster") - os.Exit(1) - } - - runnables, provider, err := initializeClusterDiscovery(serverConfig, deploymentCluster, scheme) - if err != nil { - setupLog.Error(err, "unable to initialize cluster discovery") - os.Exit(1) - } - - setupLog.Info("cluster discovery mode", "mode", serverConfig.Discovery.Mode) - - ctx := ctrl.SetupSignalHandler() - - deploymentClusterClient := deploymentCluster.GetClient() - - metricsServerOptions := serverConfig.MetricsServer.Options(ctx, deploymentClusterClient) - - webhookServer := webhook.NewServer( - serverConfig.WebhookServer.Options(ctx, deploymentClusterClient), + webhookServer = networkingwebhook.NewClusterAwareWebhookServer(webhookServer, serverConfig.Discovery.Mode) + + leaseDuration := serverConfig.LeaderElection.LeaseDuration.Duration + renewDeadline := serverConfig.LeaderElection.RenewDeadline.Duration + retryPeriod := serverConfig.LeaderElection.RetryPeriod.Duration + + mcManagerOptions := []mcmanager.Option{} + if mcfg.enableClusterSharding { + setupLog.Info( + "enabling cluster sharding coordinator", + "leaseNamespace", + mcfg.clusterShardingLeaseNamespace, + "leasePrefix", + mcfg.clusterShardingLeasePrefix, + "peerWeight", + mcfg.clusterShardingPeerWeight, + ) + + clusterShardingOptions := []sharded.Option{ + sharded.WithShardLease(mcfg.clusterShardingLeaseNamespace, mcfg.clusterShardingLeasePrefix), + sharded.WithPerClusterLease(true), + } + if mcfg.clusterShardingPeerWeight > 0 { + clusterShardingOptions = append( + clusterShardingOptions, + sharded.WithPeerWeight(uint32(mcfg.clusterShardingPeerWeight)), ) + } - webhookServer = networkingwebhook.NewClusterAwareWebhookServer(webhookServer, serverConfig.Discovery.Mode) - - leaseDuration := serverConfig.LeaderElection.LeaseDuration.Duration - renewDeadline := serverConfig.LeaderElection.RenewDeadline.Duration - retryPeriod := serverConfig.LeaderElection.RetryPeriod.Duration - - mcManagerOptions := []mcmanager.Option{} - if enableClusterSharding { - setupLog.Info( - "enabling cluster sharding coordinator", - "leaseNamespace", - clusterShardingLeaseNamespace, - "leasePrefix", - clusterShardingLeasePrefix, - "peerWeight", - clusterShardingPeerWeight, - ) - - clusterShardingOptions := []sharded.Option{ - sharded.WithShardLease(clusterShardingLeaseNamespace, clusterShardingLeasePrefix), - sharded.WithPerClusterLease(true), - } - if clusterShardingPeerWeight > 0 { - clusterShardingOptions = append( - clusterShardingOptions, - sharded.WithPeerWeight(uint32(clusterShardingPeerWeight)), - ) - } - - mcManagerOptions = append( - mcManagerOptions, - mcmanager.WithCoordinator( - sharded.New( - deploymentCluster.GetClient(), - ctrl.Log.WithName("cluster-sharding-coordinator"), - clusterShardingOptions..., - ), - ), - ) - } - - primaryManagerLeaderElection := enableLeaderElection - if enableClusterSharding && enableLeaderElection { - setupLog.Info( - "disabling primary manager leader election while cluster sharding is enabled", - "singletonControllersLeaderElection", - singletonControllersLeaderElection, - ) - primaryManagerLeaderElection = false - } - - mgr, err := mcmanager.New(cfg, provider, ctrl.Options{ - Scheme: scheme, - Metrics: metricsServerOptions, - WebhookServer: webhookServer, - HealthProbeBindAddress: probeAddr, - LeaderElection: primaryManagerLeaderElection, - LeaderElectionID: "6a7d51cc.datumapis.com", - LeaderElectionNamespace: leaderElectionNamespace, - LeaseDuration: &leaseDuration, - RenewDeadline: &renewDeadline, - RetryPeriod: &retryPeriod, - // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily - // when the Manager ends. This requires the binary to immediately end when the - // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly - // speeds up voluntary leader transitions as the new leader don't have to wait - // LeaseDuration time first. - // - // In the default scaffold provided, the program ends immediately after - // the manager stops, so would be fine to enable this option. However, - // if you are doing or is intended to do any operation such as perform cleanups - // after the manager stops then its usage might be unsafe. - // LeaderElectionReleaseOnCancel: true, - }, mcManagerOptions...) - if err != nil { - setupLog.Error(err, "unable to start manager") - os.Exit(1) - } - - downstreamRestConfig, err := serverConfig.DownstreamResourceManagement.RestConfig() - if err != nil { - setupLog.Error(err, "unable to load control plane kubeconfig") - os.Exit(1) - } - serverConfig.DownstreamClient.ApplyTo(downstreamRestConfig) - - downstreamCluster, err := cluster.New(downstreamRestConfig, func(o *cluster.Options) { - o.Scheme = scheme - o.Client = client.Options{ - Cache: &client.CacheOptions{ - Unstructured: true, - }, - } - }) - if err != nil { - setupLog.Error(err, "failed to construct cluster") - os.Exit(1) - } - - var singletonMgr manager.Manager - singletonControllerMgr := mgr.GetLocalManager() - if enableClusterSharding { - singletonMgr, err = manager.New(cfg, manager.Options{ - Scheme: scheme, - Metrics: metricsserver.Options{BindAddress: "0"}, - WebhookServer: webhook.NewServer(webhook.Options{Port: 0}), - HealthProbeBindAddress: "0", - LeaderElection: singletonControllersLeaderElection, - LeaderElectionID: singletonControllersLeaderElectionID, - LeaderElectionNamespace: leaderElectionNamespace, - LeaseDuration: &leaseDuration, - RenewDeadline: &renewDeadline, - RetryPeriod: &retryPeriod, - }) - if err != nil { - setupLog.Error(err, "unable to create singleton controller manager") - os.Exit(1) - } - singletonControllerMgr = singletonMgr - } - - if err := (&controller.NetworkReconciler{}).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Network") - os.Exit(1) - } - if err := (&controller.NetworkBindingReconciler{}).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "NetworkBinding") - os.Exit(1) - } - if err := (&controller.NetworkContextReconciler{}).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "NetworkContext") - os.Exit(1) - } - if err := (&controller.NetworkPolicyReconciler{}).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "NetworkPolicy") - os.Exit(1) - } - if err := (&controller.SubnetReconciler{}).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Subnet") - os.Exit(1) - } - if err := (&controller.SubnetClaimReconciler{}).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "SubnetClaim") - os.Exit(1) - } - - if err := (&controller.HTTPProxyReconciler{ - Config: serverConfig, - DownstreamCluster: downstreamCluster, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "HTTPProxy") - os.Exit(1) - } - - if err := (&controller.GatewayReconciler{ - Config: serverConfig, - DownstreamCluster: downstreamCluster, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Gateway") - os.Exit(1) - } - if err := (&controller.GatewayClassReconciler{ - Config: serverConfig, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "GatewayClass") - os.Exit(1) - } - - if err := (&controller.GatewayDownstreamGCReconciler{ - Config: serverConfig, - DownstreamCluster: downstreamCluster, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "GatewayDownstreamGC") - os.Exit(1) - } - - if err := (&controller.GatewayResourceReplicatorReconciler{ - Config: serverConfig, - DownstreamCluster: downstreamCluster, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "GatewayResourceReplicator") - os.Exit(1) - } - - if !serverConfig.Gateway.Coraza.Disabled { - if err = (&controller.TrafficProtectionPolicyReconciler{ - Config: serverConfig, - DownstreamCluster: downstreamCluster, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "WAFSecurityPolicy") - os.Exit(1) - } - } - - if serverConfig.Gateway.EnableDownstreamCertificateSolver { - setupLog.Info("enabling GatewayDownstreamCertificateSolver controller") - if err := (&controller.GatewayDownstreamCertificateSolverReconciler{ - Config: serverConfig, - DownstreamCluster: downstreamCluster, - }).SetupWithManager(singletonControllerMgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "GatewayDownstreamCertificateSolver") - os.Exit(1) - } - } - - if err := (&controller.DomainReconciler{ - Config: serverConfig, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Domain") - os.Exit(1) - } - - if err := (&controller.ConnectorReconciler{ - Config: serverConfig, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Connector") - os.Exit(1) - } - if err := (&controller.ConnectorAdvertisementReconciler{}).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "ConnectorAdvertisement") - os.Exit(1) - } - - var irohDownstream cluster.Cluster - if serverConfig.Connector.Iroh.DNSEnabled { - irohRestCfg, err := serverConfig.Connector.Iroh.DownstreamRestConfig() - if err != nil { - setupLog.Error(err, "unable to load iroh dns downstream kubeconfig") - os.Exit(1) - } - irohDownstream, err = cluster.New(irohRestCfg, func(o *cluster.Options) { - o.Scheme = scheme - }) - if err != nil { - setupLog.Error(err, "unable to build iroh dns downstream cluster") - os.Exit(1) - } - if err := (&controller.IrohDNSReconciler{ - Config: serverConfig, - Downstream: irohDownstream, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "IrohDNS") - os.Exit(1) - } - } - - if serverConfig.Gateway.ShouldDeleteErroredChallenges() { - if err := (&controller.ChallengeReconciler{ - Config: serverConfig, - DownstreamCluster: downstreamCluster, - }).SetupWithManager(singletonControllerMgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Challenge") - os.Exit(1) - } - } - - if err := controller.AddIndexers(ctx, mgr); err != nil { - setupLog.Error(err, "unable to add indexers") - os.Exit(1) - } - - if serverConfig.Gateway.EnableDNSIntegration { - if err := controller.AddDNSZoneDomainNameIndexer(ctx, mgr); err != nil { - setupLog.Error(err, "unable to add DNSZone indexer") - os.Exit(1) - } - } - - if err := networkinggatewayv1webhooks.SetupGatewayWebhookWithManager(mgr, serverConfig); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "Gateway") - os.Exit(1) - } - - if err := networkinggatewayv1webhooks.SetupHTTPRouteWebhookWithManager(mgr, serverConfig); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "HTTPRoute") - os.Exit(1) - } - - if err := networkinggatewayv1webhooks.SetupBackendTLSPolicyWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "HTTPRoute") - os.Exit(1) - } - - if err := networkingv1alphawebhooks.SetupHTTPProxyWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "HTTPProxy") - os.Exit(1) - } - - if err := networkingv1alphawebhooks.SetupDomainWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "Domain") - os.Exit(1) - } - - if err = webhookgatewayv1alpha1.SetupBackendTrafficPolicyWebhookWithManager(mgr, serverConfig); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "BackendTrafficPolicy") - os.Exit(1) - } - - if err = webhookgatewayv1alpha1.SetupSecurityPolicyWebhookWithManager(mgr, serverConfig); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "SecurityPolicy") - os.Exit(1) - } - - if err = webhookgatewayv1alpha1.SetupHTTPRouteFilterWebhookWithManager(mgr, serverConfig); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "HTTPRouteFilter") - os.Exit(1) - } - - if err = webhookgatewayv1alpha1.SetupBackendWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "Backend") - os.Exit(1) - } - - // +kubebuilder:scaffold:builder - - if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up health check") - os.Exit(1) - } - if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up ready check") - os.Exit(1) - } - - g, ctx := errgroup.WithContext(ctx) - for _, runnable := range runnables { - g.Go(func() error { - return ignoreCanceled(runnable.Start(ctx)) - }) - } - - // Providers that still implement the legacy Run(ctx, mgr) shape (e.g. the - // Milo provider) must be started by us. Providers that implement upstream's - // multicluster.ProviderRunnable interface (e.g. mcsingle) are started - // automatically by mgr.Start, so we skip them here. - if runner, ok := provider.(legacyRunnableProvider); ok { - setupLog.Info("starting cluster discovery provider") - g.Go(func() error { - return ignoreCanceled(runner.Run(ctx, mgr)) - }) - } - - g.Go(func() error { - return ignoreCanceled(downstreamCluster.Start(ctx)) - }) - - if irohDownstream != nil { - g.Go(func() error { - return ignoreCanceled(irohDownstream.Start(ctx)) - }) - } - - setupLog.Info("starting multicluster manager") - g.Go(func() error { - return ignoreCanceled(mgr.Start(ctx)) - }) - - if singletonMgr != nil { - setupLog.Info("starting singleton controller manager") - g.Go(func() error { - return ignoreCanceled(singletonMgr.Start(ctx)) - }) - } - - if err := g.Wait(); err != nil { - setupLog.Error(err, "unable to start") - os.Exit(1) - } - - return nil - }, + mcManagerOptions = append( + mcManagerOptions, + mcmanager.WithCoordinator( + sharded.New( + deploymentCluster.GetClient(), + ctrl.Log.WithName("cluster-sharding-coordinator"), + clusterShardingOptions..., + ), + ), + ) } - cmd.Flags().AddGoFlagSet(fs) - return cmd + primaryManagerLeaderElection := mcfg.enableLeaderElection + if mcfg.enableClusterSharding && mcfg.enableLeaderElection { + setupLog.Info( + "disabling primary manager leader election while cluster sharding is enabled", + "singletonControllersLeaderElection", + mcfg.singletonControllersLeaderElection, + ) + primaryManagerLeaderElection = false + } + + mgr, err := mcmanager.New(restCfg, provider, ctrl.Options{ + Scheme: scheme, + Metrics: metricsServerOptions, + WebhookServer: webhookServer, + HealthProbeBindAddress: mcfg.probeAddr, + LeaderElection: primaryManagerLeaderElection, + LeaderElectionID: "6a7d51cc.datumapis.com", + LeaderElectionNamespace: mcfg.leaderElectionNamespace, + LeaseDuration: &leaseDuration, + RenewDeadline: &renewDeadline, + RetryPeriod: &retryPeriod, + // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily + // when the Manager ends. This requires the binary to immediately end when the + // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly + // speeds up voluntary leader transitions as the new leader don't have to wait + // LeaseDuration time first. + // + // In the default scaffold provided, the program ends immediately after + // the manager stops, so would be fine to enable this option. However, + // if you are doing or is intended to do any operation such as perform cleanups + // after the manager stops then its usage might be unsafe. + // LeaderElectionReleaseOnCancel: true, + }, mcManagerOptions...) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + + downstreamRestConfig, err := serverConfig.DownstreamResourceManagement.RestConfig() + if err != nil { + setupLog.Error(err, "unable to load control plane kubeconfig") + os.Exit(1) + } + serverConfig.DownstreamClient.ApplyTo(downstreamRestConfig) + + downstreamCluster, err := cluster.New(downstreamRestConfig, func(o *cluster.Options) { + o.Scheme = scheme + o.Client = client.Options{ + Cache: &client.CacheOptions{ + Unstructured: true, + }, + } + }) + if err != nil { + setupLog.Error(err, "failed to construct cluster") + os.Exit(1) + } + + var singletonMgr manager.Manager + singletonControllerMgr := mgr.GetLocalManager() + if mcfg.enableClusterSharding { + singletonMgr, err = manager.New(restCfg, manager.Options{ + Scheme: scheme, + Metrics: metricsserver.Options{BindAddress: "0"}, + WebhookServer: webhook.NewServer(webhook.Options{Port: 0}), + HealthProbeBindAddress: "0", + LeaderElection: mcfg.singletonControllersLeaderElection, + LeaderElectionID: mcfg.singletonControllersLeaderElectionID, + LeaderElectionNamespace: mcfg.leaderElectionNamespace, + LeaseDuration: &leaseDuration, + RenewDeadline: &renewDeadline, + RetryPeriod: &retryPeriod, + }) + if err != nil { + setupLog.Error(err, "unable to create singleton controller manager") + os.Exit(1) + } + singletonControllerMgr = singletonMgr + } + + if err := (&controller.NetworkReconciler{}).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Network") + os.Exit(1) + } + if err := (&controller.NetworkBindingReconciler{}).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "NetworkBinding") + os.Exit(1) + } + if err := (&controller.NetworkContextReconciler{}).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "NetworkContext") + os.Exit(1) + } + if err := (&controller.NetworkPolicyReconciler{}).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "NetworkPolicy") + os.Exit(1) + } + if err := (&controller.SubnetReconciler{}).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Subnet") + os.Exit(1) + } + if err := (&controller.SubnetClaimReconciler{}).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "SubnetClaim") + os.Exit(1) + } + + if err := (&controller.HTTPProxyReconciler{ + Config: serverConfig, + DownstreamCluster: downstreamCluster, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "HTTPProxy") + os.Exit(1) + } + + if err := (&controller.GatewayReconciler{ + Config: serverConfig, + DownstreamCluster: downstreamCluster, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Gateway") + os.Exit(1) + } + if err := (&controller.GatewayClassReconciler{ + Config: serverConfig, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "GatewayClass") + os.Exit(1) + } + + if err := (&controller.GatewayDownstreamGCReconciler{ + Config: serverConfig, + DownstreamCluster: downstreamCluster, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "GatewayDownstreamGC") + os.Exit(1) + } + + if err := (&controller.OrphanedDownstreamHTTPRouteGCReconciler{ + DownstreamCluster: downstreamCluster, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "OrphanedDownstreamHTTPRouteGC") + os.Exit(1) + } + + if err := (&controller.GatewayResourceReplicatorReconciler{ + Config: serverConfig, + DownstreamCluster: downstreamCluster, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "GatewayResourceReplicator") + os.Exit(1) + } + + if !serverConfig.Gateway.Coraza.Disabled { + if err = (&controller.TrafficProtectionPolicyReconciler{ + Config: serverConfig, + DownstreamCluster: downstreamCluster, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "WAFSecurityPolicy") + os.Exit(1) + } + } + + if serverConfig.Gateway.EnableDownstreamCertificateSolver { + setupLog.Info("enabling GatewayDownstreamCertificateSolver controller") + if err := (&controller.GatewayDownstreamCertificateSolverReconciler{ + Config: serverConfig, + DownstreamCluster: downstreamCluster, + }).SetupWithManager(singletonControllerMgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "GatewayDownstreamCertificateSolver") + os.Exit(1) + } + } + + if err := (&controller.DomainReconciler{ + Config: serverConfig, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Domain") + os.Exit(1) + } + + if err := (&controller.ConnectorReconciler{ + Config: serverConfig, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Connector") + os.Exit(1) + } + if err := (&controller.ConnectorAdvertisementReconciler{}).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "ConnectorAdvertisement") + os.Exit(1) + } + + var irohDownstream cluster.Cluster + if serverConfig.Connector.Iroh.DNSEnabled { + irohRestCfg, err := serverConfig.Connector.Iroh.DownstreamRestConfig() + if err != nil { + setupLog.Error(err, "unable to load iroh dns downstream kubeconfig") + os.Exit(1) + } + irohDownstream, err = cluster.New(irohRestCfg, func(o *cluster.Options) { + o.Scheme = scheme + }) + if err != nil { + setupLog.Error(err, "unable to build iroh dns downstream cluster") + os.Exit(1) + } + if err := (&controller.IrohDNSReconciler{ + Config: serverConfig, + Downstream: irohDownstream, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "IrohDNS") + os.Exit(1) + } + } + + if serverConfig.Gateway.ShouldDeleteErroredChallenges() { + if err := (&controller.ChallengeReconciler{ + Config: serverConfig, + DownstreamCluster: downstreamCluster, + }).SetupWithManager(singletonControllerMgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Challenge") + os.Exit(1) + } + } + + if err := controller.AddIndexers(ctx, mgr); err != nil { + setupLog.Error(err, "unable to add indexers") + os.Exit(1) + } + + if serverConfig.Gateway.EnableDNSIntegration { + if err := controller.AddDNSZoneDomainNameIndexer(ctx, mgr); err != nil { + setupLog.Error(err, "unable to add DNSZone indexer") + os.Exit(1) + } + } + + if err := networkinggatewayv1webhooks.SetupGatewayWebhookWithManager(mgr, serverConfig); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "Gateway") + os.Exit(1) + } + + if err := networkinggatewayv1webhooks.SetupHTTPRouteWebhookWithManager(mgr, serverConfig); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "HTTPRoute") + os.Exit(1) + } + + if err := networkinggatewayv1webhooks.SetupBackendTLSPolicyWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "HTTPRoute") + os.Exit(1) + } + + if err := networkingv1alphawebhooks.SetupHTTPProxyWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "HTTPProxy") + os.Exit(1) + } + + if err := networkingv1alphawebhooks.SetupDomainWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "Domain") + os.Exit(1) + } + + if err = webhookgatewayv1alpha1.SetupBackendTrafficPolicyWebhookWithManager(mgr, serverConfig); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "BackendTrafficPolicy") + os.Exit(1) + } + + if err = webhookgatewayv1alpha1.SetupSecurityPolicyWebhookWithManager(mgr, serverConfig); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "SecurityPolicy") + os.Exit(1) + } + + if err = webhookgatewayv1alpha1.SetupHTTPRouteFilterWebhookWithManager(mgr, serverConfig); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "HTTPRouteFilter") + os.Exit(1) + } + + if err = webhookgatewayv1alpha1.SetupBackendWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "Backend") + os.Exit(1) + } + + // +kubebuilder:scaffold:builder + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up ready check") + os.Exit(1) + } + + g, ctx := errgroup.WithContext(ctx) + for _, runnable := range runnables { + g.Go(func() error { + return ignoreCanceled(runnable.Start(ctx)) + }) + } + + // Providers that still implement the legacy Run(ctx, mgr) shape (e.g. the + // Milo provider) must be started by us. Providers that implement upstream's + // multicluster.ProviderRunnable interface (e.g. mcsingle) are started + // automatically by mgr.Start, so we skip them here. + if runner, ok := provider.(legacyRunnableProvider); ok { + setupLog.Info("starting cluster discovery provider") + g.Go(func() error { + return ignoreCanceled(runner.Run(ctx, mgr)) + }) + } + + g.Go(func() error { + return ignoreCanceled(downstreamCluster.Start(ctx)) + }) + + if irohDownstream != nil { + g.Go(func() error { + return ignoreCanceled(irohDownstream.Start(ctx)) + }) + } + + setupLog.Info("starting multicluster manager") + g.Go(func() error { + return ignoreCanceled(mgr.Start(ctx)) + }) + + if singletonMgr != nil { + setupLog.Info("starting singleton controller manager") + g.Go(func() error { + return ignoreCanceled(singletonMgr.Start(ctx)) + }) + } + + if err := g.Wait(); err != nil { + setupLog.Error(err, "unable to start") + os.Exit(1) + } + + return nil } // legacyRunnableProvider matches providers that still expose the pre-upstream diff --git a/internal/controller/gateway_orphaned_httproute_gc_controller.go b/internal/controller/gateway_orphaned_httproute_gc_controller.go new file mode 100644 index 00000000..dad231e8 --- /dev/null +++ b/internal/controller/gateway_orphaned_httproute_gc_controller.go @@ -0,0 +1,205 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "fmt" + "strings" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" + mchandler "sigs.k8s.io/multicluster-runtime/pkg/handler" + mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" + "sigs.k8s.io/multicluster-runtime/pkg/multicluster" + mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + mcsource "sigs.k8s.io/multicluster-runtime/pkg/source" + + downstreamclient "go.datum.net/network-services-operator/internal/downstreamclient" +) + +// orphanMinAge is the minimum time a downstream HTTPRoute must exist without a +// reachable upstream Gateway before it is considered orphaned and eligible for +// deletion. +const orphanMinAge = 5 * time.Minute + +// OrphanedDownstreamHTTPRouteGCReconciler watches downstream HTTPRoutes on the +// Karmada cluster and deletes any that reference a Gateway which no longer +// exists on the upstream project cluster. This catches orphans left behind by +// failed or raced gateway finalization (e.g. the pre-v0.23.6 bug where +// detachHTTPRoutes checked Status.Parents instead of Spec.ParentRefs). +// +// Reconcile requests identify the upstream HTTPRoute that the downstream route +// was created from: ClusterName=upstream cluster, NamespacedName=upstream +// HTTPRoute namespace/name. This key is derived from the labels NSO stamps on +// every downstream HTTPRoute via SetControllerReference. +type OrphanedDownstreamHTTPRouteGCReconciler struct { + mgr mcmanager.Manager + DownstreamCluster cluster.Cluster + + // minOrphanAge overrides the orphanMinAge constant. Zero means use the + // constant. Exposed for unit tests. + minOrphanAge time.Duration +} + +func (r *OrphanedDownstreamHTTPRouteGCReconciler) effectiveMinAge() time.Duration { + if r.minOrphanAge > 0 { + return r.minOrphanAge + } + return orphanMinAge +} + +// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes,verbs=list;watch;delete + +func (r *OrphanedDownstreamHTTPRouteGCReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx).WithValues( + "cluster", req.ClusterName, + "namespace", req.Namespace, + jsonKeyName, req.Name, + ) + + upstreamCluster, err := r.mgr.GetCluster(ctx, req.ClusterName) + if err != nil { + return ctrl.Result{}, err + } + + // Find all downstream HTTPRoutes that were projected from this upstream + // HTTPRoute. There may be multiple if NSO wrote the route into more than + // one downstream namespace, though in practice there is one per upstream. + labelValue := fmt.Sprintf("cluster-%s", strings.ReplaceAll(string(req.ClusterName), "/", "_")) + + var routes gatewayv1.HTTPRouteList + if err := r.DownstreamCluster.GetClient().List(ctx, &routes, + client.MatchingLabels{ + downstreamclient.UpstreamOwnerClusterNameLabel: labelValue, + downstreamclient.UpstreamOwnerNamespaceLabel: req.Namespace, + downstreamclient.UpstreamOwnerNameLabel: req.Name, + }, + ); err != nil { + return ctrl.Result{}, fmt.Errorf("listing downstream HTTPRoutes: %w", err) + } + + for i := range routes.Items { + route := &routes.Items[i] + + if !route.DeletionTimestamp.IsZero() { + continue + } + + age := time.Since(route.CreationTimestamp.Time) + if age < r.effectiveMinAge() { + logger.Info("downstream HTTPRoute too young; requeueing", + "route", route.Name, "age", age.Round(time.Second)) + return ctrl.Result{RequeueAfter: r.effectiveMinAge() - age}, nil + } + + // Check whether any upstream Gateway referenced by this route still exists. + // If at least one live parent Gateway is found the route is healthy. + hasLiveParent := false + for _, ref := range route.Spec.ParentRefs { + if ptr.Deref(ref.Group, gatewayv1.GroupName) != gatewayv1.GroupName || + ptr.Deref(ref.Kind, KindGateway) != KindGateway { + continue + } + + key := types.NamespacedName{ + Namespace: req.Namespace, + Name: string(ref.Name), + } + gw := &gatewayv1.Gateway{} + if err := upstreamCluster.GetClient().Get(ctx, key, gw); err != nil { + if !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("checking upstream Gateway %s: %w", key, err) + } + // NotFound: this parent is gone, keep checking others. + continue + } + hasLiveParent = true + break + } + + if hasLiveParent { + continue + } + + logger.Info("deleting orphaned downstream HTTPRoute: no live upstream Gateway", + "route", route.Name, "namespace", route.Namespace) + + if err := r.DownstreamCluster.GetClient().Delete(ctx, route); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("deleting orphaned downstream HTTPRoute %s/%s: %w", + route.Namespace, route.Name, err) + } + } + + return ctrl.Result{}, nil +} + +func (r *OrphanedDownstreamHTTPRouteGCReconciler) SetupWithManager(mgr mcmanager.Manager) error { + r.mgr = mgr + + downstreamHTTPRouteSrc := mcsource.Kind( + &gatewayv1.HTTPRoute{}, + r.enqueueFromDownstreamHTTPRoute, + ) + downstreamSrc, _, err := downstreamHTTPRouteSrc.ForCluster("", r.DownstreamCluster) + if err != nil { + return fmt.Errorf("building downstream HTTPRoute watch source: %w", err) + } + + return mcbuilder.ControllerManagedBy(mgr). + // Watch upstream HTTPRoutes so we reconcile when they are synced or deleted. + // On deletion the reconcile will find no live Gateway and clean up any + // surviving downstream HTTPRoute. + Watches( + &gatewayv1.HTTPRoute{}, + mchandler.EnqueueRequestsFromMapFunc(func(_ context.Context, obj client.Object) []ctrl.Request { + return []ctrl.Request{{NamespacedName: client.ObjectKeyFromObject(obj)}} + }), + ). + // Watch downstream HTTPRoutes so we reconcile when a new one appears on + // Karmada — this covers existing orphans present at startup. + WatchesRawSource(downstreamSrc). + Named("orphaned_downstream_httproute_gc"). + Complete(r) +} + +// enqueueFromDownstreamHTTPRoute maps a downstream HTTPRoute event to the +// reconcile request that identifies its upstream HTTPRoute. +func (r *OrphanedDownstreamHTTPRouteGCReconciler) enqueueFromDownstreamHTTPRoute( + _ multicluster.ClusterName, + _ cluster.Cluster, +) handler.TypedEventHandler[*gatewayv1.HTTPRoute, mcreconcile.Request] { + return handler.TypedEnqueueRequestsFromMapFunc(func(_ context.Context, route *gatewayv1.HTTPRoute) []mcreconcile.Request { + if !route.DeletionTimestamp.IsZero() { + return nil + } + + labels := route.GetLabels() + clusterLabel := labels[downstreamclient.UpstreamOwnerClusterNameLabel] + upstreamNamespace := labels[downstreamclient.UpstreamOwnerNamespaceLabel] + upstreamName := labels[downstreamclient.UpstreamOwnerNameLabel] + + if clusterLabel == "" || upstreamNamespace == "" || upstreamName == "" { + return nil + } + + return []mcreconcile.Request{{ + ClusterName: multicluster.ClusterName(downstreamclient.UpstreamClusterNameFromLabel(clusterLabel)), + Request: ctrl.Request{ + NamespacedName: types.NamespacedName{ + Namespace: upstreamNamespace, + Name: upstreamName, + }, + }, + }} + }) +} diff --git a/internal/controller/gateway_orphaned_httproute_gc_controller_test.go b/internal/controller/gateway_orphaned_httproute_gc_controller_test.go new file mode 100644 index 00000000..28471767 --- /dev/null +++ b/internal/controller/gateway_orphaned_httproute_gc_controller_test.go @@ -0,0 +1,201 @@ +package controller + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/multicluster-runtime/pkg/multicluster" + mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + + downstreamclient "go.datum.net/network-services-operator/internal/downstreamclient" +) + +func TestOrphanedDownstreamHTTPRouteGC(t *testing.T) { + testScheme := runtime.NewScheme() + assert.NoError(t, scheme.AddToScheme(testScheme)) + assert.NoError(t, gatewayv1.Install(testScheme)) + + const upstreamCluster = "test-cluster" + const upstreamNamespace = "test-ns" + const upstreamHTTPRouteName = "test-route" + const gatewayName = "test-gateway" + + clusterLabel := fmt.Sprintf("cluster-%s", upstreamCluster) + downstreamNamespace := fmt.Sprintf("ns-%s", uuid.NewUUID()) + + makeDownstreamRoute := func(age time.Duration) *gatewayv1.HTTPRoute { + return &gatewayv1.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: downstreamNamespace, + Name: upstreamHTTPRouteName, + CreationTimestamp: metav1.NewTime(time.Now().Add(-age)), + Labels: map[string]string{ + downstreamclient.UpstreamOwnerClusterNameLabel: clusterLabel, + downstreamclient.UpstreamOwnerNamespaceLabel: upstreamNamespace, + downstreamclient.UpstreamOwnerNameLabel: upstreamHTTPRouteName, + }, + }, + Spec: gatewayv1.HTTPRouteSpec{ + CommonRouteSpec: gatewayv1.CommonRouteSpec{ + ParentRefs: []gatewayv1.ParentReference{ + {Name: gatewayName}, + }, + }, + }, + } + } + + upstreamGateway := &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: upstreamNamespace, + Name: gatewayName, + }, + } + + makeRequest := func() mcreconcile.Request { + return mcreconcile.Request{ + ClusterName: multicluster.ClusterName(upstreamCluster), + Request: reconcile.Request{ + NamespacedName: client.ObjectKey{ + Namespace: upstreamNamespace, + Name: upstreamHTTPRouteName, + }, + }, + } + } + + t.Run("orphaned route deleted when upstream gateway missing", func(t *testing.T) { + downstreamRoute := makeDownstreamRoute(10 * time.Minute) + + fakeUpstreamClient := fake.NewClientBuilder(). + WithScheme(testScheme). + WithObjects(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: upstreamNamespace}}). + Build() + + fakeDownstreamClient := fake.NewClientBuilder(). + WithScheme(testScheme). + WithObjects(downstreamRoute). + Build() + + reconciler := &OrphanedDownstreamHTTPRouteGCReconciler{ + mgr: &fakeMockManager{cl: fakeUpstreamClient}, + DownstreamCluster: &fakeCluster{cl: fakeDownstreamClient}, + minOrphanAge: 1 * time.Millisecond, + } + + _, err := reconciler.Reconcile(context.Background(), makeRequest()) + assert.NoError(t, err) + + err = fakeDownstreamClient.Get(context.Background(), client.ObjectKeyFromObject(downstreamRoute), &gatewayv1.HTTPRoute{}) + assert.True(t, apierrors.IsNotFound(err), "orphaned downstream HTTPRoute should have been deleted") + }) + + t.Run("healthy route not deleted when upstream gateway exists", func(t *testing.T) { + downstreamRoute := makeDownstreamRoute(10 * time.Minute) + + fakeUpstreamClient := fake.NewClientBuilder(). + WithScheme(testScheme). + WithObjects(upstreamGateway). + Build() + + fakeDownstreamClient := fake.NewClientBuilder(). + WithScheme(testScheme). + WithObjects(downstreamRoute). + Build() + + reconciler := &OrphanedDownstreamHTTPRouteGCReconciler{ + mgr: &fakeMockManager{cl: fakeUpstreamClient}, + DownstreamCluster: &fakeCluster{cl: fakeDownstreamClient}, + minOrphanAge: 1 * time.Millisecond, + } + + _, err := reconciler.Reconcile(context.Background(), makeRequest()) + assert.NoError(t, err) + + err = fakeDownstreamClient.Get(context.Background(), client.ObjectKeyFromObject(downstreamRoute), &gatewayv1.HTTPRoute{}) + assert.NoError(t, err, "healthy downstream HTTPRoute should not have been deleted") + }) + + t.Run("young route not deleted; requeue returned", func(t *testing.T) { + downstreamRoute := makeDownstreamRoute(1 * time.Second) + + fakeUpstreamClient := fake.NewClientBuilder(). + WithScheme(testScheme). + Build() + + fakeDownstreamClient := fake.NewClientBuilder(). + WithScheme(testScheme). + WithObjects(downstreamRoute). + Build() + + reconciler := &OrphanedDownstreamHTTPRouteGCReconciler{ + mgr: &fakeMockManager{cl: fakeUpstreamClient}, + DownstreamCluster: &fakeCluster{cl: fakeDownstreamClient}, + minOrphanAge: orphanMinAge, + } + + result, err := reconciler.Reconcile(context.Background(), makeRequest()) + assert.NoError(t, err) + assert.Greater(t, result.RequeueAfter, time.Duration(0), "should requeue until min age is reached") + + err = fakeDownstreamClient.Get(context.Background(), client.ObjectKeyFromObject(downstreamRoute), &gatewayv1.HTTPRoute{}) + assert.NoError(t, err, "young downstream HTTPRoute should not have been deleted") + }) + + t.Run("route with multiple parentRefs kept if any gateway still exists", func(t *testing.T) { + downstreamRoute := makeDownstreamRoute(10 * time.Minute) + downstreamRoute.Spec.ParentRefs = []gatewayv1.ParentReference{ + {Name: "missing-gateway"}, + {Name: gatewayName}, + } + + fakeUpstreamClient := fake.NewClientBuilder(). + WithScheme(testScheme). + WithObjects(upstreamGateway). + Build() + + fakeDownstreamClient := fake.NewClientBuilder(). + WithScheme(testScheme). + WithObjects(downstreamRoute). + Build() + + reconciler := &OrphanedDownstreamHTTPRouteGCReconciler{ + mgr: &fakeMockManager{cl: fakeUpstreamClient}, + DownstreamCluster: &fakeCluster{cl: fakeDownstreamClient}, + minOrphanAge: 1 * time.Millisecond, + } + + _, err := reconciler.Reconcile(context.Background(), makeRequest()) + assert.NoError(t, err) + + err = fakeDownstreamClient.Get(context.Background(), client.ObjectKeyFromObject(downstreamRoute), &gatewayv1.HTTPRoute{}) + assert.NoError(t, err, "route with one live parent should not be deleted") + }) + + t.Run("no downstream routes is a no-op", func(t *testing.T) { + fakeUpstreamClient := fake.NewClientBuilder().WithScheme(testScheme).Build() + fakeDownstreamClient := fake.NewClientBuilder().WithScheme(testScheme).Build() + + reconciler := &OrphanedDownstreamHTTPRouteGCReconciler{ + mgr: &fakeMockManager{cl: fakeUpstreamClient}, + DownstreamCluster: &fakeCluster{cl: fakeDownstreamClient}, + minOrphanAge: 1 * time.Millisecond, + } + + _, err := reconciler.Reconcile(context.Background(), makeRequest()) + assert.NoError(t, err) + }) +}