diff --git a/cli/cmd/get.go b/cli/cmd/get.go index bf5d6079d..108bc991d 100644 --- a/cli/cmd/get.go +++ b/cli/cmd/get.go @@ -107,7 +107,7 @@ func isInjectedPod(pod corev1.Pod) bool { return true } for _, container := range allContainers(pod) { - if container.Name == inject.ProxylessXServerContainerName { + if container.Name == inject.ProxylessGRPCInboundContainerName { return true } for _, env := range container.Env { diff --git a/cli/cmd/get_test.go b/cli/cmd/get_test.go index 9b8b5a15f..ef9af75c8 100644 --- a/cli/cmd/get_test.go +++ b/cli/cmd/get_test.go @@ -29,10 +29,10 @@ func TestIsInjectedPod(t *testing.T) { want: true, }, { - name: "xserver container", + name: "grpc-inbound container", pod: corev1.Pod{ Spec: corev1.PodSpec{ - Containers: []corev1.Container{{Name: inject.ProxylessXServerContainerName}}, + Containers: []corev1.Container{{Name: inject.ProxylessGRPCInboundContainerName}}, }, }, want: true, @@ -80,14 +80,14 @@ func TestPrintInjectedPods(t *testing.T) { }, Spec: corev1.PodSpec{ NodeName: "node-a", - Containers: []corev1.Container{{Name: "app"}, {Name: inject.ProxylessXServerContainerName}}, + Containers: []corev1.Container{{Name: "app"}, {Name: inject.ProxylessGRPCInboundContainerName}}, }, Status: corev1.PodStatus{ Phase: corev1.PodRunning, PodIP: "10.0.0.7", ContainerStatuses: []corev1.ContainerStatus{ {Name: "app", Ready: true, RestartCount: 1}, - {Name: inject.ProxylessXServerContainerName, Ready: true}, + {Name: inject.ProxylessGRPCInboundContainerName, Ready: true}, }, }, } diff --git a/cli/cmd/multicluster.go b/cli/cmd/multicluster.go index e51bfe95b..1d3766b10 100644 --- a/cli/cmd/multicluster.go +++ b/cli/cmd/multicluster.go @@ -136,7 +136,7 @@ func generateEastWestGatewayCmd() *cobra.Command { namespace: "dubbo-system", serviceType: "LoadBalancer", port: 15443, - targetPort: inject.ProxylessXServerPort, + targetPort: inject.ProxylessGRPCInboundPort, } command := &cobra.Command{ Use: "generate-eastwest-gateway", @@ -160,7 +160,7 @@ func generateEastWestGatewayCmd() *cobra.Command { flags.StringVarP(&args.namespace, "namespace", "n", args.namespace, "Gateway namespace") flags.StringVar(&args.serviceType, "service-type", args.serviceType, "Managed gateway Service type") flags.Int32Var(&args.port, "port", args.port, "Externally reachable east-west Gateway port") - flags.IntVar(&args.targetPort, "target-port", args.targetPort, "Injected xserver target port used by the managed gateway Service") + flags.IntVar(&args.targetPort, "target-port", args.targetPort, "Injected grpc-inbound target port used by the managed gateway Service") flags.Int32Var(&args.nodePort, "node-port", 0, "Optional managed gateway Service nodePort") flags.StringVar(&args.xdsAddress, "xds-address", "", "ADS address used by the remote dxgate data plane") _ = command.MarkFlagRequired("xds-address") diff --git a/cni/main.go b/cni/main.go index 7e512f4ee..094372693 100644 --- a/cni/main.go +++ b/cni/main.go @@ -129,7 +129,7 @@ func installerFlagSet(name string, opts *cni.InstallerOptions) *flag.FlagSet { flags.StringVar(&opts.ManagedLabelValue, "managed-label-value", opts.ManagedLabelValue, "label value that marks mesh-managed Pods") flags.StringVar(&opts.IPTablesPath, "iptables-path", opts.IPTablesPath, "iptables binary used by the CNI plugin") flags.StringVar(&opts.IPSetPath, "ipset-path", opts.IPSetPath, "ipset binary used by the CNI plugin") - flags.IntVar(&opts.XServerPort, "xserver-port", opts.XServerPort, "xserver inbound port") + flags.IntVar(&opts.GRPCInboundPort, "grpc-inbound-port", opts.GRPCInboundPort, "grpc-inbound inbound port") return flags } diff --git a/dubbod/discovery/cmd/app/cmd.go b/dubbod/discovery/cmd/app/cmd.go index 3a5dbe09e..0751fe52e 100644 --- a/dubbod/discovery/cmd/app/cmd.go +++ b/dubbod/discovery/cmd/app/cmd.go @@ -34,10 +34,10 @@ var ( ) const ( - executeLogScope = "setup" - waitLogScope = "wait" - xclientLogScope = "xclient" - xserverLogScope = "xserver" + executeLogScope = "setup" + waitLogScope = "wait" + grpcOutboundLogScope = "grpc-outbound" + grpcInboundLogScope = "grpc-inbound" ) func NewRootCommand() *cobra.Command { @@ -61,8 +61,8 @@ func NewRootCommand() *cobra.Command { cmd.AddFlags(rootCmd) rootCmd.AddCommand(waitCmd) - rootCmd.AddCommand(newXClientCommand()) - rootCmd.AddCommand(newXServerCommand()) + rootCmd.AddCommand(newGRPCOutboundCommand()) + rootCmd.AddCommand(newGRPCInboundCommand()) return rootCmd } diff --git a/dubbod/discovery/cmd/app/cmd_test.go b/dubbod/discovery/cmd/app/cmd_test.go index dda5b982e..654ddd419 100644 --- a/dubbod/discovery/cmd/app/cmd_test.go +++ b/dubbod/discovery/cmd/app/cmd_test.go @@ -32,12 +32,12 @@ func TestRootCommandRegistersRenamedCommands(t *testing.T) { commands[command.Name()] = true } - for _, name := range []string{"execute", "xclient", "xserver"} { + for _, name := range []string{"execute", "grpc-outbound", "grpc-inbound"} { if !commands[name] { t.Fatalf("expected command %q to be registered; commands=%v", name, commands) } } - for _, name := range []string{"discovery", "startup", "xds-client"} { + for _, name := range []string{"discovery", "startup"} { if commands[name] { t.Fatalf("old command %q is still registered; commands=%v", name, commands) } @@ -47,7 +47,7 @@ func TestRootCommandRegistersRenamedCommands(t *testing.T) { func TestCommandsSetLogScopes(t *testing.T) { defer func() { log.SetDefaultScope("log") - for _, name := range []string{"log", executeLogScope, waitLogScope, xclientLogScope, xserverLogScope} { + for _, name := range []string{"log", executeLogScope, waitLogScope, grpcOutboundLogScope, grpcInboundLogScope} { if scope := log.FindScope(name); scope != nil { scope.SetOutput(os.Stderr) } @@ -61,8 +61,8 @@ func TestCommandsSetLogScopes(t *testing.T) { }{ {command: "execute", scope: "setup"}, {command: "wait", scope: "wait"}, - {command: "xclient", scope: "xclient"}, - {command: "xserver", scope: "xserver"}, + {command: "grpc-outbound", scope: "grpc-outbound"}, + {command: "grpc-inbound", scope: "grpc-inbound"}, } { t.Run(tt.command, func(t *testing.T) { var out bytes.Buffer diff --git a/dubbod/discovery/cmd/app/xds_server.go b/dubbod/discovery/cmd/app/grpc_inbound.go similarity index 63% rename from dubbod/discovery/cmd/app/xds_server.go rename to dubbod/discovery/cmd/app/grpc_inbound.go index 29ac6bed2..f31736b98 100644 --- a/dubbod/discovery/cmd/app/xds_server.go +++ b/dubbod/discovery/cmd/app/grpc_inbound.go @@ -36,7 +36,7 @@ import ( "github.com/spf13/cobra" ) -type xdsServerOptions struct { +type grpcInboundOptions struct { listen string upstream string bootstrapPath string @@ -46,30 +46,30 @@ type xdsServerOptions struct { connectTimeout time.Duration } -type xserverMTLSMode string +type grpcInboundMTLSMode string const ( - xserverMTLSModeDisable xserverMTLSMode = "DISABLE" - xserverMTLSModePermissive xserverMTLSMode = "PERMISSIVE" - xserverMTLSModeStrict xserverMTLSMode = "STRICT" + grpcInboundMTLSModeDisable grpcInboundMTLSMode = "DISABLE" + grpcInboundMTLSModePermissive grpcInboundMTLSMode = "PERMISSIVE" + grpcInboundMTLSModeStrict grpcInboundMTLSMode = "STRICT" ) -func newXServerCommand() *cobra.Command { - opts := &xdsServerOptions{ - listen: firstNonEmpty(os.Getenv("DUBBO_XSERVER_LISTEN"), fmt.Sprintf(":%d", inject.ProxylessXServerPort)), - upstream: firstNonEmpty(os.Getenv("DUBBO_XSERVER_UPSTREAM"), "127.0.0.1:80"), +func newGRPCInboundCommand() *cobra.Command { + opts := &grpcInboundOptions{ + listen: firstNonEmpty(os.Getenv("DUBBO_GRPC_INBOUND_LISTEN"), fmt.Sprintf(":%d", inject.ProxylessGRPCInboundPort)), + upstream: firstNonEmpty(os.Getenv("DUBBO_GRPC_INBOUND_UPSTREAM"), "127.0.0.1:80"), bootstrapPath: os.Getenv("GRPC_XDS_BOOTSTRAP"), runtimeConfig: firstNonEmpty(os.Getenv(inject.ProxylessGRPCConfigEnvName), inject.ProxylessGRPCConfigPath), - mtlsMode: os.Getenv("DUBBO_XSERVER_MTLS_MODE"), - acceptTimeout: durationSecondsFromEnv("DUBBO_XSERVER_ACCEPT_TIMEOUT", 0), - connectTimeout: durationSecondsFromEnv("DUBBO_XSERVER_CONNECT_TIMEOUT", 5*time.Second), + mtlsMode: os.Getenv("DUBBO_GRPC_INBOUND_MTLS_MODE"), + acceptTimeout: durationSecondsFromEnv("DUBBO_GRPC_INBOUND_ACCEPT_TIMEOUT", 0), + connectTimeout: durationSecondsFromEnv("DUBBO_GRPC_INBOUND_CONNECT_TIMEOUT", 5*time.Second), } c := &cobra.Command{ - Use: "xserver", + Use: "grpc-inbound", Short: "run an inbound mTLS data-plane proxy for proxyless workloads", Args: cobra.NoArgs, PreRunE: func(cmd *cobra.Command, args []string) error { - log.SetDefaultScope(xserverLogScope) + log.SetDefaultScope(grpcInboundLogScope) return nil }, RunE: func(cmd *cobra.Command, _ []string) error { @@ -86,33 +86,33 @@ func newXServerCommand() *cobra.Command { return c } -func (o *xdsServerOptions) run(ctx context.Context) error { +func (o *grpcInboundOptions) run(ctx context.Context) error { if o.bootstrapPath == "" { - return fmt.Errorf("xserver requires GRPC_XDS_BOOTSTRAP or --bootstrap") + return fmt.Errorf("grpc-inbound requires GRPC_XDS_BOOTSTRAP or --bootstrap") } if o.listen == "" { - return fmt.Errorf("xserver listen address is required") + return fmt.Errorf("grpc-inbound listen address is required") } if o.upstream == "" { - return fmt.Errorf("xserver upstream address is required") + return fmt.Errorf("grpc-inbound upstream address is required") } bootstrap, err := xdsresolver.ParseBootstrap(o.bootstrapPath) if err != nil { return err } - tlsConfig, err := xserverTLSConfigFromBootstrap(bootstrap) + tlsConfig, err := grpcInboundTLSConfigFromBootstrap(bootstrap) if err != nil { return err } lis, err := net.Listen("tcp", o.listen) if err != nil { - return fmt.Errorf("listen xserver %s: %w", o.listen, err) + return fmt.Errorf("listen grpc-inbound %s: %w", o.listen, err) } defer lis.Close() - return serveXServer(ctx, lis, tlsConfig, o.upstream, o.effectiveMTLSMode, o.acceptTimeout, o.connectTimeout) + return serveGRPCInbound(ctx, lis, tlsConfig, o.upstream, o.effectiveMTLSMode, o.acceptTimeout, o.connectTimeout) } -func xserverTLSConfigFromBootstrap(bootstrap *xdsresolver.BootstrapConfig) (*tls.Config, error) { +func grpcInboundTLSConfigFromBootstrap(bootstrap *xdsresolver.BootstrapConfig) (*tls.Config, error) { if bootstrap == nil { return nil, fmt.Errorf("bootstrap config is nil") } @@ -121,22 +121,22 @@ func xserverTLSConfigFromBootstrap(bootstrap *xdsresolver.BootstrapConfig) (*tls return nil, fmt.Errorf("certificate_providers[default] not found") } if cfg.CertificateFile == "" || cfg.PrivateKeyFile == "" { - return nil, fmt.Errorf("xserver mTLS requires certificate_file and private_key_file") + return nil, fmt.Errorf("grpc-inbound mTLS requires certificate_file and private_key_file") } if cfg.CACertificateFile == "" { - return nil, fmt.Errorf("xserver mTLS requires ca_certificate_file") + return nil, fmt.Errorf("grpc-inbound mTLS requires ca_certificate_file") } cert, err := tls.LoadX509KeyPair(cfg.CertificateFile, cfg.PrivateKeyFile) if err != nil { - return nil, fmt.Errorf("load xserver certificate/key: %w", err) + return nil, fmt.Errorf("load grpc-inbound certificate/key: %w", err) } rootPEM, err := os.ReadFile(cfg.CACertificateFile) if err != nil { - return nil, fmt.Errorf("read xserver CA certificate %s: %w", cfg.CACertificateFile, err) + return nil, fmt.Errorf("read grpc-inbound CA certificate %s: %w", cfg.CACertificateFile, err) } clientCAs := x509.NewCertPool() if !clientCAs.AppendCertsFromPEM(rootPEM) { - return nil, fmt.Errorf("parse xserver CA certificate %s: no certificates found", cfg.CACertificateFile) + return nil, fmt.Errorf("parse grpc-inbound CA certificate %s: no certificates found", cfg.CACertificateFile) } return &tls.Config{ MinVersion: tls.VersionTLS12, @@ -146,7 +146,7 @@ func xserverTLSConfigFromBootstrap(bootstrap *xdsresolver.BootstrapConfig) (*tls }, nil } -func serveXServer(ctx context.Context, lis net.Listener, tlsConfig *tls.Config, upstream string, mode func() xserverMTLSMode, acceptTimeout, connectTimeout time.Duration) error { +func serveGRPCInbound(ctx context.Context, lis net.Listener, tlsConfig *tls.Config, upstream string, mode func() grpcInboundMTLSMode, acceptTimeout, connectTimeout time.Duration) error { go func() { <-ctx.Done() _ = lis.Close() @@ -161,11 +161,11 @@ func serveXServer(ctx context.Context, lis net.Listener, tlsConfig *tls.Config, return err } } - go proxyXServerConnection(conn, tlsConfig, upstream, mode(), acceptTimeout, connectTimeout) + go proxyGRPCInboundConnection(conn, tlsConfig, upstream, mode(), acceptTimeout, connectTimeout) } } -func proxyXServerConnection(inbound net.Conn, tlsConfig *tls.Config, upstream string, mode xserverMTLSMode, acceptTimeout, connectTimeout time.Duration) { +func proxyGRPCInboundConnection(inbound net.Conn, tlsConfig *tls.Config, upstream string, mode grpcInboundMTLSMode, acceptTimeout, connectTimeout time.Duration) { defer inbound.Close() if acceptTimeout > 0 { _ = inbound.SetDeadline(time.Now().Add(acceptTimeout)) @@ -178,7 +178,7 @@ func proxyXServerConnection(inbound net.Conn, tlsConfig *tls.Config, upstream st } buffered := &bufferedConn{Conn: inbound, reader: reader} if isTLSClientHello(first[0]) { - if mode == xserverMTLSModeDisable { + if mode == grpcInboundMTLSModeDisable { return } tlsConn := tls.Server(buffered, tlsConfig) @@ -187,7 +187,7 @@ func proxyXServerConnection(inbound net.Conn, tlsConfig *tls.Config, upstream st } inbound = tlsConn } else { - if mode == xserverMTLSModeStrict { + if mode == grpcInboundMTLSModeStrict { return } inbound = buffered @@ -216,24 +216,24 @@ func isTLSClientHello(first byte) bool { return first == 0x16 } -func (o *xdsServerOptions) effectiveMTLSMode() xserverMTLSMode { - if mode, ok := parseXServerMTLSMode(o.mtlsMode); ok { +func (o *grpcInboundOptions) effectiveMTLSMode() grpcInboundMTLSMode { + if mode, ok := parseGRPCInboundMTLSMode(o.mtlsMode); ok { return mode } - if mode, ok := xserverMTLSModeFromRuntimeConfig(o.runtimeConfig, upstreamPort(o.upstream)); ok { + if mode, ok := grpcInboundMTLSModeFromRuntimeConfig(o.runtimeConfig, upstreamPort(o.upstream)); ok { return mode } - return xserverMTLSModePermissive + return grpcInboundMTLSModePermissive } -func parseXServerMTLSMode(mode string) (xserverMTLSMode, bool) { +func parseGRPCInboundMTLSMode(mode string) (grpcInboundMTLSMode, bool) { switch strings.ToUpper(strings.TrimSpace(mode)) { - case string(xserverMTLSModeDisable): - return xserverMTLSModeDisable, true - case string(xserverMTLSModePermissive): - return xserverMTLSModePermissive, true - case string(xserverMTLSModeStrict): - return xserverMTLSModeStrict, true + case string(grpcInboundMTLSModeDisable): + return grpcInboundMTLSModeDisable, true + case string(grpcInboundMTLSModePermissive): + return grpcInboundMTLSModePermissive, true + case string(grpcInboundMTLSModeStrict): + return grpcInboundMTLSModeStrict, true default: return "", false } @@ -251,7 +251,7 @@ func upstreamPort(upstream string) int { return out } -func xserverMTLSModeFromRuntimeConfig(path string, port int) (xserverMTLSMode, bool) { +func grpcInboundMTLSModeFromRuntimeConfig(path string, port int) (grpcInboundMTLSMode, bool) { if path == "" { return "", false } @@ -278,22 +278,22 @@ func xserverMTLSModeFromRuntimeConfig(path string, port int) (xserverMTLSMode, b if port != 0 && svcPort.Port != port { continue } - mode, ok := parseXServerMTLSMode(svcPort.MTLSMode) + mode, ok := parseGRPCInboundMTLSMode(svcPort.MTLSMode) if !ok { continue } - if mode == xserverMTLSModeStrict { - return xserverMTLSModeStrict, true + if mode == grpcInboundMTLSModeStrict { + return grpcInboundMTLSModeStrict, true } - foundPermissive = foundPermissive || mode == xserverMTLSModePermissive - foundDisable = foundDisable || mode == xserverMTLSModeDisable + foundPermissive = foundPermissive || mode == grpcInboundMTLSModePermissive + foundDisable = foundDisable || mode == grpcInboundMTLSModeDisable } } if foundPermissive { - return xserverMTLSModePermissive, true + return grpcInboundMTLSModePermissive, true } if foundDisable { - return xserverMTLSModeDisable, true + return grpcInboundMTLSModeDisable, true } return "", false } diff --git a/dubbod/discovery/cmd/app/xds_server_test.go b/dubbod/discovery/cmd/app/grpc_inbound_test.go similarity index 84% rename from dubbod/discovery/cmd/app/xds_server_test.go rename to dubbod/discovery/cmd/app/grpc_inbound_test.go index 8b26fe036..fadac08f0 100644 --- a/dubbod/discovery/cmd/app/xds_server_test.go +++ b/dubbod/discovery/cmd/app/grpc_inbound_test.go @@ -22,10 +22,10 @@ import ( xdsresolver "github.com/kdubbo/xds-api/grpc/resolver" ) -func TestXServerRequiresClientCertificateAndProxiesHTTP(t *testing.T) { +func TestGRPCInboundRequiresClientCertificateAndProxiesHTTP(t *testing.T) { caCert, caKey := newTestCA(t) - serverCert, serverKey := newSignedCert(t, caCert, caKey, "xserver") - clientCert, clientKey := newSignedCert(t, caCert, caKey, "xclient") + serverCert, serverKey := newSignedCert(t, caCert, caKey, "grpc-inbound") + clientCert, clientKey := newSignedCert(t, caCert, caKey, "grpc-outbound") dir := t.TempDir() writePEM(t, filepath.Join(dir, "root-cert.pem"), "CERTIFICATE", caCert.Raw) writePEM(t, filepath.Join(dir, "cert-chain.pem"), "CERTIFICATE", serverCert.Raw) @@ -33,7 +33,7 @@ func TestXServerRequiresClientCertificateAndProxiesHTTP(t *testing.T) { writePEM(t, filepath.Join(dir, "client-cert.pem"), "CERTIFICATE", clientCert.Raw) writePEM(t, filepath.Join(dir, "client-key.pem"), "RSA PRIVATE KEY", x509.MarshalPKCS1PrivateKey(clientKey)) - tlsConfig, err := xserverTLSConfigFromBootstrap(&xdsresolver.BootstrapConfig{ + tlsConfig, err := grpcInboundTLSConfigFromBootstrap(&xdsresolver.BootstrapConfig{ CertProviders: map[string]xdsresolver.FileWatcherCertConfig{ "default": { CertificateFile: filepath.Join(dir, "cert-chain.pem"), @@ -43,7 +43,7 @@ func TestXServerRequiresClientCertificateAndProxiesHTTP(t *testing.T) { }, }) if err != nil { - t.Fatalf("xserverTLSConfigFromBootstrap() failed: %v", err) + t.Fatalf("grpcInboundTLSConfigFromBootstrap() failed: %v", err) } upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -60,8 +60,8 @@ func TestXServerRequiresClientCertificateAndProxiesHTTP(t *testing.T) { defer cancel() errCh := make(chan error, 1) go func() { - errCh <- serveXServer(ctx, listener, tlsConfig, upstreamAddr, func() xserverMTLSMode { - return xserverMTLSModeStrict + errCh <- serveGRPCInbound(ctx, listener, tlsConfig, upstreamAddr, func() grpcInboundMTLSMode { + return grpcInboundMTLSModeStrict }, time.Second, time.Second) }() t.Cleanup(func() { @@ -69,10 +69,10 @@ func TestXServerRequiresClientCertificateAndProxiesHTTP(t *testing.T) { select { case err := <-errCh: if err != nil { - t.Fatalf("serveXServer() error = %v", err) + t.Fatalf("serveGRPCInbound() error = %v", err) } case <-time.After(time.Second): - t.Fatalf("serveXServer() did not stop") + t.Fatalf("serveGRPCInbound() did not stop") } }) @@ -110,10 +110,10 @@ func TestXServerRequiresClientCertificateAndProxiesHTTP(t *testing.T) { } } -func TestXServerPermissiveAcceptsPlaintextAndMTLS(t *testing.T) { +func TestGRPCInboundPermissiveAcceptsPlaintextAndMTLS(t *testing.T) { caCert, caKey := newTestCA(t) - serverCert, serverKey := newSignedCert(t, caCert, caKey, "xserver") - clientCert, clientKey := newSignedCert(t, caCert, caKey, "xclient") + serverCert, serverKey := newSignedCert(t, caCert, caKey, "grpc-inbound") + clientCert, clientKey := newSignedCert(t, caCert, caKey, "grpc-outbound") dir := t.TempDir() writePEM(t, filepath.Join(dir, "root-cert.pem"), "CERTIFICATE", caCert.Raw) writePEM(t, filepath.Join(dir, "cert-chain.pem"), "CERTIFICATE", serverCert.Raw) @@ -121,7 +121,7 @@ func TestXServerPermissiveAcceptsPlaintextAndMTLS(t *testing.T) { writePEM(t, filepath.Join(dir, "client-cert.pem"), "CERTIFICATE", clientCert.Raw) writePEM(t, filepath.Join(dir, "client-key.pem"), "RSA PRIVATE KEY", x509.MarshalPKCS1PrivateKey(clientKey)) - tlsConfig, err := xserverTLSConfigFromBootstrap(&xdsresolver.BootstrapConfig{ + tlsConfig, err := grpcInboundTLSConfigFromBootstrap(&xdsresolver.BootstrapConfig{ CertProviders: map[string]xdsresolver.FileWatcherCertConfig{ "default": { CertificateFile: filepath.Join(dir, "cert-chain.pem"), @@ -131,7 +131,7 @@ func TestXServerPermissiveAcceptsPlaintextAndMTLS(t *testing.T) { }, }) if err != nil { - t.Fatalf("xserverTLSConfigFromBootstrap() failed: %v", err) + t.Fatalf("grpcInboundTLSConfigFromBootstrap() failed: %v", err) } upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -147,8 +147,8 @@ func TestXServerPermissiveAcceptsPlaintextAndMTLS(t *testing.T) { defer cancel() errCh := make(chan error, 1) go func() { - errCh <- serveXServer(ctx, listener, tlsConfig, upstream.Listener.Addr().String(), func() xserverMTLSMode { - return xserverMTLSModePermissive + errCh <- serveGRPCInbound(ctx, listener, tlsConfig, upstream.Listener.Addr().String(), func() grpcInboundMTLSMode { + return grpcInboundMTLSModePermissive }, time.Second, time.Second) }() t.Cleanup(func() { @@ -156,10 +156,10 @@ func TestXServerPermissiveAcceptsPlaintextAndMTLS(t *testing.T) { select { case err := <-errCh: if err != nil { - t.Fatalf("serveXServer() error = %v", err) + t.Fatalf("serveGRPCInbound() error = %v", err) } case <-time.After(time.Second): - t.Fatalf("serveXServer() did not stop") + t.Fatalf("serveGRPCInbound() did not stop") } }) @@ -197,7 +197,7 @@ func TestXServerPermissiveAcceptsPlaintextAndMTLS(t *testing.T) { } } -func TestXServerMTLSModeFromRuntimeConfig(t *testing.T) { +func TestGRPCInboundMTLSModeFromRuntimeConfig(t *testing.T) { path := filepath.Join(t.TempDir(), "dubbo-grpc-xds.json") if err := os.WriteFile(path, []byte(`{ "services": [ @@ -208,10 +208,10 @@ func TestXServerMTLSModeFromRuntimeConfig(t *testing.T) { t.Fatalf("os.WriteFile() failed: %v", err) } - if got, ok := xserverMTLSModeFromRuntimeConfig(path, 80); !ok || got != xserverMTLSModePermissive { + if got, ok := grpcInboundMTLSModeFromRuntimeConfig(path, 80); !ok || got != grpcInboundMTLSModePermissive { t.Fatalf("mode for 80 = %q, %v; want PERMISSIVE, true", got, ok) } - if got, ok := xserverMTLSModeFromRuntimeConfig(path, 8080); !ok || got != xserverMTLSModeStrict { + if got, ok := grpcInboundMTLSModeFromRuntimeConfig(path, 8080); !ok || got != grpcInboundMTLSModeStrict { t.Fatalf("mode for 8080 = %q, %v; want STRICT, true", got, ok) } } diff --git a/dubbod/discovery/cmd/app/xds_client.go b/dubbod/discovery/cmd/app/grpc_outbound.go similarity index 98% rename from dubbod/discovery/cmd/app/xds_client.go rename to dubbod/discovery/cmd/app/grpc_outbound.go index 02e68c72b..2974b95dc 100644 --- a/dubbod/discovery/cmd/app/xds_client.go +++ b/dubbod/discovery/cmd/app/grpc_outbound.go @@ -54,7 +54,7 @@ import ( "google.golang.org/protobuf/types/known/structpb" ) -type xdsClientOptions struct { +type grpcOutboundOptions struct { host string port int path string @@ -126,19 +126,19 @@ type sampleADSClient struct { bootstrap *xdsresolver.BootstrapConfig } -func newXClientCommand() *cobra.Command { +func newGRPCOutboundCommand() *cobra.Command { namespace := firstNonEmpty(os.Getenv("POD_NAMESPACE"), "default") trustDomain := firstNonEmpty(os.Getenv("TRUST_DOMAIN"), constants.DefaultClusterLocalDomain) domainSuffix := firstNonEmpty(os.Getenv("DOMAIN_SUFFIX"), trustDomain, constants.DefaultClusterLocalDomain) host, port, hostErr := autoDiscoverServiceTarget(os.Environ(), namespace, domainSuffix) - opts := &xdsClientOptions{ + opts := &grpcOutboundOptions{ host: firstNonEmpty(os.Getenv("DUBBO_SERVICE_HOST"), host), port: firstIntFromEnv(int(port), "DUBBO_SERVICE_PORT"), path: firstNonEmpty(os.Getenv("REQUEST_PATH"), "/"), xdsAddress: firstNonEmpty(os.Getenv("XDS_ADDRESS"), "dubbod.dubbo-system.svc:26010"), bootstrapPath: os.Getenv("GRPC_XDS_BOOTSTRAP"), namespace: namespace, - podName: firstNonEmpty(os.Getenv("POD_NAME"), os.Getenv("HOSTNAME"), "xclient"), + podName: firstNonEmpty(os.Getenv("POD_NAME"), os.Getenv("HOSTNAME"), "grpc-outbound"), podIP: firstNonEmpty(os.Getenv("INSTANCE_IP"), os.Getenv("POD_IP"), "127.0.0.1"), serviceAccount: firstNonEmpty(os.Getenv("SERVICE_ACCOUNT"), "default"), trustDomain: trustDomain, @@ -150,11 +150,11 @@ func newXClientCommand() *cobra.Command { } c := &cobra.Command{ - Use: "xclient [count]", + Use: "grpc-outbound [count]", Short: "run a no-proxy ADS stream client for service-to-service sample traffic", Args: cobra.MaximumNArgs(1), PreRunE: func(cmd *cobra.Command, args []string) error { - log.SetDefaultScope(xclientLogScope) + log.SetDefaultScope(grpcOutboundLogScope) return nil }, RunE: func(cmd *cobra.Command, args []string) error { @@ -195,7 +195,7 @@ func newXClientCommand() *cobra.Command { return c } -func (o *xdsClientOptions) run(ctx context.Context) error { +func (o *grpcOutboundOptions) run(ctx context.Context) error { if o.host == "" { return fmt.Errorf("target host is required; set --host or DUBBO_SERVICE_HOST, or expose exactly one Service in the namespace before starting the pod") } @@ -241,7 +241,7 @@ func (o *xdsClientOptions) run(ctx context.Context) error { return runSampleRequests(ctx, client, snapshot, o.count, o.requestInterval, o.requestTimeout) } -func newSampleADSClient(ctx context.Context, opts *xdsClientOptions) (*sampleADSClient, error) { +func newSampleADSClient(ctx context.Context, opts *grpcOutboundOptions) (*sampleADSClient, error) { node, addr, dialOpts, err := adsDialConfig(opts) if err != nil { return nil, err @@ -275,7 +275,7 @@ func newSampleADSClient(ctx context.Context, opts *xdsClientOptions) (*sampleADS }, nil } -func adsDialConfig(opts *xdsClientOptions) (*corev1.Node, string, []grpc.DialOption, error) { +func adsDialConfig(opts *grpcOutboundOptions) (*corev1.Node, string, []grpc.DialOption, error) { if opts.bootstrapPath != "" { if bootstrap, err := xdsresolver.ParseBootstrap(opts.bootstrapPath); err == nil { creds, err := xdsresolver.TransportCredentialsFromBootstrap(bootstrap) @@ -302,7 +302,7 @@ func adsDialConfig(opts *xdsClientOptions) (*corev1.Node, string, []grpc.DialOpt return node, opts.xdsAddress, []grpc.DialOption{creds}, nil } -func buildADSNode(opts *xdsClientOptions) (*corev1.Node, error) { +func buildADSNode(opts *grpcOutboundOptions) (*corev1.Node, error) { proxyConfig := meshconfig.DefaultProxyConfig() proxyConfig.DiscoveryAddress = opts.xdsAddress nodeID := strings.Join([]string{ diff --git a/dubbod/discovery/cmd/app/xds_client_test.go b/dubbod/discovery/cmd/app/grpc_outbound_test.go similarity index 100% rename from dubbod/discovery/cmd/app/xds_client_test.go rename to dubbod/discovery/cmd/app/grpc_outbound_test.go diff --git a/dubbod/discovery/cmd/app/xds_test_grpc.go b/dubbod/discovery/cmd/app/xds_test_grpc.go index b9732b546..3c0aa9b33 100644 --- a/dubbod/discovery/cmd/app/xds_test_grpc.go +++ b/dubbod/discovery/cmd/app/xds_test_grpc.go @@ -67,7 +67,7 @@ func startXDSTestGRPCServer(stop <-chan struct{}) error { } func (s *xdsTestGRPCServer) ForwardHTTP(ctx context.Context, req *testproto.ForwardHTTPRequest) (*testproto.ForwardHTTPResponse, error) { - opts, err := xdsOptionsFromForwardHTTPRequest(req) + opts, err := grpcOutboundOptionsFromForwardHTTPRequest(req) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } @@ -94,7 +94,7 @@ func (s *xdsTestGRPCServer) ForwardHTTP(ctx context.Context, req *testproto.Forw return &testproto.ForwardHTTPResponse{Output: output}, nil } -func xdsOptionsFromForwardHTTPRequest(req *testproto.ForwardHTTPRequest) (*xdsClientOptions, error) { +func grpcOutboundOptionsFromForwardHTTPRequest(req *testproto.ForwardHTTPRequest) (*grpcOutboundOptions, error) { if req == nil { return nil, fmt.Errorf("request is required") } @@ -124,7 +124,7 @@ func xdsOptionsFromForwardHTTPRequest(req *testproto.ForwardHTTPRequest) (*xdsCl if count <= 0 { count = 1 } - return &xdsClientOptions{ + return &grpcOutboundOptions{ host: host, port: port, path: normalizeRequestPath(req.GetPath()), @@ -132,7 +132,7 @@ func xdsOptionsFromForwardHTTPRequest(req *testproto.ForwardHTTPRequest) (*xdsCl target: net.JoinHostPort(host, strconv.Itoa(port)), xdsAddress: firstNonEmpty(os.Getenv("XDS_TEST_XDS_ADDRESS"), "127.0.0.1:26010"), namespace: namespace, - podName: firstNonEmpty(os.Getenv("POD_NAME"), os.Getenv("HOSTNAME"), "xclient"), + podName: firstNonEmpty(os.Getenv("POD_NAME"), os.Getenv("HOSTNAME"), "grpc-outbound"), podIP: firstNonEmpty(os.Getenv("INSTANCE_IP"), os.Getenv("POD_IP"), "127.0.0.1"), serviceAccount: serviceAccount, trustDomain: firstNonEmpty(os.Getenv("TRUST_DOMAIN"), constants.DefaultClusterLocalDomain), diff --git a/dubbod/discovery/pkg/bootstrap/proxyless_grpc_controller.go b/dubbod/discovery/pkg/bootstrap/proxyless_grpc_controller.go index 20984967f..57b688130 100644 --- a/dubbod/discovery/pkg/bootstrap/proxyless_grpc_controller.go +++ b/dubbod/discovery/pkg/bootstrap/proxyless_grpc_controller.go @@ -238,7 +238,7 @@ func proxylessGRPCRuntimeConfigNeedsUpdate(req *discoverymodel.PushRequest) bool } for cfg := range req.ConfigsUpdated { switch cfg.Kind { - case kind.HTTPRoute, kind.PeerAuthentication, kind.RequestAuthentication, kind.AuthorizationPolicy, kind.Service, kind.EndpointSlice, kind.Endpoints, kind.Pod, kind.Namespace: + case kind.HTTPRoute, kind.CircuitBreakerPolicy, kind.PeerAuthentication, kind.RequestAuthentication, kind.AuthorizationPolicy, kind.Service, kind.EndpointSlice, kind.Endpoints, kind.Pod, kind.Namespace: return true } } @@ -823,7 +823,7 @@ func rewriteRuntimeEndpointPortsForTLS(endpoints []proxylessGRPCEndpointRuntimeC out := make([]proxylessGRPCEndpointRuntimeConfig, len(endpoints)) copy(out, endpoints) for i := range out { - out[i].Port = inject.ProxylessXServerPort + out[i].Port = inject.ProxylessGRPCInboundPort } return out } diff --git a/dubbod/discovery/pkg/bootstrap/server.go b/dubbod/discovery/pkg/bootstrap/server.go index f1330ac91..f17a7827c 100644 --- a/dubbod/discovery/pkg/bootstrap/server.go +++ b/dubbod/discovery/pkg/bootstrap/server.go @@ -511,6 +511,8 @@ func (s *Server) initRegistryEventHandlers() { configKind = kind.KubernetesGateway case "HTTPRoute": configKind = kind.HTTPRoute + case "CircuitBreakerPolicy": + configKind = kind.CircuitBreakerPolicy default: log.Debugf("unknown schema identifier %s for %v, skipping", schemaID, cfg.GroupVersionKind) return @@ -532,7 +534,8 @@ func (s *Server) initRegistryEventHandlers() { needsFullPush := configKind == kind.PeerAuthentication || configKind == kind.RequestAuthentication || configKind == kind.AuthorizationPolicy || - configKind == kind.HTTPRoute + configKind == kind.HTTPRoute || + configKind == kind.CircuitBreakerPolicy // Trigger ConfigUpdate to push changes to all connected proxies s.XDSServer.ConfigUpdate(&model.PushRequest{ diff --git a/dubbod/discovery/pkg/config/kube/crdclient/types.gen.go b/dubbod/discovery/pkg/config/kube/crdclient/types.gen.go index 23ad57da3..c5bf20efc 100755 --- a/dubbod/discovery/pkg/config/kube/crdclient/types.gen.go +++ b/dubbod/discovery/pkg/config/kube/crdclient/types.gen.go @@ -15,7 +15,9 @@ import ( "github.com/apache/dubbo-kubernetes/pkg/kube" githubcomkdubboapimetav1alpha1 "github.com/kdubbo/api/meta/v1alpha1" + githubcomkdubboapinetworkingv1alpha3 "github.com/kdubbo/api/networking/v1alpha3" githubcomkdubboapisecurityv1alpha3 "github.com/kdubbo/api/security/v1alpha3" + apigithubcomapachedubbokubernetesapinetworkingv1alpha3 "github.com/kdubbo/client-go/pkg/apis/networking/v1alpha3" apigithubcomapachedubbokubernetesapisecurityv1alpha3 "github.com/kdubbo/client-go/pkg/apis/security/v1alpha3" k8sioapiadmissionregistrationv1 "k8s.io/api/admissionregistration/v1" k8sioapiappsv1 "k8s.io/api/apps/v1" @@ -35,6 +37,11 @@ func create(c kube.Client, cfg config.Config, objMeta metav1.ObjectMeta) (metav1 ObjectMeta: objMeta, Spec: *(cfg.Spec.(*githubcomkdubboapisecurityv1alpha3.AuthorizationPolicy)), }, metav1.CreateOptions{}) + case gvk.CircuitBreakerPolicy: + return c.Dubbo().NetworkingV1alpha3().CircuitBreakerPolicies(cfg.Namespace).Create(context.TODO(), &apigithubcomapachedubbokubernetesapinetworkingv1alpha3.CircuitBreakerPolicy{ + ObjectMeta: objMeta, + Spec: *(cfg.Spec.(*githubcomkdubboapinetworkingv1alpha3.CircuitBreakerPolicy)), + }, metav1.CreateOptions{}) case gvk.GatewayClass: return c.GatewayAPI().GatewayV1().GatewayClasses().Create(context.TODO(), &sigsk8siogatewayapiapisv1.GatewayClass{ ObjectMeta: objMeta, @@ -72,6 +79,11 @@ func update(c kube.Client, cfg config.Config, objMeta metav1.ObjectMeta) (metav1 ObjectMeta: objMeta, Spec: *(cfg.Spec.(*githubcomkdubboapisecurityv1alpha3.AuthorizationPolicy)), }, metav1.UpdateOptions{}) + case gvk.CircuitBreakerPolicy: + return c.Dubbo().NetworkingV1alpha3().CircuitBreakerPolicies(cfg.Namespace).Update(context.TODO(), &apigithubcomapachedubbokubernetesapinetworkingv1alpha3.CircuitBreakerPolicy{ + ObjectMeta: objMeta, + Spec: *(cfg.Spec.(*githubcomkdubboapinetworkingv1alpha3.CircuitBreakerPolicy)), + }, metav1.UpdateOptions{}) case gvk.GatewayClass: return c.GatewayAPI().GatewayV1().GatewayClasses().Update(context.TODO(), &sigsk8siogatewayapiapisv1.GatewayClass{ ObjectMeta: objMeta, @@ -109,6 +121,11 @@ func updateStatus(c kube.Client, cfg config.Config, objMeta metav1.ObjectMeta) ( ObjectMeta: objMeta, Status: *(cfg.Status.(*githubcomkdubboapimetav1alpha1.DubboStatus)), }, metav1.UpdateOptions{}) + case gvk.CircuitBreakerPolicy: + return c.Dubbo().NetworkingV1alpha3().CircuitBreakerPolicies(cfg.Namespace).UpdateStatus(context.TODO(), &apigithubcomapachedubbokubernetesapinetworkingv1alpha3.CircuitBreakerPolicy{ + ObjectMeta: objMeta, + Status: *(cfg.Status.(*githubcomkdubboapimetav1alpha1.DubboStatus)), + }, metav1.UpdateOptions{}) case gvk.GatewayClass: return c.GatewayAPI().GatewayV1().GatewayClasses().UpdateStatus(context.TODO(), &sigsk8siogatewayapiapisv1.GatewayClass{ ObjectMeta: objMeta, @@ -159,6 +176,21 @@ func patch(c kube.Client, orig config.Config, origMeta metav1.ObjectMeta, mod co } return c.Dubbo().SecurityV1alpha3().AuthorizationPolicies(orig.Namespace). Patch(context.TODO(), orig.Name, typ, patchBytes, metav1.PatchOptions{FieldManager: "pilot-discovery"}) + case gvk.CircuitBreakerPolicy: + oldRes := &apigithubcomapachedubbokubernetesapinetworkingv1alpha3.CircuitBreakerPolicy{ + ObjectMeta: origMeta, + Spec: *(orig.Spec.(*githubcomkdubboapinetworkingv1alpha3.CircuitBreakerPolicy)), + } + modRes := &apigithubcomapachedubbokubernetesapinetworkingv1alpha3.CircuitBreakerPolicy{ + ObjectMeta: modMeta, + Spec: *(mod.Spec.(*githubcomkdubboapinetworkingv1alpha3.CircuitBreakerPolicy)), + } + patchBytes, err := genPatchBytes(oldRes, modRes, typ) + if err != nil { + return nil, err + } + return c.Dubbo().NetworkingV1alpha3().CircuitBreakerPolicies(orig.Namespace). + Patch(context.TODO(), orig.Name, typ, patchBytes, metav1.PatchOptions{FieldManager: "pilot-discovery"}) case gvk.GatewayClass: oldRes := &sigsk8siogatewayapiapisv1.GatewayClass{ ObjectMeta: origMeta, @@ -247,6 +279,8 @@ func delete(c kube.Client, typ config.GroupVersionKind, name, namespace string, switch typ { case gvk.AuthorizationPolicy: return c.Dubbo().SecurityV1alpha3().AuthorizationPolicies(namespace).Delete(context.TODO(), name, deleteOptions) + case gvk.CircuitBreakerPolicy: + return c.Dubbo().NetworkingV1alpha3().CircuitBreakerPolicies(namespace).Delete(context.TODO(), name, deleteOptions) case gvk.GatewayClass: return c.GatewayAPI().GatewayV1().GatewayClasses().Delete(context.TODO(), name, deleteOptions) case gvk.HTTPRoute: @@ -282,6 +316,25 @@ var translationMap = map[config.GroupVersionKind]func(r runtime.Object) config.C Status: &obj.Status, } }, + gvk.CircuitBreakerPolicy: func(r runtime.Object) config.Config { + obj := r.(*apigithubcomapachedubbokubernetesapinetworkingv1alpha3.CircuitBreakerPolicy) + return config.Config{ + Meta: config.Meta{ + GroupVersionKind: gvk.CircuitBreakerPolicy, + Name: obj.Name, + Namespace: obj.Namespace, + Labels: obj.Labels, + Annotations: obj.Annotations, + ResourceVersion: obj.ResourceVersion, + CreationTimestamp: obj.CreationTimestamp.Time, + OwnerReferences: obj.OwnerReferences, + UID: string(obj.UID), + Generation: obj.Generation, + }, + Spec: &obj.Spec, + Status: &obj.Status, + } + }, gvk.ConfigMap: func(r runtime.Object) config.Config { obj := r.(*k8sioapicorev1.ConfigMap) return config.Config{ diff --git a/dubbod/discovery/pkg/config/kube/gateway/deployment_controller.go b/dubbod/discovery/pkg/config/kube/gateway/deployment_controller.go index c5befbaa1..324e07ec7 100644 --- a/dubbod/discovery/pkg/config/kube/gateway/deployment_controller.go +++ b/dubbod/discovery/pkg/config/kube/gateway/deployment_controller.go @@ -42,13 +42,19 @@ import ( "github.com/apache/dubbo-kubernetes/dubbod/discovery/pkg/features" "github.com/apache/dubbo-kubernetes/dubbod/discovery/pkg/model" "github.com/apache/dubbo-kubernetes/pkg/cluster" + "github.com/apache/dubbo-kubernetes/pkg/config" "github.com/apache/dubbo-kubernetes/pkg/config/constants" + "github.com/apache/dubbo-kubernetes/pkg/config/schema/gvk" "github.com/apache/dubbo-kubernetes/pkg/config/schema/gvr" "github.com/apache/dubbo-kubernetes/pkg/kube" "github.com/apache/dubbo-kubernetes/pkg/kube/controllers" "github.com/apache/dubbo-kubernetes/pkg/kube/inject" "github.com/apache/dubbo-kubernetes/pkg/kube/kclient" dubbolog "github.com/apache/dubbo-kubernetes/pkg/log" + networking "github.com/kdubbo/api/networking/v1alpha3" + clientnetworking "github.com/kdubbo/client-go/pkg/apis/networking/v1alpha3" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/wrapperspb" ) var logger = dubbolog.RegisterScope("gateway-deployment-controller", "gateway deployment controller debugging") @@ -117,6 +123,7 @@ type DeploymentController struct { serviceAccounts kclient.Client[*corev1.ServiceAccount] configMaps kclient.Client[*corev1.ConfigMap] httpRoutes kclient.Client[*gateway.HTTPRoute] + circuitBreakers kclient.Client[*clientnetworking.CircuitBreakerPolicy] namespaces kclient.Client[*corev1.Namespace] tagWatcher TagWatcher revision string @@ -221,6 +228,13 @@ func NewDeploymentController( } })) + dc.circuitBreakers = kclient.NewFiltered[*clientnetworking.CircuitBreakerPolicy](client, filter) + dc.circuitBreakers.AddEventHandler(controllers.ObjectHandler(func(o controllers.Object) { + for _, gw := range dc.gateways.List(metav1.NamespaceAll, klabels.Everything()) { + dc.queue.AddObject(gw) + } + })) + // Namespace is a cluster-scoped resource, use New instead of NewFiltered dc.namespaces = kclient.New[*corev1.Namespace](client) dc.namespaces.AddEventHandler(controllers.ObjectHandler(func(o controllers.Object) { @@ -270,6 +284,7 @@ func (d *DeploymentController) Run(stop <-chan struct{}) { d.serviceAccounts.HasSynced, d.configMaps.HasSynced, d.httpRoutes.HasSynced, + d.circuitBreakers.HasSynced, d.gateways.HasSynced, d.gatewayClasses.HasSynced, ) @@ -285,6 +300,7 @@ func (d *DeploymentController) Run(stop <-chan struct{}) { d.serviceAccounts, d.configMaps, d.httpRoutes, + d.circuitBreakers, d.gateways, d.gatewayClasses, ) @@ -479,8 +495,26 @@ type dxgateWeightedCluster struct { } type dxgateCluster struct { - Name string `json:"name" yaml:"name"` - Endpoints []dxgateEndpoint `json:"endpoints" yaml:"endpoints"` + Name string `json:"name" yaml:"name"` + Endpoints []dxgateEndpoint `json:"endpoints" yaml:"endpoints"` + CircuitBreaker *dxgateCircuitBreaker `json:"circuit_breaker,omitempty" yaml:"circuit_breaker,omitempty"` + Outlier *dxgateOutlierDetection `json:"outlier_detection,omitempty" yaml:"outlier_detection,omitempty"` +} + +type dxgateCircuitBreaker struct { + MaxConnections int32 `json:"max_connections,omitempty" yaml:"max_connections,omitempty"` + HTTP1MaxPendingRequests int32 `json:"http1_max_pending_requests,omitempty" yaml:"http1_max_pending_requests,omitempty"` + HTTP2MaxRequests int32 `json:"http2_max_requests,omitempty" yaml:"http2_max_requests,omitempty"` + MaxRequestsPerConnection int32 `json:"max_requests_per_connection,omitempty" yaml:"max_requests_per_connection,omitempty"` + MaxRetries int32 `json:"max_retries,omitempty" yaml:"max_retries,omitempty"` +} + +type dxgateOutlierDetection struct { + Consecutive5xxErrors uint32 `json:"consecutive_5xx_errors,omitempty" yaml:"consecutive_5xx_errors,omitempty"` + Interval string `json:"interval,omitempty" yaml:"interval,omitempty"` + BaseEjectionTime string `json:"base_ejection_time,omitempty" yaml:"base_ejection_time,omitempty"` + MaxEjectionPercent int32 `json:"max_ejection_percent,omitempty" yaml:"max_ejection_percent,omitempty"` + MinHealthPercent int32 `json:"min_health_percent,omitempty" yaml:"min_health_percent,omitempty"` } type dxgateEndpoint struct { @@ -567,13 +601,44 @@ func formatGatewayServicePorts(ports []corev1.ServicePort) string { func (d *DeploymentController) buildDxgateRuntimeConfig(gw gateway.Gateway) (string, string, error) { routes := d.httpRoutes.List(metav1.NamespaceAll, klabels.Everything()) - return buildDxgateRuntimeConfig(gw, routes, d.domainSuffix()) + return buildDxgateRuntimeConfig(gw, routes, d.circuitBreakerPolicyConfigs(), d.domainSuffix()) +} + +func (d *DeploymentController) circuitBreakerPolicyConfigs() []config.Config { + if d.circuitBreakers != nil { + policies := d.circuitBreakers.List(metav1.NamespaceAll, klabels.Everything()) + out := make([]config.Config, 0, len(policies)) + for _, policy := range policies { + out = append(out, config.Config{ + Meta: config.Meta{ + GroupVersionKind: gvk.CircuitBreakerPolicy, + Name: policy.Name, + Namespace: policy.Namespace, + Labels: policy.Labels, + Annotations: policy.Annotations, + ResourceVersion: policy.ResourceVersion, + CreationTimestamp: policy.CreationTimestamp.Time, + OwnerReferences: policy.OwnerReferences, + UID: string(policy.UID), + Generation: policy.Generation, + }, + Spec: policy.Spec.DeepCopy(), + Status: policy.Status.DeepCopy(), + }) + } + return out + } + if d.env != nil { + return d.env.List(gvk.CircuitBreakerPolicy, model.NamespaceAll) + } + return nil } -func buildDxgateRuntimeConfig(gw gateway.Gateway, routes []*gateway.HTTPRoute, domainSuffix string) (string, string, error) { +func buildDxgateRuntimeConfig(gw gateway.Gateway, routes []*gateway.HTTPRoute, policies []config.Config, domainSuffix string) (string, string, error) { if domainSuffix == "" { domainSuffix = constants.DefaultClusterLocalDomain } + circuitBreakers := circuitBreakerPoliciesByTarget(policies) sort.Slice(routes, func(i, j int) bool { if routes[i].Namespace != routes[j].Namespace { @@ -601,7 +666,7 @@ func buildDxgateRuntimeConfig(gw gateway.Gateway, routes []*gateway.HTTPRoute, d if !httpRouteReferencesGateway(hr, &gw) { continue } - vh, clusters := buildDxgateVirtualHost(gw, hr, domainSuffix) + vh, clusters := buildDxgateVirtualHost(gw, hr, circuitBreakers, domainSuffix) if len(vh.Routes) == 0 { continue } @@ -635,7 +700,7 @@ func dxgateRuntimeVersion(gw gateway.Gateway, routes []*gateway.HTTPRoute) strin return strings.Join(parts, ";") } -func buildDxgateVirtualHost(gw gateway.Gateway, hr *gateway.HTTPRoute, domainSuffix string) (dxgateVirtualHost, []dxgateCluster) { +func buildDxgateVirtualHost(gw gateway.Gateway, hr *gateway.HTTPRoute, circuitBreakers map[string]dxgateBackendCircuitBreaker, domainSuffix string) (dxgateVirtualHost, []dxgateCluster) { vh := dxgateVirtualHost{ Name: fmt.Sprintf("%s-%s", hr.Namespace, hr.Name), Domains: dxgateRouteDomains(gw, hr), @@ -668,13 +733,16 @@ func buildDxgateVirtualHost(gw gateway.Gateway, hr *gateway.HTTPRoute, domainSuf } port := uint16(*backendRef.Port) clusterName := fmt.Sprintf("%s-%s-%d-%d", hr.Namespace, hr.Name, ruleIdx, backendIdx) + policy := circuitBreakers[namespacedServiceKey(backendNamespace, string(backendRef.Name))] route.WeightedClusters = append(route.WeightedClusters, dxgateWeightedCluster{ Name: clusterName, Weight: weight, }) clusters = append(clusters, dxgateCluster{ - Name: clusterName, + Name: clusterName, + CircuitBreaker: policy.CircuitBreaker, + Outlier: policy.Outlier, Endpoints: []dxgateEndpoint{ { Address: fmt.Sprintf("%s.%s.svc.%s", backendRef.Name, backendNamespace, domainSuffix), @@ -692,6 +760,107 @@ func buildDxgateVirtualHost(gw gateway.Gateway, hr *gateway.HTTPRoute, domainSuf return vh, clusters } +type dxgateBackendCircuitBreaker struct { + CircuitBreaker *dxgateCircuitBreaker + Outlier *dxgateOutlierDetection +} + +func circuitBreakerPoliciesByTarget(policies []config.Config) map[string]dxgateBackendCircuitBreaker { + out := map[string]dxgateBackendCircuitBreaker{} + sort.Slice(policies, func(i, j int) bool { + if policies[i].CreationTimestamp != policies[j].CreationTimestamp { + return policies[i].CreationTimestamp.Before(policies[j].CreationTimestamp) + } + if policies[i].Namespace != policies[j].Namespace { + return policies[i].Namespace < policies[j].Namespace + } + return policies[i].Name < policies[j].Name + }) + for _, cfg := range policies { + spec, ok := cfg.Spec.(*networking.CircuitBreakerPolicy) + if !ok || spec == nil { + continue + } + policy := dxgateCircuitBreakerFromPolicy(spec) + if policy.CircuitBreaker == nil && policy.Outlier == nil { + continue + } + for _, target := range spec.TargetRefs { + if !isCircuitBreakerServiceTarget(target) { + continue + } + key := namespacedServiceKey(cfg.Namespace, target.Name) + if _, found := out[key]; !found { + out[key] = policy + } + } + } + return out +} + +func isCircuitBreakerServiceTarget(target *networking.PolicyTargetReference) bool { + if target == nil || target.Name == "" || target.SectionName != "" { + return false + } + group := strings.TrimSpace(target.Group) + kind := strings.TrimSpace(target.Kind) + return (group == "" || group == "core") && strings.EqualFold(kind, "Service") +} + +func dxgateCircuitBreakerFromPolicy(policy *networking.CircuitBreakerPolicy) dxgateBackendCircuitBreaker { + out := dxgateBackendCircuitBreaker{} + if cp := policy.GetConnectionPool(); cp != nil { + cb := &dxgateCircuitBreaker{ + MaxConnections: positiveInt32(cp.GetMaxConnections()), + HTTP1MaxPendingRequests: positiveInt32(cp.GetHttp1MaxPendingRequests()), + HTTP2MaxRequests: positiveInt32(cp.GetHttp2MaxRequests()), + MaxRequestsPerConnection: positiveInt32(cp.GetMaxRequestsPerConnection()), + MaxRetries: positiveInt32(cp.GetMaxRetries()), + } + if cb.MaxConnections > 0 || cb.HTTP1MaxPendingRequests > 0 || cb.HTTP2MaxRequests > 0 || cb.MaxRequestsPerConnection > 0 || cb.MaxRetries > 0 { + out.CircuitBreaker = cb + } + } + if od := policy.GetOutlierDetection(); od != nil { + outlier := &dxgateOutlierDetection{ + Consecutive5xxErrors: uint32Value(od.GetConsecutive_5XxErrors()), + Interval: durationString(od.GetInterval()), + BaseEjectionTime: durationString(od.GetBaseEjectionTime()), + MaxEjectionPercent: positiveInt32(od.GetMaxEjectionPercent()), + MinHealthPercent: positiveInt32(od.GetMinHealthPercent()), + } + if outlier.Consecutive5xxErrors > 0 || outlier.Interval != "" || outlier.BaseEjectionTime != "" || outlier.MaxEjectionPercent > 0 || outlier.MinHealthPercent > 0 { + out.Outlier = outlier + } + } + return out +} + +func positiveInt32(value int32) int32 { + if value <= 0 { + return 0 + } + return value +} + +func uint32Value(value *wrapperspb.UInt32Value) uint32 { + if value == nil { + return 0 + } + return value.GetValue() +} + +func durationString(value *durationpb.Duration) string { + if value == nil { + return "" + } + return value.AsDuration().String() +} + +func namespacedServiceKey(namespace, name string) string { + return namespace + "/" + name +} + func dxgateRouteDomains(gw gateway.Gateway, hr *gateway.HTTPRoute) []string { domains := map[string]struct{}{} for _, hostname := range hr.Spec.Hostnames { @@ -1093,7 +1262,7 @@ func gatewayServiceTargetPort(gw gateway.Gateway) intstr.IntOrString { return intstr.FromInt(port) } if gw.Annotations[eastWestGatewayAnnotation] == "true" { - return intstr.FromInt(inject.ProxylessXServerPort) + return intstr.FromInt(inject.ProxylessGRPCInboundPort) } return intstr.FromString("http") } diff --git a/dubbod/discovery/pkg/config/kube/gateway/deployment_controller_test.go b/dubbod/discovery/pkg/config/kube/gateway/deployment_controller_test.go index 31f8bc1a0..1764c6e65 100644 --- a/dubbod/discovery/pkg/config/kube/gateway/deployment_controller_test.go +++ b/dubbod/discovery/pkg/config/kube/gateway/deployment_controller_test.go @@ -22,9 +22,15 @@ import ( "path/filepath" "strings" "testing" + "time" + "github.com/apache/dubbo-kubernetes/pkg/config" + "github.com/apache/dubbo-kubernetes/pkg/config/schema/gvk" "github.com/apache/dubbo-kubernetes/pkg/kube/inject" "github.com/google/go-cmp/cmp" + networking "github.com/kdubbo/api/networking/v1alpha3" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/wrapperspb" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -116,7 +122,7 @@ func TestBuildDxgateRuntimeConfigFromHTTPRoute(t *testing.T) { }, } - raw, hash, err := buildDxgateRuntimeConfig(gw, []*gatewayv1.HTTPRoute{route}, "cluster.local") + raw, hash, err := buildDxgateRuntimeConfig(gw, []*gatewayv1.HTTPRoute{route}, nil, "cluster.local") if err != nil { t.Fatal(err) } @@ -153,6 +159,94 @@ func TestBuildDxgateRuntimeConfigFromHTTPRoute(t *testing.T) { } } +func TestBuildDxgateRuntimeConfigAppliesCircuitBreakerPolicy(t *testing.T) { + backendPort := gatewayv1.PortNumber(9080) + gw := gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{Name: "public", Namespace: "app", ResourceVersion: "10"}, + Spec: gatewayv1.GatewaySpec{ + GatewayClassName: "dubbo", + Listeners: []gatewayv1.Listener{ + {Name: "http", Protocol: gatewayv1.HTTPProtocolType, Port: 80}, + }, + }, + } + route := &gatewayv1.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{Name: "reviews", Namespace: "app", ResourceVersion: "20"}, + Spec: gatewayv1.HTTPRouteSpec{ + CommonRouteSpec: gatewayv1.CommonRouteSpec{ + ParentRefs: []gatewayv1.ParentReference{{Name: "public"}}, + }, + Rules: []gatewayv1.HTTPRouteRule{ + { + BackendRefs: []gatewayv1.HTTPBackendRef{ + { + BackendRef: gatewayv1.BackendRef{ + BackendObjectReference: gatewayv1.BackendObjectReference{ + Name: "reviews", + Port: &backendPort, + }, + }, + }, + }, + }, + }, + }, + } + policies := []config.Config{ + { + Meta: config.Meta{ + GroupVersionKind: gvk.CircuitBreakerPolicy, + Name: "reviews-circuit-breaker", + Namespace: "app", + CreationTimestamp: time.Unix(1, 0), + }, + Spec: &networking.CircuitBreakerPolicy{ + TargetRefs: []*networking.PolicyTargetReference{ + {Kind: "Service", Name: "reviews"}, + }, + ConnectionPool: &networking.ConnectionPoolSettings{ + MaxConnections: 1, + Http1MaxPendingRequests: 1, + Http2MaxRequests: 2, + MaxRetries: 3, + }, + OutlierDetection: &networking.OutlierDetection{ + Consecutive_5XxErrors: wrapperspb.UInt32(1), + Interval: durationpb.New(time.Second), + BaseEjectionTime: durationpb.New(30 * time.Second), + MaxEjectionPercent: 100, + }, + }, + }, + } + + raw, _, err := buildDxgateRuntimeConfig(gw, []*gatewayv1.HTTPRoute{route}, policies, "cluster.local") + if err != nil { + t.Fatal(err) + } + var cfg dxgateRuntimeConfig + if err := yaml.Unmarshal([]byte(raw), &cfg); err != nil { + t.Fatal(err) + } + if len(cfg.Clusters) != 1 { + t.Fatalf("clusters = %d, want 1", len(cfg.Clusters)) + } + cb := cfg.Clusters[0].CircuitBreaker + if cb == nil { + t.Fatal("cluster circuit_breaker is nil") + } + if cb.MaxConnections != 1 || cb.HTTP1MaxPendingRequests != 1 || cb.HTTP2MaxRequests != 2 || cb.MaxRetries != 3 { + t.Fatalf("unexpected circuit breaker: %#v", cb) + } + outlier := cfg.Clusters[0].Outlier + if outlier == nil { + t.Fatal("cluster outlier_detection is nil") + } + if outlier.Consecutive5xxErrors != 1 || outlier.Interval != "1s" || outlier.BaseEjectionTime != "30s" || outlier.MaxEjectionPercent != 100 { + t.Fatalf("unexpected outlier detection: %#v", outlier) + } +} + func TestBuildDxgateRuntimeConfigFiltersUnattachedHTTPRoutes(t *testing.T) { backendPort := gatewayv1.PortNumber(8080) gw := gatewayv1.Gateway{ @@ -184,7 +278,7 @@ func TestBuildDxgateRuntimeConfigFiltersUnattachedHTTPRoutes(t *testing.T) { }, } - raw, _, err := buildDxgateRuntimeConfig(gw, []*gatewayv1.HTTPRoute{route}, "cluster.local") + raw, _, err := buildDxgateRuntimeConfig(gw, []*gatewayv1.HTTPRoute{route}, nil, "cluster.local") if err != nil { t.Fatal(err) } @@ -275,7 +369,7 @@ func TestExtractServicePortsTargetsDxgateContainerPorts(t *testing.T) { } } -func TestExtractServicePortsTargetsXServerForEastWestGateway(t *testing.T) { +func TestExtractServicePortsTargetsGRPCInboundForEastWestGateway(t *testing.T) { gw := gatewayv1.Gateway{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ diff --git a/dubbod/discovery/pkg/xds/endpoints/endpoint_builder.go b/dubbod/discovery/pkg/xds/endpoints/endpoint_builder.go index 7c080f73e..a6d78d2cc 100644 --- a/dubbod/discovery/pkg/xds/endpoints/endpoint_builder.go +++ b/dubbod/discovery/pkg/xds/endpoints/endpoint_builder.go @@ -321,13 +321,13 @@ func (b *EndpointBuilder) eastWestGatewayForCluster(endpointCluster cluster.ID, } func (b *EndpointBuilder) endpointPort(ep *model.DubboEndpoint) uint32 { - if b.useXServerEndpointPort() { - return inject.ProxylessXServerPort + if b.useGRPCInboundEndpointPort() { + return inject.ProxylessGRPCInboundPort } return ep.EndpointPort } -func (b *EndpointBuilder) useXServerEndpointPort() bool { +func (b *EndpointBuilder) useGRPCInboundEndpointPort() bool { if b == nil || b.proxy == nil || !b.proxy.IsProxylessGrpc() || b.push == nil || b.service == nil { return false } diff --git a/go.mod b/go.mod index 842f61876..6d0f075ab 100644 --- a/go.mod +++ b/go.mod @@ -254,6 +254,9 @@ replace ( github.com/docker/distribution => github.com/docker/distribution v2.8.2+incompatible github.com/docker/docker => github.com/docker/docker v23.0.1+incompatible github.com/google/go-containerregistry => github.com/google/go-containerregistry v0.20.2 + github.com/kdubbo/api => ../api + github.com/kdubbo/client-go => ../client-go + github.com/kdubbo/xds-api => ../xds-api github.com/moby/buildkit => github.com/moby/buildkit v0.10.6 github.com/moby/dockerfile => github.com/moby/dockerfile v1.4.1 ) diff --git a/manifests/charts/base/files/crd-all.gen.yaml b/manifests/charts/base/files/crd-all.gen.yaml index 29a1408d6..43b20ca46 100644 --- a/manifests/charts/base/files/crd-all.gen.yaml +++ b/manifests/charts/base/files/crd-all.gen.yaml @@ -1,6 +1,157 @@ # DO NOT EDIT - Generated by Cue OpenAPI generator based on Dubbo APIs. apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + "helm.sh/resource-policy": keep + labels: + app: dubbo + chart: dubbo + dubbo: networking + heritage: Tiller + release: dubbo + name: circuitbreakerpolicies.networking.dubbo.apache.org +spec: + group: networking.dubbo.apache.org + names: + categories: + - dubbo + - networking + kind: CircuitBreakerPolicy + listKind: CircuitBreakerPolicyList + plural: circuitbreakerpolicies + shortNames: + - cbp + singular: circuitbreakerpolicy + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Gateway API targets. + jsonPath: .spec.targetRefs[*].name + name: Targets + type: string + - description: CreationTimestamp is a timestamp representing the server time when + this object was created. + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha3 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Gateway API policy attachment for backend circuit breaking. + See more details at: ' + properties: + connectionPool: + description: Connection pool limits. + properties: + http1MaxPendingRequests: + description: Maximum queued requests waiting for an upstream connection. + format: int32 + type: integer + http2MaxRequests: + description: Maximum concurrent requests for HTTP/2 or gRPC upstreams. + format: int32 + type: integer + maxConnections: + description: Maximum concurrent upstream connections. + format: int32 + type: integer + maxRequestsPerConnection: + description: Maximum requests before a connection is drained. + format: int32 + type: integer + maxRetries: + description: Maximum retries allowed to the backend. + format: int32 + type: integer + type: object + outlierDetection: + description: Outlier detection controls passive ejection of failing + endpoints. + properties: + baseEjectionTime: + description: Minimum ejection time for an unhealthy endpoint. + type: string + x-kubernetes-validations: + - message: must be a valid duration greater than 1ms + rule: duration(self) >= duration('1ms') + consecutive5xxErrors: + description: Number of consecutive 5xx responses before an endpoint + is considered an outlier. + maximum: 4294967295 + minimum: 0 + nullable: true + type: integer + interval: + description: Time between outlier detection sweeps. + type: string + x-kubernetes-validations: + - message: must be a valid duration greater than 1ms + rule: duration(self) >= duration('1ms') + maxEjectionPercent: + description: Maximum percentage of endpoints that can be ejected. + format: int32 + type: integer + minHealthPercent: + description: Minimum healthy percentage before outlier detection + is disabled. + format: int32 + type: integer + type: object + targetRefs: + description: Gateway API policy targets. + items: + properties: + group: + description: API group of the target. + type: string + kind: + description: Kind of the target. + type: string + name: + description: Name of the target object. + type: string + sectionName: + description: Optional section name for future per-port attachment. + type: string + required: + - kind + - name + type: object + type: array + required: + - targetRefs + type: object + status: + properties: + conditions: + items: + properties: + observedGeneration: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + reason: + type: string + status: + type: string + type: + type: string + type: object + type: array + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: "helm.sh/resource-policy": keep diff --git a/manifests/charts/dubbod/files/grpc-engine.yaml b/manifests/charts/dubbod/files/grpc-engine.yaml index 997497cda..0fd76a830 100644 --- a/manifests/charts/dubbod/files/grpc-engine.yaml +++ b/manifests/charts/dubbod/files/grpc-engine.yaml @@ -75,7 +75,7 @@ spec: fieldPath: status.hostIP {{- if and (gt (len .Spec.Containers) 0) (contains "kdubbo/dubbod" ((index .Spec.Containers 0).Image | default "")) }} args: - - xclient + - grpc-outbound - --watch {{- end }} volumeMounts: @@ -86,17 +86,17 @@ spec: {{- if and (gt (len .Spec.Containers) 0) (gt (len ((index .Spec.Containers 0).Ports)) 0) }} {{- $appPort = int ((index ((index .Spec.Containers 0).Ports) 0).ContainerPort) }} {{- end }} - - name: dubbo-xserver + - name: dubbo-grpc-inbound image: {{ .ProxyImage | quote }} imagePullPolicy: IfNotPresent args: - - xserver + - grpc-inbound - --listen - :15080 - --upstream - 127.0.0.1:{{ $appPort }} ports: - - name: xserver + - name: grpc-inbound containerPort: 15080 protocol: TCP volumeMounts: diff --git a/manifests/charts/dubbod/templates/clusterrole.yaml b/manifests/charts/dubbod/templates/clusterrole.yaml index 7b365dfac..5e5250701 100644 --- a/manifests/charts/dubbod/templates/clusterrole.yaml +++ b/manifests/charts/dubbod/templates/clusterrole.yaml @@ -21,6 +21,9 @@ rules: - apiGroups: [ "security.dubbo.apache.org" ] verbs: [ "get", "watch", "list" ] resources: [ "*" ] + - apiGroups: [ "networking.dubbo.apache.org" ] + verbs: [ "get", "watch", "list" ] + resources: [ "*" ] - apiGroups: ["admissionregistration.k8s.io"] resources: ["mutatingwebhookconfigurations"] verbs: ["get", "list", "watch", "update", "patch"] diff --git a/manifests/charts/dubbod/templates/cni-daemonset.yaml b/manifests/charts/dubbod/templates/cni-daemonset.yaml index a4e9a7778..fd8986161 100644 --- a/manifests/charts/dubbod/templates/cni-daemonset.yaml +++ b/manifests/charts/dubbod/templates/cni-daemonset.yaml @@ -32,7 +32,7 @@ {{- $binDir := coalesce $cni.binDir $defaultCNI.binDir "/opt/cni/bin" }} {{- $confDir := coalesce $cni.confDir $defaultCNI.confDir "/etc/cni/net.d" }} {{- $stateDir := coalesce $cni.stateDir $defaultCNI.stateDir "/var/run/dubbo-cni" }} -{{- $xserverPort := int (coalesce $cni.xserverPort $defaultCNI.xserverPort 15080) }} +{{- $grpcInboundPort := int (coalesce $cni.grpcInboundPort $defaultCNI.grpcInboundPort 15080) }} {{- $managedLabel := coalesce $cni.managedLabel $defaultCNI.managedLabel "proxyless.dubbo.apache.org/managed" }} {{- $managedLabelValue := coalesce $cni.managedLabelValue $defaultCNI.managedLabelValue "true" }} {{- $iptablesPath := coalesce $cni.iptablesPath $defaultCNI.iptablesPath "iptables" }} @@ -76,7 +76,7 @@ spec: - --kubeconfig={{ printf "%s/dubbo-cni-kubeconfig" $confDir }} - --token-file={{ printf "%s/token" $stateDir }} - --ca-file={{ printf "%s/ca.crt" $stateDir }} - - --xserver-port={{ $xserverPort }} + - --grpc-inbound-port={{ $grpcInboundPort }} - --managed-label={{ $managedLabel }} - --managed-label-value={{ $managedLabelValue }} - --iptables-path={{ $iptablesPath }} diff --git a/manifests/charts/dubbod/templates/configmap-values.yaml b/manifests/charts/dubbod/templates/configmap-values.yaml index c740eea3d..f25278d61 100644 --- a/manifests/charts/dubbod/templates/configmap-values.yaml +++ b/manifests/charts/dubbod/templates/configmap-values.yaml @@ -58,7 +58,7 @@ data: binDir: {{ coalesce $cni.binDir $defaultCNI.binDir "/opt/cni/bin" | quote }} confDir: {{ coalesce $cni.confDir $defaultCNI.confDir "/etc/cni/net.d" | quote }} stateDir: {{ coalesce $cni.stateDir $defaultCNI.stateDir "/var/run/dubbo-cni" | quote }} - xserverPort: {{ int (coalesce $cni.xserverPort $defaultCNI.xserverPort 15080) }} + grpcInboundPort: {{ int (coalesce $cni.grpcInboundPort $defaultCNI.grpcInboundPort 15080) }} managedLabel: {{ coalesce $cni.managedLabel $defaultCNI.managedLabel "proxyless.dubbo.apache.org/managed" | quote }} managedLabelValue: {{ coalesce $cni.managedLabelValue $defaultCNI.managedLabelValue "true" | quote }} iptablesPath: {{ coalesce $cni.iptablesPath $defaultCNI.iptablesPath "iptables" | quote }} diff --git a/manifests/charts/dubbod/values.yaml b/manifests/charts/dubbod/values.yaml index 78b4c1cff..765a487e0 100644 --- a/manifests/charts/dubbod/values.yaml +++ b/manifests/charts/dubbod/values.yaml @@ -27,7 +27,7 @@ _internal_default_values_not_set: binDir: "/opt/cni/bin" confDir: "/etc/cni/net.d" stateDir: "/var/run/dubbo-cni" - xserverPort: 15080 + grpcInboundPort: 15080 managedLabel: "proxyless.dubbo.apache.org/managed" managedLabelValue: "true" iptablesPath: "iptables" diff --git a/operator/pkg/apis/proto/values_types.proto b/operator/pkg/apis/proto/values_types.proto index 02ff1bb22..29b596a0a 100644 --- a/operator/pkg/apis/proto/values_types.proto +++ b/operator/pkg/apis/proto/values_types.proto @@ -49,7 +49,7 @@ message MeshCNIConfig { string stateDir = 5; - int64 xserverPort = 6; + int64 grpcInboundPort = 6; string managedLabel = 7; diff --git a/operator/pkg/apis/values_types.pb.go b/operator/pkg/apis/values_types.pb.go index 0ad1cf1e9..9db76c533 100644 --- a/operator/pkg/apis/values_types.pb.go +++ b/operator/pkg/apis/values_types.pb.go @@ -185,7 +185,7 @@ type MeshCNIConfig struct { BinDir string `protobuf:"bytes,3,opt,name=binDir,proto3" json:"binDir,omitempty"` ConfDir string `protobuf:"bytes,4,opt,name=confDir,proto3" json:"confDir,omitempty"` StateDir string `protobuf:"bytes,5,opt,name=stateDir,proto3" json:"stateDir,omitempty"` - XserverPort int64 `protobuf:"varint,6,opt,name=xserverPort,proto3" json:"xserverPort,omitempty"` + GrpcInboundPort int64 `protobuf:"varint,6,opt,name=grpcInboundPort,proto3" json:"grpcInboundPort,omitempty"` ManagedLabel string `protobuf:"bytes,7,opt,name=managedLabel,proto3" json:"managedLabel,omitempty"` ManagedLabelValue string `protobuf:"bytes,8,opt,name=managedLabelValue,proto3" json:"managedLabelValue,omitempty"` IptablesPath string `protobuf:"bytes,9,opt,name=iptablesPath,proto3" json:"iptablesPath,omitempty"` @@ -260,9 +260,9 @@ func (x *MeshCNIConfig) GetStateDir() string { return "" } -func (x *MeshCNIConfig) GetXserverPort() int64 { +func (x *MeshCNIConfig) GetGrpcInboundPort() int64 { if x != nil { - return x.XserverPort + return x.GrpcInboundPort } return 0 } @@ -808,14 +808,14 @@ const file_values_types_proto_rawDesc = "" + "\vProxyConfig\x12$\n" + "\rclusterDomain\x18\x01 \x01(\tR\rclusterDomain\"K\n" + "\x0fProxylessConfig\x128\n" + - "\x03cni\x18\x01 \x01(\v2&.dubbo.operator.v1alpha1.MeshCNIConfigR\x03cni\"\xed\x02\n" + + "\x03cni\x18\x01 \x01(\v2&.dubbo.operator.v1alpha1.MeshCNIConfigR\x03cni\"\xf5\x02\n" + "\rMeshCNIConfig\x12\x18\n" + "\aenabled\x18\x01 \x01(\bR\aenabled\x12\x14\n" + "\x05image\x18\x02 \x01(\tR\x05image\x12\x16\n" + "\x06binDir\x18\x03 \x01(\tR\x06binDir\x12\x18\n" + "\aconfDir\x18\x04 \x01(\tR\aconfDir\x12\x1a\n" + - "\bstateDir\x18\x05 \x01(\tR\bstateDir\x12 \n" + - "\vxserverPort\x18\x06 \x01(\x03R\vxserverPort\x12\"\n" + + "\bstateDir\x18\x05 \x01(\tR\bstateDir\x12(\n" + + "\x0fgrpcInboundPort\x18\x06 \x01(\x03R\x0fgrpcInboundPort\x12\"\n" + "\fmanagedLabel\x18\a \x01(\tR\fmanagedLabel\x12,\n" + "\x11managedLabelValue\x18\b \x01(\tR\x11managedLabelValue\x12\"\n" + "\fiptablesPath\x18\t \x01(\tR\fiptablesPath\x12\x1c\n" + diff --git a/operator/pkg/render/manifest_test.go b/operator/pkg/render/manifest_test.go index 09b915545..55db157bb 100644 --- a/operator/pkg/render/manifest_test.go +++ b/operator/pkg/render/manifest_test.go @@ -214,7 +214,7 @@ func TestGenerateManifestProxylessCNIIsGlobalByDefault(t *testing.T) { "values.global.proxyless.cni.binDir=/var/lib/cni/bin", "values.global.proxyless.cni.confDir=/var/lib/cni/net.d", "values.global.proxyless.cni.stateDir=/run/dubbo-cni", - "values.global.proxyless.cni.xserverPort=16080", + "values.global.proxyless.cni.grpcInboundPort=16080", "values.global.proxyless.cni.ipsetPath=/usr/sbin/ipset", "values.global.proxyless.cni.refreshInterval=30s", }, nil, nil) @@ -239,7 +239,7 @@ func TestGenerateManifestProxylessCNIIsGlobalByDefault(t *testing.T) { "--bin-dir=/var/lib/cni/bin", "--conf-dir=/var/lib/cni/net.d", "--state-dir=/run/dubbo-cni", - "--xserver-port=16080", + "--grpc-inbound-port=16080", "--ipset-path=/usr/sbin/ipset", "--refresh-interval=30s", } { diff --git a/pkg/cni/config.go b/pkg/cni/config.go index 82620bdb3..669d0ac87 100644 --- a/pkg/cni/config.go +++ b/pkg/cni/config.go @@ -42,7 +42,7 @@ type NetConf struct { KubeConfig string `json:"kubeconfig,omitempty"` ManagedLabel string `json:"managedLabel,omitempty"` ManagedLabelValue string `json:"managedLabelValue,omitempty"` - XServerPort int `json:"xserverPort,omitempty"` + GRPCInboundPort int `json:"grpcInboundPort,omitempty"` StateDir string `json:"stateDir,omitempty"` IPTablesPath string `json:"iptablesPath,omitempty"` IPSetPath string `json:"ipsetPath,omitempty"` @@ -69,8 +69,8 @@ func ParseNetConf(data []byte) (NetConf, error) { if conf.ManagedLabelValue == "" { conf.ManagedLabelValue = inject.ProxylessManagedLabelValue } - if conf.XServerPort == 0 { - conf.XServerPort = inject.ProxylessXServerPort + if conf.GRPCInboundPort == 0 { + conf.GRPCInboundPort = inject.ProxylessGRPCInboundPort } if conf.IPTablesPath == "" { conf.IPTablesPath = defaultIPTablesPath diff --git a/pkg/cni/config_test.go b/pkg/cni/config_test.go index f4454a915..d0773c3df 100644 --- a/pkg/cni/config_test.go +++ b/pkg/cni/config_test.go @@ -34,8 +34,8 @@ func TestParseNetConfDefaults(t *testing.T) { t.Fatalf("managed label = %s/%s, want %s/%s", conf.ManagedLabel, conf.ManagedLabelValue, inject.ProxylessManagedLabel, inject.ProxylessManagedLabelValue) } - if conf.XServerPort != inject.ProxylessXServerPort { - t.Fatalf("xserverPort = %d, want %d", conf.XServerPort, inject.ProxylessXServerPort) + if conf.GRPCInboundPort != inject.ProxylessGRPCInboundPort { + t.Fatalf("grpcInboundPort = %d, want %d", conf.GRPCInboundPort, inject.ProxylessGRPCInboundPort) } if conf.IPTablesPath != defaultIPTablesPath || conf.IPSetPath != defaultIPSetPath { t.Fatalf("iptables/ipset path = %s/%s, want %s/%s", conf.IPTablesPath, conf.IPSetPath, defaultIPTablesPath, defaultIPSetPath) diff --git a/pkg/cni/install.go b/pkg/cni/install.go index df8bdb93c..2d9ef1166 100644 --- a/pkg/cni/install.go +++ b/pkg/cni/install.go @@ -54,7 +54,7 @@ type InstallerOptions struct { ManagedLabelValue string IPTablesPath string IPSetPath string - XServerPort int + GRPCInboundPort int } func DefaultInstallerOptions() InstallerOptions { @@ -68,7 +68,7 @@ func DefaultInstallerOptions() InstallerOptions { ManagedLabelValue: inject.ProxylessManagedLabelValue, IPTablesPath: defaultIPTablesPath, IPSetPath: defaultIPSetPath, - XServerPort: inject.ProxylessXServerPort, + GRPCInboundPort: inject.ProxylessGRPCInboundPort, } } @@ -124,8 +124,8 @@ func (o *InstallerOptions) applyDefaults(requireAPIServer bool) error { if o.IPSetPath == "" { o.IPSetPath = defaults.IPSetPath } - if o.XServerPort == 0 { - o.XServerPort = defaults.XServerPort + if o.GRPCInboundPort == 0 { + o.GRPCInboundPort = defaults.GRPCInboundPort } if requireAPIServer && o.APIServer == "" { return fmt.Errorf("kubernetes API server is required") @@ -391,7 +391,7 @@ func pluginConfig(opts InstallerOptions) map[string]any { "kubernetes": map[string]any{"kubeconfig": opts.KubeConfigPath}, "managedLabel": opts.ManagedLabel, "managedLabelValue": opts.ManagedLabelValue, - "xserverPort": opts.XServerPort, + "grpcInboundPort": opts.GRPCInboundPort, "stateDir": opts.StateDir, "iptablesPath": opts.IPTablesPath, "ipsetPath": opts.IPSetPath, diff --git a/pkg/cni/iptables.go b/pkg/cni/iptables.go index 85f976f80..9829b8b5c 100644 --- a/pkg/cni/iptables.go +++ b/pkg/cni/iptables.go @@ -24,8 +24,8 @@ import ( ) const ( - meshInboundChain = "DUBBO-XSERVER-INBOUND" - meshPodIPSet = "DUBBO-XSERVER-PODS" + meshInboundChain = "DUBBO-GRPC-INBOUND" + meshPodIPSet = "DUBBO-GRPC-INBOUND-PODS" ) type CommandRunner interface { @@ -40,20 +40,20 @@ func (execCommandRunner) Run(ctx context.Context, name string, args ...string) ( } type IPTablesRuleManager struct { - path string - ipsetPath string - xserverPort int - dryRun bool - runner CommandRunner + path string + ipsetPath string + grpcInboundPort int + dryRun bool + runner CommandRunner } func NewIPTablesRuleManager(conf NetConf) *IPTablesRuleManager { return &IPTablesRuleManager{ - path: conf.IPTablesPath, - ipsetPath: conf.IPSetPath, - xserverPort: conf.XServerPort, - dryRun: conf.DryRun, - runner: execCommandRunner{}, + path: conf.IPTablesPath, + ipsetPath: conf.IPSetPath, + grpcInboundPort: conf.GRPCInboundPort, + dryRun: conf.DryRun, + runner: execCommandRunner{}, } } @@ -97,11 +97,11 @@ func (m *IPTablesRuleManager) ensureBase(ctx context.Context) error { return err } } - allowXServer := []string{"-m", "set", "--match-set", meshPodIPSet, "dst", "-p", "tcp", "--dport", fmt.Sprint(m.xserverPort), "-j", "RETURN"} + allowGRPCInbound := []string{"-m", "set", "--match-set", meshPodIPSet, "dst", "-p", "tcp", "--dport", fmt.Sprint(m.grpcInboundPort), "-j", "RETURN"} rejectOtherTCP := []string{"-m", "set", "--match-set", meshPodIPSet, "dst", "-p", "tcp", "-j", "REJECT"} - m.deleteRepeated(ctx, allowXServer...) + m.deleteRepeated(ctx, allowGRPCInbound...) m.deleteRepeated(ctx, rejectOtherTCP...) - if err := m.appendRule(ctx, allowXServer...); err != nil { + if err := m.appendRule(ctx, allowGRPCInbound...); err != nil { return err } return m.appendRule(ctx, rejectOtherTCP...) diff --git a/pkg/cni/iptables_test.go b/pkg/cni/iptables_test.go index 845104f8a..56cc67fcd 100644 --- a/pkg/cni/iptables_test.go +++ b/pkg/cni/iptables_test.go @@ -21,9 +21,9 @@ import ( "testing" ) -func TestIPTablesRuleManagerAddsXServerBoundaryRules(t *testing.T) { +func TestIPTablesRuleManagerAddsGRPCInboundBoundaryRules(t *testing.T) { runner := &recordingRunner{} - conf, err := ParseNetConf([]byte(`{"xserverPort":15080}`)) + conf, err := ParseNetConf([]byte(`{"grpcInboundPort":15080}`)) if err != nil { t.Fatalf("ParseNetConf() failed: %v", err) } @@ -35,13 +35,13 @@ func TestIPTablesRuleManagerAddsXServerBoundaryRules(t *testing.T) { joined := strings.Join(runner.commands, "\n") for _, want := range []string{ - "ipset create DUBBO-XSERVER-PODS hash:ip -exist", - "-N DUBBO-XSERVER-INBOUND", - "-I FORWARD 1 -j DUBBO-XSERVER-INBOUND", - "-I OUTPUT 1 -j DUBBO-XSERVER-INBOUND", - "-A DUBBO-XSERVER-INBOUND -m set --match-set DUBBO-XSERVER-PODS dst -p tcp --dport 15080 -j RETURN", - "-A DUBBO-XSERVER-INBOUND -m set --match-set DUBBO-XSERVER-PODS dst -p tcp -j REJECT", - "ipset add DUBBO-XSERVER-PODS 10.244.0.12 -exist", + "ipset create DUBBO-GRPC-INBOUND-PODS hash:ip -exist", + "-N DUBBO-GRPC-INBOUND", + "-I FORWARD 1 -j DUBBO-GRPC-INBOUND", + "-I OUTPUT 1 -j DUBBO-GRPC-INBOUND", + "-A DUBBO-GRPC-INBOUND -m set --match-set DUBBO-GRPC-INBOUND-PODS dst -p tcp --dport 15080 -j RETURN", + "-A DUBBO-GRPC-INBOUND -m set --match-set DUBBO-GRPC-INBOUND-PODS dst -p tcp -j REJECT", + "ipset add DUBBO-GRPC-INBOUND-PODS 10.244.0.12 -exist", } { if !strings.Contains(joined, want) { t.Fatalf("commands missing %q:\n%s", want, joined) diff --git a/pkg/config/schema/codegen/templates/clients.go.tmpl b/pkg/config/schema/codegen/templates/clients.go.tmpl index 4308f6100..9f51bfd5d 100644 --- a/pkg/config/schema/codegen/templates/clients.go.tmpl +++ b/pkg/config/schema/codegen/templates/clients.go.tmpl @@ -17,6 +17,7 @@ import ( ktypes "github.com/apache/dubbo-kubernetes/pkg/kube/kubetypes" "github.com/apache/dubbo-kubernetes/pkg/util/ptr" + apigithubcomapachedubbokubernetesapinetworkingv1alpha3 "github.com/kdubbo/client-go/pkg/apis/networking/v1alpha3" apigithubcomapachedubbokubernetesapisecurityv1alpha3 "github.com/kdubbo/client-go/pkg/apis/security/v1alpha3" {{- range .Packages}} {{.ImportName}} "{{.PackageName}}" diff --git a/pkg/config/schema/codegen/templates/crdclient.go.tmpl b/pkg/config/schema/codegen/templates/crdclient.go.tmpl index fc2a4c841..4a439f08f 100644 --- a/pkg/config/schema/codegen/templates/crdclient.go.tmpl +++ b/pkg/config/schema/codegen/templates/crdclient.go.tmpl @@ -14,6 +14,7 @@ import ( "github.com/apache/dubbo-kubernetes/pkg/config/schema/gvk" "github.com/apache/dubbo-kubernetes/pkg/kube" + apigithubcomapachedubbokubernetesapinetworkingv1alpha3 "github.com/kdubbo/client-go/pkg/apis/networking/v1alpha3" apigithubcomapachedubbokubernetesapisecurityv1alpha3 "github.com/kdubbo/client-go/pkg/apis/security/v1alpha3" {{- range .Packages}} {{.ImportName}} "{{.PackageName}}" diff --git a/pkg/config/schema/codegen/templates/types.go.tmpl b/pkg/config/schema/codegen/templates/types.go.tmpl index f064a4206..0dbf5e19c 100644 --- a/pkg/config/schema/codegen/templates/types.go.tmpl +++ b/pkg/config/schema/codegen/templates/types.go.tmpl @@ -8,6 +8,7 @@ import ( {{- range .Packages}} {{.ImportName}} "{{.PackageName}}" {{- end}} + apigithubcomapachedubbokubernetesapinetworkingv1alpha3 "github.com/kdubbo/client-go/pkg/apis/networking/v1alpha3" apigithubcomapachedubbokubernetesapisecurityv1alpha3 "github.com/kdubbo/client-go/pkg/apis/security/v1alpha3" ) diff --git a/pkg/config/schema/collections/collections.gen.go b/pkg/config/schema/collections/collections.gen.go index 3deca42de..60ace61c5 100755 --- a/pkg/config/schema/collections/collections.gen.go +++ b/pkg/config/schema/collections/collections.gen.go @@ -13,6 +13,7 @@ import ( "github.com/apache/dubbo-kubernetes/pkg/config/validation" githubcomkdubboapimeshv1alpha1 "github.com/kdubbo/api/mesh/v1alpha1" githubcomkdubboapimetav1alpha1 "github.com/kdubbo/api/meta/v1alpha1" + githubcomkdubboapinetworkingv1alpha3 "github.com/kdubbo/api/networking/v1alpha3" githubcomkdubboapisecurityv1alpha3 "github.com/kdubbo/api/security/v1alpha3" k8sioapiadmissionregistrationv1 "k8s.io/api/admissionregistration/v1" k8sioapiappsv1 "k8s.io/api/apps/v1" @@ -41,6 +42,21 @@ var ( ValidateProto: validation.EmptyValidate, }.MustBuild() + CircuitBreakerPolicy = resource.Builder{ + Identifier: "CircuitBreakerPolicy", + Group: "networking.dubbo.apache.org", + Kind: "CircuitBreakerPolicy", + Plural: "circuitbreakerpolicies", + Version: "v1alpha3", + Proto: "dubbo.networking.v1alpha3.CircuitBreakerPolicy", StatusProto: "dubbo.meta.v1alpha1.DubboStatus", + ReflectType: reflect.TypeOf(&githubcomkdubboapinetworkingv1alpha3.CircuitBreakerPolicy{}).Elem(), StatusType: reflect.TypeOf(&githubcomkdubboapimetav1alpha1.DubboStatus{}).Elem(), + ProtoPackage: "github.com/kdubbo/api/networking/v1alpha3", StatusPackage: "github.com/kdubbo/api/meta/v1alpha1", + ClusterScoped: false, + Synthetic: false, + Builtin: false, + ValidateProto: validation.EmptyValidate, + }.MustBuild() + ConfigMap = resource.Builder{ Identifier: "ConfigMap", Group: "", @@ -413,6 +429,7 @@ var ( // All contains all collections in the system. All = collection.NewSchemasBuilder(). MustAdd(AuthorizationPolicy). + MustAdd(CircuitBreakerPolicy). MustAdd(ConfigMap). MustAdd(CustomResourceDefinition). MustAdd(DaemonSet). @@ -467,6 +484,7 @@ var ( // Dubbo contains only collections used by Dubbo. Dubbo = collection.NewSchemasBuilder(). MustAdd(AuthorizationPolicy). + MustAdd(CircuitBreakerPolicy). MustAdd(PeerAuthentication). MustAdd(RequestAuthentication). Build() @@ -474,6 +492,7 @@ var ( // dubboGatewayAPI contains only collections used by Dubbo, including the full Gateway API. dubboGatewayAPI = collection.NewSchemasBuilder(). MustAdd(AuthorizationPolicy). + MustAdd(CircuitBreakerPolicy). MustAdd(GatewayClass). MustAdd(HTTPRoute). MustAdd(KubernetesGateway). @@ -484,6 +503,7 @@ var ( // dubboStableGatewayAPI contains only collections used by Dubbo, including beta+ Gateway API. dubboStableGatewayAPI = collection.NewSchemasBuilder(). MustAdd(AuthorizationPolicy). + MustAdd(CircuitBreakerPolicy). MustAdd(GatewayClass). MustAdd(HTTPRoute). MustAdd(KubernetesGateway). diff --git a/pkg/config/schema/gvk/resources.gen.go b/pkg/config/schema/gvk/resources.gen.go index 777f04e27..f1540b623 100755 --- a/pkg/config/schema/gvk/resources.gen.go +++ b/pkg/config/schema/gvk/resources.gen.go @@ -11,6 +11,7 @@ import ( var ( AuthorizationPolicy = config.GroupVersionKind{Group: "security.dubbo.apache.org", Version: "v1alpha3", Kind: "AuthorizationPolicy"} + CircuitBreakerPolicy = config.GroupVersionKind{Group: "networking.dubbo.apache.org", Version: "v1alpha3", Kind: "CircuitBreakerPolicy"} ConfigMap = config.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} CustomResourceDefinition = config.GroupVersionKind{Group: "apiextensions.k8s.io", Version: "v1", Kind: "CustomResourceDefinition"} DaemonSet = config.GroupVersionKind{Group: "apps", Version: "v1", Kind: "DaemonSet"} @@ -45,6 +46,8 @@ func ToGVR(g config.GroupVersionKind) (schema.GroupVersionResource, bool) { switch g { case AuthorizationPolicy: return gvr.AuthorizationPolicy, true + case CircuitBreakerPolicy: + return gvr.CircuitBreakerPolicy, true case ConfigMap: return gvr.ConfigMap, true case CustomResourceDefinition: @@ -108,6 +111,8 @@ func MustToKind(g config.GroupVersionKind) kind.Kind { switch g { case AuthorizationPolicy: return kind.AuthorizationPolicy + case CircuitBreakerPolicy: + return kind.CircuitBreakerPolicy case ConfigMap: return kind.ConfigMap case CustomResourceDefinition: @@ -176,6 +181,8 @@ func FromGVR(g schema.GroupVersionResource) (config.GroupVersionKind, bool) { switch g { case gvr.AuthorizationPolicy: return AuthorizationPolicy, true + case gvr.CircuitBreakerPolicy: + return CircuitBreakerPolicy, true case gvr.ConfigMap: return ConfigMap, true case gvr.CustomResourceDefinition: diff --git a/pkg/config/schema/gvr/resources.gen.go b/pkg/config/schema/gvr/resources.gen.go index 5affdbcc6..52be18a48 100755 --- a/pkg/config/schema/gvr/resources.gen.go +++ b/pkg/config/schema/gvr/resources.gen.go @@ -6,6 +6,7 @@ import "k8s.io/apimachinery/pkg/runtime/schema" var ( AuthorizationPolicy = schema.GroupVersionResource{Group: "security.dubbo.apache.org", Version: "v1alpha3", Resource: "authorizationpolicies"} + CircuitBreakerPolicy = schema.GroupVersionResource{Group: "networking.dubbo.apache.org", Version: "v1alpha3", Resource: "circuitbreakerpolicies"} ConfigMap = schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"} CustomResourceDefinition = schema.GroupVersionResource{Group: "apiextensions.k8s.io", Version: "v1", Resource: "customresourcedefinitions"} DaemonSet = schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "daemonsets"} @@ -39,6 +40,8 @@ func IsClusterScoped(g schema.GroupVersionResource) bool { switch g { case AuthorizationPolicy: return false + case CircuitBreakerPolicy: + return false case ConfigMap: return false case CustomResourceDefinition: diff --git a/pkg/config/schema/kind/resources.gen.go b/pkg/config/schema/kind/resources.gen.go index 4707c01f9..f31e2742d 100755 --- a/pkg/config/schema/kind/resources.gen.go +++ b/pkg/config/schema/kind/resources.gen.go @@ -6,6 +6,7 @@ const ( Unknown Kind = iota Address AuthorizationPolicy + CircuitBreakerPolicy ConfigMap CustomResourceDefinition DNSName @@ -39,6 +40,8 @@ func (k Kind) String() string { return "Address" case AuthorizationPolicy: return "AuthorizationPolicy" + case CircuitBreakerPolicy: + return "CircuitBreakerPolicy" case ConfigMap: return "ConfigMap" case CustomResourceDefinition: @@ -100,6 +103,8 @@ func FromString(s string) Kind { return Address case "AuthorizationPolicy": return AuthorizationPolicy + case "CircuitBreakerPolicy": + return CircuitBreakerPolicy case "ConfigMap": return ConfigMap case "CustomResourceDefinition": diff --git a/pkg/config/schema/kubeclient/resources.gen.go b/pkg/config/schema/kubeclient/resources.gen.go index 23ac843ab..e07946d81 100755 --- a/pkg/config/schema/kubeclient/resources.gen.go +++ b/pkg/config/schema/kubeclient/resources.gen.go @@ -17,6 +17,7 @@ import ( ktypes "github.com/apache/dubbo-kubernetes/pkg/kube/kubetypes" "github.com/apache/dubbo-kubernetes/pkg/util/ptr" + apigithubcomapachedubbokubernetesapinetworkingv1alpha3 "github.com/kdubbo/client-go/pkg/apis/networking/v1alpha3" apigithubcomapachedubbokubernetesapisecurityv1alpha3 "github.com/kdubbo/client-go/pkg/apis/security/v1alpha3" k8sioapiadmissionregistrationv1 "k8s.io/api/admissionregistration/v1" k8sioapiappsv1 "k8s.io/api/apps/v1" @@ -33,6 +34,8 @@ func GetWriteClient[T runtime.Object](c ClientGetter, namespace string) ktypes.W switch any(ptr.Empty[T]()).(type) { case *apigithubcomapachedubbokubernetesapisecurityv1alpha3.AuthorizationPolicy: return c.Dubbo().SecurityV1alpha3().AuthorizationPolicies(namespace).(ktypes.WriteAPI[T]) + case *apigithubcomapachedubbokubernetesapinetworkingv1alpha3.CircuitBreakerPolicy: + return c.Dubbo().NetworkingV1alpha3().CircuitBreakerPolicies(namespace).(ktypes.WriteAPI[T]) case *k8sioapicorev1.ConfigMap: return c.Kube().CoreV1().ConfigMaps(namespace).(ktypes.WriteAPI[T]) case *k8sioapiextensionsapiserverpkgapisapiextensionsv1.CustomResourceDefinition: @@ -88,6 +91,8 @@ func GetClient[T, TL runtime.Object](c ClientGetter, namespace string) ktypes.Re switch any(ptr.Empty[T]()).(type) { case *apigithubcomapachedubbokubernetesapisecurityv1alpha3.AuthorizationPolicy: return c.Dubbo().SecurityV1alpha3().AuthorizationPolicies(namespace).(ktypes.ReadWriteAPI[T, TL]) + case *apigithubcomapachedubbokubernetesapinetworkingv1alpha3.CircuitBreakerPolicy: + return c.Dubbo().NetworkingV1alpha3().CircuitBreakerPolicies(namespace).(ktypes.ReadWriteAPI[T, TL]) case *k8sioapicorev1.ConfigMap: return c.Kube().CoreV1().ConfigMaps(namespace).(ktypes.ReadWriteAPI[T, TL]) case *k8sioapiextensionsapiserverpkgapisapiextensionsv1.CustomResourceDefinition: @@ -143,6 +148,8 @@ func gvrToObject(g schema.GroupVersionResource) runtime.Object { switch g { case gvr.AuthorizationPolicy: return &apigithubcomapachedubbokubernetesapisecurityv1alpha3.AuthorizationPolicy{} + case gvr.CircuitBreakerPolicy: + return &apigithubcomapachedubbokubernetesapinetworkingv1alpha3.CircuitBreakerPolicy{} case gvr.ConfigMap: return &k8sioapicorev1.ConfigMap{} case gvr.CustomResourceDefinition: @@ -206,6 +213,13 @@ func getInformerFiltered(c ClientGetter, opts ktypes.InformerOptions, g schema.G w = func(options metav1.ListOptions) (watch.Interface, error) { return c.Dubbo().SecurityV1alpha3().AuthorizationPolicies(opts.Namespace).Watch(context.Background(), options) } + case gvr.CircuitBreakerPolicy: + l = func(options metav1.ListOptions) (runtime.Object, error) { + return c.Dubbo().NetworkingV1alpha3().CircuitBreakerPolicies(opts.Namespace).List(context.Background(), options) + } + w = func(options metav1.ListOptions) (watch.Interface, error) { + return c.Dubbo().NetworkingV1alpha3().CircuitBreakerPolicies(opts.Namespace).Watch(context.Background(), options) + } case gvr.ConfigMap: l = func(options metav1.ListOptions) (runtime.Object, error) { return c.Kube().CoreV1().ConfigMaps(opts.Namespace).List(context.Background(), options) diff --git a/pkg/config/schema/kubetypes/resources.gen.go b/pkg/config/schema/kubetypes/resources.gen.go index 00b5cac39..910b664f0 100755 --- a/pkg/config/schema/kubetypes/resources.gen.go +++ b/pkg/config/schema/kubetypes/resources.gen.go @@ -6,7 +6,9 @@ import ( "github.com/apache/dubbo-kubernetes/pkg/config" "github.com/apache/dubbo-kubernetes/pkg/config/schema/gvk" githubcomkdubboapimeshv1alpha1 "github.com/kdubbo/api/mesh/v1alpha1" + githubcomkdubboapinetworkingv1alpha3 "github.com/kdubbo/api/networking/v1alpha3" githubcomkdubboapisecurityv1alpha3 "github.com/kdubbo/api/security/v1alpha3" + apigithubcomapachedubbokubernetesapinetworkingv1alpha3 "github.com/kdubbo/client-go/pkg/apis/networking/v1alpha3" apigithubcomapachedubbokubernetesapisecurityv1alpha3 "github.com/kdubbo/client-go/pkg/apis/security/v1alpha3" k8sioapiadmissionregistrationv1 "k8s.io/api/admissionregistration/v1" k8sioapiappsv1 "k8s.io/api/apps/v1" @@ -25,6 +27,10 @@ func getGvk(obj any) (config.GroupVersionKind, bool) { return gvk.AuthorizationPolicy, true case *apigithubcomapachedubbokubernetesapisecurityv1alpha3.AuthorizationPolicy: return gvk.AuthorizationPolicy, true + case *githubcomkdubboapinetworkingv1alpha3.CircuitBreakerPolicy: + return gvk.CircuitBreakerPolicy, true + case *apigithubcomapachedubbokubernetesapinetworkingv1alpha3.CircuitBreakerPolicy: + return gvk.CircuitBreakerPolicy, true case *k8sioapicorev1.ConfigMap: return gvk.ConfigMap, true case *k8sioapiextensionsapiserverpkgapisapiextensionsv1.CustomResourceDefinition: diff --git a/pkg/config/schema/metadata.yaml b/pkg/config/schema/metadata.yaml index 36059afae..c62921b00 100644 --- a/pkg/config/schema/metadata.yaml +++ b/pkg/config/schema/metadata.yaml @@ -201,6 +201,15 @@ resources: statusProto: "dubbo.meta.v1alpha1.DubboStatus" statusProtoPackage: "github.com/kdubbo/api/meta/v1alpha1" + - kind: CircuitBreakerPolicy + plural: "circuitbreakerpolicies" + group: "networking.dubbo.apache.org" + version: "v1alpha3" + proto: "dubbo.networking.v1alpha3.CircuitBreakerPolicy" + protoPackage: "github.com/kdubbo/api/networking/v1alpha3" + statusProto: "dubbo.meta.v1alpha1.DubboStatus" + statusProtoPackage: "github.com/kdubbo/api/meta/v1alpha1" + - kind: "MeshConfig" plural: "meshconfigs" group: "" diff --git a/pkg/kube/inject/proxyless.go b/pkg/kube/inject/proxyless.go index 350041b9f..cb4aa98ff 100644 --- a/pkg/kube/inject/proxyless.go +++ b/pkg/kube/inject/proxyless.go @@ -42,8 +42,8 @@ const ( ProxylessGRPCKeepaliveTimeout = "10s" ProxylessGRPCConfigFileName = "dubbo-grpc-xds.json" ProxylessGRPCConfigPath = ProxylessXDSMountPath + "/" + ProxylessGRPCConfigFileName - ProxylessXServerContainerName = "dubbo-xserver" - ProxylessXServerPort = 15080 + ProxylessGRPCInboundContainerName = "dubbo-grpc-inbound" + ProxylessGRPCInboundPort = 15080 ProxylessManagedLabel = "proxyless.dubbo.apache.org/managed" ProxylessManagedLabelValue = "true" ) diff --git a/pkg/kube/inject/proxyless_test.go b/pkg/kube/inject/proxyless_test.go index 400ea1bb3..e67fe97ec 100644 --- a/pkg/kube/inject/proxyless_test.go +++ b/pkg/kube/inject/proxyless_test.go @@ -84,17 +84,17 @@ func TestInstallerGRPCEngineTemplateInjectsDirectXDSConnection(t *testing.T) { } if len(injectedPod.Spec.Containers) != 2 { - t.Fatalf("template containers = %d, want app overlay plus xserver", len(injectedPod.Spec.Containers)) + t.Fatalf("template containers = %d, want app overlay plus grpc-inbound", len(injectedPod.Spec.Containers)) } if err := postProcessPod(mergedPod, *injectedPod, req); err != nil { t.Fatalf("postProcessPod() failed: %v", err) } if len(mergedPod.Spec.Containers) != 2 { - t.Fatalf("containers = %d, want application container plus xserver", len(mergedPod.Spec.Containers)) + t.Fatalf("containers = %d, want application container plus grpc-inbound", len(mergedPod.Spec.Containers)) } assertDirectXDSConnection(t, mergedPod, "app", ProxylessGRPCSecretNameForMeta(pod.ObjectMeta)) - assertXServerContainer(t, mergedPod) + assertGRPCInboundContainer(t, mergedPod) } func TestInstallerGRPCEngineTemplateUsesGenerateNameForDeploymentPods(t *testing.T) { @@ -154,10 +154,10 @@ func TestInstallerGRPCEngineTemplateUsesGenerateNameForDeploymentPods(t *testing t.Fatalf("postProcessPod() failed: %v", err) } if len(mergedPod.Spec.Containers) != 2 { - t.Fatalf("containers = %d, want original nginx container plus xserver", len(mergedPod.Spec.Containers)) + t.Fatalf("containers = %d, want original nginx container plus grpc-inbound", len(mergedPod.Spec.Containers)) } assertDirectXDSConnection(t, mergedPod, "nginx", ProxylessGRPCSecretNameForMeta(pod.ObjectMeta)) - assertXServerContainer(t, mergedPod) + assertGRPCInboundContainer(t, mergedPod) if got := mergedPod.Spec.Volumes[0].Secret.SecretName; got == ProxylessGRPCSecretName("") { t.Fatalf("secret name = %q, want generateName-based secret", got) } @@ -242,21 +242,21 @@ func assertNoArgs(t *testing.T, pod *corev1.Pod) { } } -func assertXServerContainer(t *testing.T, pod *corev1.Pod) { +func assertGRPCInboundContainer(t *testing.T, pod *corev1.Pod) { t.Helper() - container := FindContainer(ProxylessXServerContainerName, pod.Spec.Containers) + container := FindContainer(ProxylessGRPCInboundContainerName, pod.Spec.Containers) if container == nil { - t.Fatalf("%s container missing", ProxylessXServerContainerName) + t.Fatalf("%s container missing", ProxylessGRPCInboundContainerName) } if container.Image != "kdubbo/dubbod:debug" { - t.Fatalf("xserver image = %q, want kdubbo/dubbod:debug", container.Image) + t.Fatalf("grpc-inbound image = %q, want kdubbo/dubbod:debug", container.Image) } - wantArgs := []string{"xserver", "--listen", ":15080", "--upstream", "127.0.0.1:80"} + wantArgs := []string{"grpc-inbound", "--listen", ":15080", "--upstream", "127.0.0.1:80"} if strings.Join(container.Args, ",") != strings.Join(wantArgs, ",") { - t.Fatalf("xserver args = %v, want %v", container.Args, wantArgs) + t.Fatalf("grpc-inbound args = %v, want %v", container.Args, wantArgs) } if !hasMount(container.VolumeMounts, ProxylessXDSVolumeName, ProxylessXDSMountPath, true) { - t.Fatalf("xserver proxyless xds mount missing") + t.Fatalf("grpc-inbound proxyless xds mount missing") } } @@ -445,7 +445,7 @@ func TestInstallerGRPCEngineTemplateConfiguresXDSClientForDubbodImage(t *testing } container := mergedPod.Spec.Containers[0] - wantArgs := []string{"xclient", "--watch"} + wantArgs := []string{"grpc-outbound", "--watch"} if strings.Join(container.Args, ",") != strings.Join(wantArgs, ",") { t.Fatalf("args = %v, want %v", container.Args, wantArgs) } diff --git a/pkg/kube/inject/service_test.go b/pkg/kube/inject/service_test.go index 5ed11441f..6e36c4916 100644 --- a/pkg/kube/inject/service_test.go +++ b/pkg/kube/inject/service_test.go @@ -28,7 +28,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) -func TestRewriteProxylessServiceTargetPortsRoutesTCPServiceToXServer(t *testing.T) { +func TestRewriteProxylessServiceTargetPortsRoutesTCPServiceToGRPCInbound(t *testing.T) { svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{Name: "nginx", Namespace: "app"}, Spec: corev1.ServiceSpec{ @@ -44,8 +44,8 @@ func TestRewriteProxylessServiceTargetPortsRoutesTCPServiceToXServer(t *testing. t.Fatalf("rewriteProxylessServiceTargetPorts() = false, want true") } for _, port := range svc.Spec.Ports { - if got := port.TargetPort.IntVal; got != ProxylessXServerPort { - t.Fatalf("port %s targetPort = %d, want %d", port.Name, got, ProxylessXServerPort) + if got := port.TargetPort.IntVal; got != ProxylessGRPCInboundPort { + t.Fatalf("port %s targetPort = %d, want %d", port.Name, got, ProxylessGRPCInboundPort) } } } @@ -110,7 +110,7 @@ func TestRewriteProxylessServiceTargetPortsSkipsNonTCPPorts(t *testing.T) { } } -func TestInjectServiceCreatesXServerTargetPortPatch(t *testing.T) { +func TestInjectServiceCreatesGRPCInboundTargetPortPatch(t *testing.T) { svc := corev1.Service{ ObjectMeta: metav1.ObjectMeta{Name: "nginx", Namespace: "app"}, Spec: corev1.ServiceSpec{ diff --git a/pkg/kube/inject/webhook.go b/pkg/kube/inject/webhook.go index 08f6fd338..8804891ab 100644 --- a/pkg/kube/inject/webhook.go +++ b/pkg/kube/inject/webhook.go @@ -700,7 +700,7 @@ func reorderPod(pod *corev1.Pod, req InjectionParameters) error { // Proxy container should be last to ensure `kubectl exec` and similar commands // continue to default to the user's container pod.Spec.Containers = modifyContainers(pod.Spec.Containers, ProxyContainerName, MoveLast) - pod.Spec.Containers = modifyContainers(pod.Spec.Containers, ProxylessXServerContainerName, MoveLast) + pod.Spec.Containers = modifyContainers(pod.Spec.Containers, ProxylessGRPCInboundContainerName, MoveLast) return nil } @@ -739,10 +739,10 @@ func rewriteProxylessServiceTargetPorts(svc *corev1.Service) bool { if port.Protocol != "" && port.Protocol != corev1.ProtocolTCP { continue } - if port.TargetPort.Type == intstr.Int && port.TargetPort.IntVal == ProxylessXServerPort { + if port.TargetPort.Type == intstr.Int && port.TargetPort.IntVal == ProxylessGRPCInboundPort { continue } - port.TargetPort = intstr.FromInt(ProxylessXServerPort) + port.TargetPort = intstr.FromInt(ProxylessGRPCInboundPort) changed = true } return changed