feat(health): add /healthz and /readyz endpoints

cmd/ratify-gatekeeper-provider/main.go

@@ -18,14 +18,20 @@ package main
 import (
 	"errors"
 	"flag"
+	"fmt"
 	"time"
 
+	"github.com/notaryproject/ratify/v2/internal/healthprobe"
 	"github.com/notaryproject/ratify/v2/internal/httpserver"
 	"github.com/notaryproject/ratify/v2/internal/manager"
 	"github.com/sirupsen/logrus"
 )
 
-var startManagerFunc = manager.StartManager
+var managerReadySignal *manager.ReadySignal
+
+var startManagerFunc = func(certRotatorReady chan struct{}, disableMutation bool, disableCRDManager bool) {
+	manager.StartManager(certRotatorReady, managerReadySignal, disableMutation, disableCRDManager)
+}
 
 // main is the entry point for the Ratify server.
 func main() {
@@ -38,6 +44,7 @@ func main() {
 type options struct {
 	configFilePath       string
 	httpServerAddress    string
+	healthPort           int
 	certFile             string
 	keyFile              string
 	gatekeeperCACertFile string
@@ -52,6 +59,7 @@ func parse() *options {
 	opts := &options{}
 	flag.StringVar(&opts.configFilePath, "config", "", "Path to the Ratify configuration file")
 	flag.StringVar(&opts.httpServerAddress, "address", "", "HTTP server address")
+	flag.IntVar(&opts.healthPort, "health-port", 9090, "Dedicated health probe port")
 	flag.StringVar(&opts.certFile, "cert-file", "", "Path to the TLS certificate file")
 	flag.StringVar(&opts.keyFile, "key-file", "", "Path to the TLS key file")
 	flag.StringVar(&opts.gatekeeperCACertFile, "gatekeeper-ca-cert-file", "", "Path to the Gatekeeper CA certificate file")
@@ -67,13 +75,35 @@ func parse() *options {
 }
 
 func startRatify(opts *options) error {
+	if opts == nil {
+		return errors.New("options are required")
+	}
 	if len(opts.httpServerAddress) == 0 {
 		return errors.New("HTTP server address is required")
 	}
+	if opts.healthPort <= 0 {
+		return errors.New("health port must be greater than zero")
+	}
+
 	var certRotatorReady chan struct{}
 	if !opts.disableCertRotation {
 		certRotatorReady = make(chan struct{})
 	}
+
+	healthRegistry := healthprobe.NewRegistry()
+	managerReadySignal = manager.NewReadySignal()
+	defer func() {
+		managerReadySignal = nil
+	}()
+	if err := healthRegistry.RegisterReadiness(managerReadySignal.Checker()); err != nil {
+		return fmt.Errorf("failed to register manager readiness checker: %w", err)
+	}
+
+	healthServer, err := healthprobe.NewServer(fmt.Sprintf(":%d", opts.healthPort), healthRegistry)
+	if err != nil {
+		return fmt.Errorf("failed to create health probe server: %w", err)
+	}
+
 	serverOpts := &httpserver.ServerOptions{
 		HTTPServerAddress:    opts.httpServerAddress,
 		CertFile:             opts.certFile,
@@ -84,8 +114,14 @@ func startRatify(opts *options) error {
 		DisableMutation:      opts.disableMutation,
 		DisableCRDManager:    opts.disableCRDManager,
 		CertRotatorReady:     certRotatorReady,
+		HealthRegistry:       healthRegistry,
 	}
 
+	go func() {
+		if err := healthServer.Start(); err != nil {
+			logrus.WithError(err).Fatal("failed to start health probe server")
+		}
+	}()
 	go startManagerFunc(certRotatorReady, serverOpts.DisableMutation, serverOpts.DisableCRDManager)
 	return httpserver.StartServer(serverOpts, opts.configFilePath)
 }

cmd/ratify-gatekeeper-provider/main_test.go

@@ -63,6 +63,7 @@ func TestParse(t *testing.T) {
 			expected: &options{
 				configFilePath:    "config.json",
 				httpServerAddress: ":8080",
+				healthPort:        9090,
 				certFile:          "cert.pem",
 				keyFile:           "key.pem",
 				verifyTimeout:     10 * time.Second,
@@ -76,6 +77,7 @@ func TestParse(t *testing.T) {
 				"-mutate-timeout=10s",
 			},
 			expected: &options{
+				healthPort:    9090,
 				verifyTimeout: 30 * time.Second,
 				mutateTimeout: 10 * time.Second,
 			},
@@ -84,6 +86,7 @@ func TestParse(t *testing.T) {
 			name: "default values",
 			args: []string{},
 			expected: &options{
+				healthPort:    9090,
 				verifyTimeout: 5 * time.Second,
 				mutateTimeout: 2 * time.Second,
 			},
@@ -129,6 +132,7 @@ func TestStartRatify(t *testing.T) {
 			name: "failed to start the server",
 			opts: &options{
 				httpServerAddress:   ":8080",
+				healthPort:          9090,
 				configFilePath:      "config.yaml",
 				certFile:            "cert.pem",
 				disableCertRotation: true,

deployments/ratify-gatekeeper-provider/README.md

@@ -40,6 +40,7 @@ Values marked `# DEPRECATED` in the `values.yaml` as well as **DEPRECATED** in t
 | `image.tag`                               | Image tag                                                                                                                                                                                                                                   | `<INSERT THE LATEST RELEASE TAG>`               |
 | `image.pullPolicy`                        | Image pull policy                                                                                                                                                                                                                           | `IfNotPresent`                                  |
 | `replicaCount`                            | Number of replicas to run                                                                                                                                                                                                                   | `1`                                             |
+| `provider.healthPort`                     | Port exposed for the liveness and readiness health endpoints.                                                                                                                                                                               | `9090`                                          |
 | `notation.scopes`                         | Scopes that Notation verifier is applicable for. See [Notation trust policy](https://github.com/notaryproject/specifications/blob/main/specs/trust-store-trust-policy.md#trust-policy).                 | `[]`                                            |
 | `notation.trustedIdentities`              | List of trusted identities for Notation verifier. See [Notation trust policy](https://github.com/notaryproject/specifications/blob/main/specs/trust-store-trust-policy.md#trust-policy).                | `[]`                                            |
 | `notation.certs`                          | List of trusted root certificates for Notation verifier.                                                                                                                                              | `[]`                                            |

deployments/ratify-gatekeeper-provider/templates/deployment.yaml

@@ -40,6 +40,8 @@ spec:
           args:
             - "--address"
             - ":6001"
+            - "--health-port"
+            - {{ .Values.provider.healthPort | quote }}
             - "--config"
             - "/usr/local/config.json"
             {{- if .Values.provider.timeout.validationTimeoutSeconds }}
@@ -65,7 +67,22 @@ spec:
             - "--gatekeeper-ca-cert-file=/usr/local/tls/client-ca/ca.crt"
             {{- end }}
           ports:
-            - containerPort: 6001
+            - name: https
+              containerPort: 6001
+            - name: health
+              containerPort: {{ .Values.provider.healthPort }}
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: {{ .Values.provider.healthPort | default 9090 }}
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: {{ .Values.provider.healthPort | default 9090 }}
+            initialDelaySeconds: 5
+            periodSeconds: 10
           volumeMounts:
             - mountPath: "/usr/local/tls"
               name: tls

deployments/ratify-gatekeeper-provider/templates/service.yaml

@@ -7,7 +7,8 @@ metadata:
 spec:
   type: ClusterIP
   ports:
-    - port: 6001
-      targetPort: 6001
+    - name: https
+      port: 6001
+      targetPort: https
   selector:
-    {{- include "ratify.selectorLabels" . | nindent 4 }}
+    {{- include "ratify.selectorLabels" . | nindent 4 }}

deployments/ratify-gatekeeper-provider/values.yaml

@@ -59,6 +59,7 @@ provider:
     key: "" # key used by ratify (httpserver), please provide your own key
     caCert: "" # CA crt used by ratify (httpserver), please provide your own CA crt
     disableCertRotation: false
+  healthPort: 9090
   disableMutation: false
   disableCRDManager: false
   timeout:

internal/healthprobe/checker.go

@@ -0,0 +1,143 @@
+/*
+Copyright The Ratify Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package healthprobe
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+)
+
+// HealthChecker reports component health to the dedicated health probe server.
+type HealthChecker interface {
+	Name() string
+	Check() error
+}
+
+// CheckerFunc adapts a function into a named health checker.
+type CheckerFunc struct {
+	name string
+	fn   func() error
+}
+
+// NewChecker creates a named health checker.
+func NewChecker(name string, fn func() error) (*CheckerFunc, error) {
+	if name == "" {
+		return nil, errors.New("checker name is required")
+	}
+	if fn == nil {
+		return nil, errors.New("checker function is required")
+	}
+	return &CheckerFunc{name: name, fn: fn}, nil
+}
+
+// MustNewChecker creates a checker and panics if the checker is invalid.
+func MustNewChecker(name string, fn func() error) *CheckerFunc {
+	checker, err := NewChecker(name, fn)
+	if err != nil {
+		panic(err)
+	}
+	return checker
+}
+
+// Name returns the checker name.
+func (c *CheckerFunc) Name() string {
+	if c == nil {
+		return ""
+	}
+	return c.name
+}
+
+// Check runs the checker function.
+func (c *CheckerFunc) Check() error {
+	if c == nil {
+		return errors.New("checker is nil")
+	}
+	if c.fn == nil {
+		return errors.New("checker function is nil")
+	}
+	return c.fn()
+}
+
+// Registry stores liveness and readiness checks for the health probe server.
+type Registry struct {
+	mu        sync.RWMutex
+	liveness  []HealthChecker
+	readiness []HealthChecker
+}
+
+// NewRegistry creates an empty checker registry.
+func NewRegistry() *Registry {
+	return &Registry{}
+}
+
+// RegisterLiveness adds a liveness checker.
+func (r *Registry) RegisterLiveness(checker HealthChecker) error {
+	return r.register(&r.liveness, checker)
+}
+
+// RegisterReadiness adds a readiness checker.
+func (r *Registry) RegisterReadiness(checker HealthChecker) error {
+	return r.register(&r.readiness, checker)
+}
+
+func (r *Registry) register(target *[]HealthChecker, checker HealthChecker) error {
+	if r == nil {
+		return errors.New("registry is nil")
+	}
+	if checker == nil {
+		return errors.New("checker is nil")
+	}
+	if checker.Name() == "" {
+		return errors.New("checker name is required")
+	}
+
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	for _, existing := range *target {
+		if existing.Name() == checker.Name() {
+			return fmt.Errorf("checker %q is already registered", checker.Name())
+		}
+	}
+
+	*target = append(*target, checker)
+	return nil
+}
+
+// LivenessCheckers returns a snapshot of the registered liveness checks.
+func (r *Registry) LivenessCheckers() []HealthChecker {
+	if r == nil {
+		return nil
+	}
+
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
+	return append([]HealthChecker(nil), r.liveness...)
+}
+
+// ReadinessCheckers returns a snapshot of the registered readiness checks.
+func (r *Registry) ReadinessCheckers() []HealthChecker {
+	if r == nil {
+		return nil
+	}
+
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
+	return append([]HealthChecker(nil), r.readiness...)
+}