feat: add metrics support

api/v1/store_env.go

@@ -16,6 +16,13 @@ const (
 	DatabaseTLSKeyFile    = DatabaseTLSMountPath + "/tls.key"
 )
 
+// Set by the cmd main
+var operatorServiceURL string
+
+func SetOperatorServiceURL(value string) {
+	operatorServiceURL = value
+}
+
 func (s Store) GetDatabaseTLSVolumes() []corev1.Volume {
 	if s.Spec.Database.TLS.SecretName == "" {
 		return nil
@@ -611,6 +618,10 @@ func (s *Store) GetEnv() []corev1.EnvVar {
 			Name:  "DATABASE_PERSISTENT_CONNECTION",
 			Value: "0",
 		},
+		{
+			Name:  "SHOPWARE_OPERATOR_URL",
+			Value: operatorServiceURL,
+		},
 	}
 
 	if s.Spec.Database.TLS.SecretName != "" {

cmd/main.go

@@ -33,6 +33,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 
 	"github.com/go-logr/zapr"
@@ -73,6 +74,8 @@ func main() {
 		os.Exit(1)
 	}
 
+	shopv1.SetOperatorServiceURL(cfg.OperatorServiceURL)
+
 	logger := logging.NewLogger(cfg.LogLevel, cfg.LogFormat).
 		With(zapz.String("service", "shopware-operator")).
 		With(zapz.String("operator_version", version)).
@@ -89,8 +92,8 @@ func main() {
 	}
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		Scheme: scheme,
-		// Metrics:                 metricsserver.Options{BindAddress: cfg.MetricsAddr},
+		Scheme:                 scheme,
+		Metrics:                metricsserver.Options{BindAddress: cfg.MetricsAddr, SecureServing: false},
 		HealthProbeBindAddress: cfg.ProbeAddr,
 		Cache: cache.Options{
 			DefaultNamespaces: map[string]cache.Config{

go.mod

@@ -45,6 +45,7 @@ require (
 	github.com/google/btree v1.1.3 // indirect
 	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.10 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/minio/crc64nvme v1.0.1 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/nats-io/nkeys v0.4.11 // indirect
@@ -81,7 +82,7 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/prometheus/client_golang v1.22.0 // indirect
+	github.com/prometheus/client_golang v1.22.0
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.62.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect

helm/templates/deployment.yaml

@@ -84,6 +84,12 @@ spec:
           value: "{{ .Values.webhook.enabled }}"
         - name: SUCCESSFUL_CR_CLEANUP_GRACE_PERIOD
           value: "{{ .Values.successfulCRCleanupGracePeriod | default "1h" }}"
+        {{- if .Values.metrics.enabled }}
+        - name: METRICS_BIND_ADDRESS
+          value: ":{{ .Values.metrics.port | default 8080 }}"
+        - name: OPERATOR_SERVICE_URL
+          value: "{{ .Values.metrics.shopwareOperatorUrl | default (printf "http://shopware-operator.%s.svc.cluster.local:%d" .Release.Namespace (.Values.metrics.port | default 8080 | int)) }}"
+        {{- end }}
         {{- if and (hasKey .Values "events") (hasKey .Values.events "nats") (.Values.events.nats.enable) }}
         - name: NATS_ENABLE
           value: "true"
@@ -111,18 +117,25 @@ spec:
           initialDelaySeconds: 15
           periodSeconds: 20
         name: operator
+
+        {{- if or .Values.metrics.enabled .Values.webhook.enabled }}
+        ports:
+        {{- if .Values.metrics.enabled }}
+          - containerPort: {{ .Values.metrics.port | default 8080 }}
+            name: http-metrics
+        {{- end }}
+        {{- if .Values.webhook.enabled }}
+          - containerPort: 9443
+            name: webhook-server
+            protocol: TCP
+        {{- end }}
+        {{- end }}
         readinessProbe:
           httpGet:
             path: /readyz
             port: 8081
           initialDelaySeconds: 5
           periodSeconds: 10
-        {{- if .Values.webhook.enabled }}
-        ports:
-        - containerPort: 9443
-          name: webhook-server
-          protocol: TCP
-        {{- end }}
         resources:
           {{- with .Values.resources }}
               {{- toYaml . | nindent 10 }}

helm/templates/metrics-service.yaml

@@ -0,0 +1,29 @@
+{{- if not .Values.crds.installOnly }}
+{{- if .Values.metrics.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: shopware-operator
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/component: metrics
+    app.kubernetes.io/created-by: shopware-operator
+    app.kubernetes.io/instance: shopware-operator
+    app.kubernetes.io/managed-by: shopware-operator
+    app.kubernetes.io/name: service
+    app.kubernetes.io/part-of: shopware-operator
+    control-plane: shopware-operator
+{{- with .Values.labels }}
+    {{- toYaml . | nindent 4 }}
+{{- end }}
+spec:
+  type: ClusterIP
+  selector:
+    control-plane: shopware-operator
+  ports:
+    - name: http-metrics
+      port: {{ .Values.metrics.port | default 8080 }}
+      targetPort: http-metrics
+      protocol: TCP
+{{- end }}
+{{- end }}

helm/templates/service-monitor.yaml

@@ -0,0 +1,35 @@
+{{- if not .Values.crds.installOnly }}
+{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: shopware-operator
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/component: metrics
+    app.kubernetes.io/created-by: shopware-operator
+    app.kubernetes.io/instance: shopware-operator
+    app.kubernetes.io/managed-by: shopware-operator
+    app.kubernetes.io/name: servicemonitor
+    app.kubernetes.io/part-of: shopware-operator
+    control-plane: shopware-operator
+{{- with .Values.labels }}
+    {{- toYaml . | nindent 4 }}
+{{- end }}
+{{- with .Values.metrics.serviceMonitor.additionalLabels }}
+    {{- toYaml . | nindent 4 }}
+{{- end }}
+spec:
+  endpoints:
+    - port: http-metrics
+      path: /metrics
+      interval: {{ .Values.metrics.serviceMonitor.interval }}
+      scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }}
+  selector:
+    matchLabels:
+      control-plane: shopware-operator
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace }}
+{{- end }}
+{{- end }}

helm/values.yaml

@@ -107,3 +107,28 @@ logFormat: json
 disableChecks: false
 # Grace period before successful StoreExec and StoreDebugInstance CRs are deleted. Set to "0" to disable cleanup.
 successfulCRCleanupGracePeriod: 1h
+
+metrics:
+  # When enabled, a Service named 'shopware-operator' is created, the metrics
+  # endpoint is exposed, and SHOPWARE_OPERATOR_URL is injected into every store
+  # container so the Shopware consumer can reach the operator.
+  enabled: false
+  port: 8080
+  # shopwareOperatorUrl overrides the SHOPWARE_OPERATOR_URL injected into store
+  # containers (admin, storefront, worker). When left empty the URL is
+  # auto-constructed as:
+  #   http://shopware-operator.<namespace>.svc.cluster.local:<port>
+  shopwareOperatorUrl: ""
+  # serviceMonitor: configure a Prometheus Operator ServiceMonitor to scrape
+  # the operator metrics. Requires the Prometheus Operator CRDs to be installed.
+  serviceMonitor:
+    enabled: false
+    # Interval at which Prometheus scrapes the metrics endpoint.
+    interval: 30s
+    # Timeout for each scrape request.
+    scrapeTimeout: 10s
+    # Additional labels added to the ServiceMonitor, e.g. to match a Prometheus
+    # Operator release label selector:
+    #   additionalLabels:
+    #     release: kube-prometheus-stack
+    additionalLabels: {}

internal/config/config.go

@@ -67,6 +67,10 @@ type StoreConfig struct {
 	MetricsAddr string `env:"METRICS_BIND_ADDRESS, default=0"`
 	ProbeAddr   string `env:"HEALTH_PROBE_BIND_ADDRESS, default=:8081"`
 
+	// OperatorServiceURL is exposed to store containers as SHOPWARE_OPERATOR_URL so
+	// the Shopware consumer knows how to reach the operator service.
+	OperatorServiceURL string `env:"OPERATOR_SERVICE_URL"`
+
 	EnableLeaderElection bool   `env:"LEADER_ELECT, default=true"`
 	EnableWebhook        bool   `env:"ENABLE_WEBHOOK, default=true"`
 	DisableChecks        bool   `env:"DISABLE_CHECKS, default=false"`

internal/controller/predicate.go

@@ -45,7 +45,7 @@ func (t TypedSkipStatusPredicate[object]) Update(e event.TypedUpdateEvent[object
 	kind := "unknown"
 	objType := reflect.TypeOf(e.ObjectNew)
 	if objType != nil {
-		if objType.Kind() == reflect.Ptr {
+		if objType.Kind() == reflect.Pointer {
 			objType = objType.Elem()
 		}
 		kind = objType.Name()
@@ -176,7 +176,7 @@ func isNil(arg any) bool {
 	}
 	v := reflect.ValueOf(arg)
 	switch v.Kind() {
-	case reflect.Ptr, reflect.Interface, reflect.Slice, reflect.Map, reflect.Chan, reflect.Func:
+	case reflect.Pointer, reflect.Interface, reflect.Slice, reflect.Map, reflect.Chan, reflect.Func:
 		return v.IsNil()
 	default:
 		return false
@@ -188,7 +188,7 @@ func (t TypedSkipStatusPredicate[object]) isInAllowList(kind string) bool {
 	for _, allowedObj := range t.AllowList {
 		objType := reflect.TypeOf(allowedObj)
 		if objType != nil {
-			if objType.Kind() == reflect.Ptr {
+			if objType.Kind() == reflect.Pointer {
 				objType = objType.Elem()
 			}
 			if objType.Name() == kind {

internal/controller/store_controller.go

@@ -15,6 +15,7 @@ import (
 	"github.com/shopware/shopware-operator/internal/job"
 	"github.com/shopware/shopware-operator/internal/k8s"
 	"github.com/shopware/shopware-operator/internal/logging"
+	"github.com/shopware/shopware-operator/internal/metrics"
 	"github.com/shopware/shopware-operator/internal/pdb"
 	"github.com/shopware/shopware-operator/internal/secret"
 	"github.com/shopware/shopware-operator/internal/service"
@@ -176,6 +177,7 @@ func (r *StoreReconciler) Reconcile(
 	// }
 
 	if !store.DeletionTimestamp.IsZero() {
+		metrics.RemoveStoreMetrics(store)
 		return shortRequeue, nil
 	}
 

internal/controller/store_status.go

@@ -8,10 +8,12 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	v1 "github.com/shopware/shopware-operator/api/v1"
+	"github.com/shopware/shopware-operator/internal/cronjob"
 	"github.com/shopware/shopware-operator/internal/deployment"
 	"github.com/shopware/shopware-operator/internal/job"
 	"github.com/shopware/shopware-operator/internal/k8s"
 	"github.com/shopware/shopware-operator/internal/logging"
+	"github.com/shopware/shopware-operator/internal/metrics"
 	"github.com/shopware/shopware-operator/internal/util"
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
@@ -117,6 +119,16 @@ func (r *StoreReconciler) reconcileCRStatus(
 
 	logging.FromContext(ctx).Infow("Update store status", zap.Any("status", store.Status))
 	r.SendEvent(ctx, *store, "Update store status")
+	metrics.UpdateStoreMetrics(store)
+
+	scheduledCronJob, err := cronjob.GetScheduledCronJob(ctx, r.Client, *store)
+	if err != nil {
+		if !k8serrors.IsNotFound(err) {
+			logging.FromContext(ctx).Warnw("failed to get scheduled task cronjob for metrics", zap.Error(err))
+		}
+		scheduledCronJob = nil
+	}
+	metrics.UpdateScheduledTaskMetrics(store, scheduledCronJob)
 
 	return writeStoreStatus(ctx, r.Client, types.NamespacedName{
 		Namespace: store.Namespace,

internal/k8s/utils.go

@@ -398,7 +398,7 @@ func HasObjectChanged(
 	}
 
 	val := reflect.ValueOf(obj)
-	if val.Kind() == reflect.Ptr {
+	if val.Kind() == reflect.Pointer {
 		val = reflect.Indirect(val)
 	}
 	oldObject := reflect.New(val.Type()).Interface().(client.Object)
@@ -468,7 +468,7 @@ func EnsureObjectWithHash(
 	obj.SetAnnotations(objAnnotations)
 
 	val := reflect.ValueOf(obj)
-	if val.Kind() == reflect.Ptr {
+	if val.Kind() == reflect.Pointer {
 		val = reflect.Indirect(val)
 	}
 	oldObject := reflect.New(val.Type()).Interface().(client.Object)