diff --git a/.github/workflows/build_and_push.yaml b/.github/workflows/build_and_push.yaml index a930be27..5eb0d37b 100644 --- a/.github/workflows/build_and_push.yaml +++ b/.github/workflows/build_and_push.yaml @@ -6,14 +6,19 @@ on: workflow_dispatch: push: branches: - - "main" + - "**" permissions: contents: write packages: write +concurrency: + group: build-and-push-${{ github.ref }} + cancel-in-progress: true + env: REGISTRY: ghcr.io + PREVIEW_CHART_REGISTRY: oci://ghcr.io/shopware/shopware-operator-preview jobs: goreleaser: @@ -71,3 +76,68 @@ jobs: repoSnapshot='ghcr.io/shopware/shopware-operator-snapshot:${{ fromJSON(steps.goreleaser.outputs.metadata).version }}' docker manifest create $repoSnapshot ${repoSnapshot}-arm64 ${repoSnapshot}-amd64 docker manifest push $repoSnapshot + + - name: Expose image tag + id: get_tag + run: echo "tag=${{ fromJSON(steps.goreleaser.outputs.metadata).version }}" >> "$GITHUB_OUTPUT" + + preview-helm-chart: + if: github.ref != 'refs/heads/main' + runs-on: runs-on=${{ github.run_id }}/runner=sw-amd64/cpu=1 + needs: goreleaser + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + fetch-depth: 0 + + - name: Setup Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 + with: + go-version-file: go.mod + + - name: Setup Helm + uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.1 + + - name: Login to GitHub Container Registry + run: | + echo '${{ secrets.GITHUB_TOKEN }}' \ + | helm registry login ghcr.io --username '${{ github.actor }}' --password-stdin + + - name: Build and push preview chart + id: chart + env: + IMAGE_TAG: ${{ needs.goreleaser.outputs.tag }} + run: | + set -euo pipefail + + # Chart versions must be semver, branch names are not. Encode the + # branch as a prerelease so every preview sorts below any release. + slug=$(echo "$IMAGE_TAG" \ + | tr '[:upper:]' '[:lower:]' \ + | tr -c '[:alnum:]' '-' \ + | sed 's/-\{1,\}/-/g; s/^-//; s/-$//') + chart_version="0.0.0-${slug}.g${GITHUB_SHA::7}" + + # version drives appVersion and the snapshot image refs inside the + # CRD templates, so it has to stay the branch-named image tag. + make helm path=preview-chart version="$IMAGE_TAG" chartVersion="$chart_version" + + helm package preview-chart --destination dist + helm push "dist/operator-${chart_version}.tgz" "$PREVIEW_CHART_REGISTRY" + + echo "version=${chart_version}" >> "$GITHUB_OUTPUT" + + - name: Job summary + env: + CHART_VERSION: ${{ steps.chart.outputs.version }} + run: | + { + echo "### Preview helm chart" + echo + echo '```sh' + echo "helm upgrade --install shopware-operator \\" + echo " ${PREVIEW_CHART_REGISTRY}/operator \\" + echo " --version ${CHART_VERSION}" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/cleanup_preview.yaml b/.github/workflows/cleanup_preview.yaml new file mode 100644 index 00000000..f61e7d6d --- /dev/null +++ b/.github/workflows/cleanup_preview.yaml @@ -0,0 +1,80 @@ +# Removes the branch scoped preview artifacts that "Build and push" creates +# for every non main branch: the snapshot images and the preview helm chart. +name: Cleanup preview artifacts + +on: + pull_request: + types: + - closed + workflow_dispatch: + inputs: + branch: + description: Branch whose preview artifacts should be removed + required: true + +permissions: + contents: read + packages: write + +jobs: + cleanup: + if: github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + steps: + - name: Delete preview packages + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BRANCH: ${{ github.event_name == 'workflow_dispatch' && inputs.branch || github.event.pull_request.head.ref }} + ORG: shopware + run: | + set -euo pipefail + + if [ "$BRANCH" = "main" ]; then + echo "Refusing to clean up main" + exit 1 + fi + + # Mirrors goreleaser's snapshot.version_template. + image_tag="${BRANCH//\//-}" + # Mirrors the chart version slug from build_and_push.yaml. + slug=$(echo "$image_tag" \ + | tr '[:upper:]' '[:lower:]' \ + | tr -c '[:alnum:]' '-' \ + | sed 's/-\{1,\}/-/g; s/^-//; s/-$//') + + # $1 = package name (path separators must be percent encoded) + # $2 = jq filter over the tag list, true keeps the version + delete_versions() { + local package="$1" filter="$2" versions + + versions=$(gh api --paginate \ + "/orgs/${ORG}/packages/container/${package}/versions" \ + --jq ".[] | select(any(.metadata.container.tags[]; ${filter})) | .id" \ + 2>/dev/null) || { + echo "::warning::Cannot read package ${package}, skipping" + return 0 + } + + if [ -z "$versions" ]; then + echo "Nothing to delete in ${package}" + return 0 + fi + + for id in $versions; do + echo "Deleting ${package} version ${id}" + gh api -X DELETE \ + "/orgs/${ORG}/packages/container/${package}/versions/${id}" \ + || echo "::warning::Failed to delete ${package} version ${id}" + done + } + + # The multi arch manifest list has to go before the per arch tags it + # references, otherwise the registry refuses to delete the children. + for package in shopware-operator shopware-operator-snapshot; do + delete_versions "$package" ". == \"${image_tag}\"" + delete_versions "$package" ". == \"${image_tag}-amd64\" or . == \"${image_tag}-arm64\"" + done + + # One chart version per commit, so match the whole branch prefix. + delete_versions "shopware-operator-preview%2Foperator" \ + "startswith(\"0.0.0-${slug}.\")" diff --git a/Makefile b/Makefile index 5ef6a227..d995174d 100644 --- a/Makefile +++ b/Makefile @@ -233,7 +233,7 @@ helm: path version manifests kustomize yq ## Undeploy controller from the K8s cl $(KUSTOMIZE) build config/crd > $(path)/templates/crds/all.yaml $(KUSTOMIZE) build config/helm > $(path)/templates/operator.yaml $(YQ) e -i '.appVersion = "$(version)"' $(path)/Chart.yaml - $(YQ) e -i '.version = "$(version)"' $(path)/Chart.yaml + $(YQ) e -i '.version = "$(if $(chartVersion),$(chartVersion),$(version))"' $(path)/Chart.yaml sed -i 's|image: ghcr.io/shopware/shopware-operator-snapshot:main|image: ghcr.io/shopware/shopware-operator-snapshot:'${version}'|g' $(path)/templates/crds/all.yaml $(YQ) $(path)/templates/crds/all.yaml -s '"$(path)/templates/crds/" + .spec.names.kind' --no-doc rm $(path)/templates/crds/all.yaml diff --git a/api/v1/store_env.go b/api/v1/store_env.go index b9783cf8..a5811960 100644 --- a/api/v1/store_env.go +++ b/api/v1/store_env.go @@ -16,6 +16,13 @@ const ( DatabaseTLSKeyFile = DatabaseTLSMountPath + "/tls.key" ) +// Set by the cmd main +var operatorServiceURL string + +func SetOperatorServiceURL(value string) { + operatorServiceURL = value +} + func (s Store) GetDatabaseTLSVolumes() []corev1.Volume { if s.Spec.Database.TLS.SecretName == "" { return nil @@ -611,6 +618,10 @@ func (s *Store) GetEnv() []corev1.EnvVar { Name: "DATABASE_PERSISTENT_CONNECTION", Value: "0", }, + { + Name: "SHOPWARE_OPERATOR_URL", + Value: operatorServiceURL, + }, } if s.Spec.Database.TLS.SecretName != "" { diff --git a/cmd/main.go b/cmd/main.go index 90a4aa51..50a58387 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -33,6 +33,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/go-logr/zapr" @@ -73,6 +74,8 @@ func main() { os.Exit(1) } + shopv1.SetOperatorServiceURL(cfg.OperatorServiceURL) + logger := logging.NewLogger(cfg.LogLevel, cfg.LogFormat). With(zapz.String("service", "shopware-operator")). With(zapz.String("operator_version", version)). @@ -89,8 +92,8 @@ func main() { } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - // Metrics: metricsserver.Options{BindAddress: cfg.MetricsAddr}, + Scheme: scheme, + Metrics: metricsserver.Options{BindAddress: cfg.MetricsAddr, SecureServing: false}, HealthProbeBindAddress: cfg.ProbeAddr, Cache: cache.Options{ DefaultNamespaces: map[string]cache.Config{ diff --git a/go.mod b/go.mod index f6c258dd..6852f94d 100644 --- a/go.mod +++ b/go.mod @@ -45,6 +45,7 @@ require ( github.com/google/btree v1.1.3 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/minio/crc64nvme v1.0.1 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/nats-io/nkeys v0.4.11 // indirect @@ -81,7 +82,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/prometheus/client_golang v1.22.0 // indirect + github.com/prometheus/client_golang v1.22.0 github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml index 7138660f..907b22ff 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployment.yaml @@ -84,6 +84,12 @@ spec: value: "{{ .Values.webhook.enabled }}" - name: SUCCESSFUL_CR_CLEANUP_GRACE_PERIOD value: "{{ .Values.successfulCRCleanupGracePeriod | default "1h" }}" + {{- if .Values.metrics.enabled }} + - name: METRICS_BIND_ADDRESS + value: ":{{ .Values.metrics.port | default 8080 }}" + - name: OPERATOR_SERVICE_URL + value: "{{ .Values.metrics.shopwareOperatorUrl | default (printf "http://shopware-operator.%s.svc.cluster.local:%d" .Release.Namespace (.Values.metrics.port | default 8080 | int)) }}" + {{- end }} {{- if and (hasKey .Values "events") (hasKey .Values.events "nats") (.Values.events.nats.enable) }} - name: NATS_ENABLE value: "true" @@ -111,18 +117,25 @@ spec: initialDelaySeconds: 15 periodSeconds: 20 name: operator + + {{- if or .Values.metrics.enabled .Values.webhook.enabled }} + ports: + {{- if .Values.metrics.enabled }} + - containerPort: {{ .Values.metrics.port | default 8080 }} + name: http-metrics + {{- end }} + {{- if .Values.webhook.enabled }} + - containerPort: 9443 + name: webhook-server + protocol: TCP + {{- end }} + {{- end }} readinessProbe: httpGet: path: /readyz port: 8081 initialDelaySeconds: 5 periodSeconds: 10 - {{- if .Values.webhook.enabled }} - ports: - - containerPort: 9443 - name: webhook-server - protocol: TCP - {{- end }} resources: {{- with .Values.resources }} {{- toYaml . | nindent 10 }} diff --git a/helm/templates/metrics-service.yaml b/helm/templates/metrics-service.yaml new file mode 100644 index 00000000..10328f9e --- /dev/null +++ b/helm/templates/metrics-service.yaml @@ -0,0 +1,29 @@ +{{- if not .Values.crds.installOnly }} +{{- if .Values.metrics.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: shopware-operator + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: metrics + app.kubernetes.io/created-by: shopware-operator + app.kubernetes.io/instance: shopware-operator + app.kubernetes.io/managed-by: shopware-operator + app.kubernetes.io/name: service + app.kubernetes.io/part-of: shopware-operator + control-plane: shopware-operator +{{- with .Values.labels }} + {{- toYaml . | nindent 4 }} +{{- end }} +spec: + type: ClusterIP + selector: + control-plane: shopware-operator + ports: + - name: http-metrics + port: {{ .Values.metrics.port | default 8080 }} + targetPort: http-metrics + protocol: TCP +{{- end }} +{{- end }} diff --git a/helm/templates/service-monitor.yaml b/helm/templates/service-monitor.yaml new file mode 100644 index 00000000..b5c6ef61 --- /dev/null +++ b/helm/templates/service-monitor.yaml @@ -0,0 +1,35 @@ +{{- if not .Values.crds.installOnly }} +{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: shopware-operator + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: metrics + app.kubernetes.io/created-by: shopware-operator + app.kubernetes.io/instance: shopware-operator + app.kubernetes.io/managed-by: shopware-operator + app.kubernetes.io/name: servicemonitor + app.kubernetes.io/part-of: shopware-operator + control-plane: shopware-operator +{{- with .Values.labels }} + {{- toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.metrics.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} +{{- end }} +spec: + endpoints: + - port: http-metrics + path: /metrics + interval: {{ .Values.metrics.serviceMonitor.interval }} + scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} + selector: + matchLabels: + control-plane: shopware-operator + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} +{{- end }} +{{- end }} diff --git a/helm/values.yaml b/helm/values.yaml index 99211dae..bc2e0af0 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -107,3 +107,28 @@ logFormat: json disableChecks: false # Grace period before successful StoreExec and StoreDebugInstance CRs are deleted. Set to "0" to disable cleanup. successfulCRCleanupGracePeriod: 1h + +metrics: + # When enabled, a Service named 'shopware-operator' is created, the metrics + # endpoint is exposed, and SHOPWARE_OPERATOR_URL is injected into every store + # container so the Shopware consumer can reach the operator. + enabled: false + port: 8080 + # shopwareOperatorUrl overrides the SHOPWARE_OPERATOR_URL injected into store + # containers (admin, storefront, worker). When left empty the URL is + # auto-constructed as: + # http://shopware-operator..svc.cluster.local: + shopwareOperatorUrl: "" + # serviceMonitor: configure a Prometheus Operator ServiceMonitor to scrape + # the operator metrics. Requires the Prometheus Operator CRDs to be installed. + serviceMonitor: + enabled: false + # Interval at which Prometheus scrapes the metrics endpoint. + interval: 30s + # Timeout for each scrape request. + scrapeTimeout: 10s + # Additional labels added to the ServiceMonitor, e.g. to match a Prometheus + # Operator release label selector: + # additionalLabels: + # release: kube-prometheus-stack + additionalLabels: {} diff --git a/internal/config/config.go b/internal/config/config.go index 19b6f6cd..b543f354 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -67,6 +67,10 @@ type StoreConfig struct { MetricsAddr string `env:"METRICS_BIND_ADDRESS, default=0"` ProbeAddr string `env:"HEALTH_PROBE_BIND_ADDRESS, default=:8081"` + // OperatorServiceURL is exposed to store containers as SHOPWARE_OPERATOR_URL so + // the Shopware consumer knows how to reach the operator service. + OperatorServiceURL string `env:"OPERATOR_SERVICE_URL"` + EnableLeaderElection bool `env:"LEADER_ELECT, default=true"` EnableWebhook bool `env:"ENABLE_WEBHOOK, default=true"` DisableChecks bool `env:"DISABLE_CHECKS, default=false"` diff --git a/internal/controller/predicate.go b/internal/controller/predicate.go index ffe9abfc..61c31a11 100644 --- a/internal/controller/predicate.go +++ b/internal/controller/predicate.go @@ -45,7 +45,7 @@ func (t TypedSkipStatusPredicate[object]) Update(e event.TypedUpdateEvent[object kind := "unknown" objType := reflect.TypeOf(e.ObjectNew) if objType != nil { - if objType.Kind() == reflect.Ptr { + if objType.Kind() == reflect.Pointer { objType = objType.Elem() } kind = objType.Name() @@ -176,7 +176,7 @@ func isNil(arg any) bool { } v := reflect.ValueOf(arg) switch v.Kind() { - case reflect.Ptr, reflect.Interface, reflect.Slice, reflect.Map, reflect.Chan, reflect.Func: + case reflect.Pointer, reflect.Interface, reflect.Slice, reflect.Map, reflect.Chan, reflect.Func: return v.IsNil() default: return false @@ -188,7 +188,7 @@ func (t TypedSkipStatusPredicate[object]) isInAllowList(kind string) bool { for _, allowedObj := range t.AllowList { objType := reflect.TypeOf(allowedObj) if objType != nil { - if objType.Kind() == reflect.Ptr { + if objType.Kind() == reflect.Pointer { objType = objType.Elem() } if objType.Name() == kind { diff --git a/internal/controller/store_controller.go b/internal/controller/store_controller.go index b64054d8..60b41a92 100644 --- a/internal/controller/store_controller.go +++ b/internal/controller/store_controller.go @@ -15,6 +15,7 @@ import ( "github.com/shopware/shopware-operator/internal/job" "github.com/shopware/shopware-operator/internal/k8s" "github.com/shopware/shopware-operator/internal/logging" + "github.com/shopware/shopware-operator/internal/metrics" "github.com/shopware/shopware-operator/internal/pdb" "github.com/shopware/shopware-operator/internal/secret" "github.com/shopware/shopware-operator/internal/service" @@ -176,6 +177,7 @@ func (r *StoreReconciler) Reconcile( // } if !store.DeletionTimestamp.IsZero() { + metrics.RemoveStoreMetrics(store) return shortRequeue, nil } diff --git a/internal/controller/store_status.go b/internal/controller/store_status.go index 686839a5..1198aaee 100644 --- a/internal/controller/store_status.go +++ b/internal/controller/store_status.go @@ -8,10 +8,12 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" v1 "github.com/shopware/shopware-operator/api/v1" + "github.com/shopware/shopware-operator/internal/cronjob" "github.com/shopware/shopware-operator/internal/deployment" "github.com/shopware/shopware-operator/internal/job" "github.com/shopware/shopware-operator/internal/k8s" "github.com/shopware/shopware-operator/internal/logging" + "github.com/shopware/shopware-operator/internal/metrics" "github.com/shopware/shopware-operator/internal/util" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" @@ -117,6 +119,16 @@ func (r *StoreReconciler) reconcileCRStatus( logging.FromContext(ctx).Infow("Update store status", zap.Any("status", store.Status)) r.SendEvent(ctx, *store, "Update store status") + metrics.UpdateStoreMetrics(store) + + scheduledCronJob, err := cronjob.GetScheduledCronJob(ctx, r.Client, *store) + if err != nil { + if !k8serrors.IsNotFound(err) { + logging.FromContext(ctx).Warnw("failed to get scheduled task cronjob for metrics", zap.Error(err)) + } + scheduledCronJob = nil + } + metrics.UpdateScheduledTaskMetrics(store, scheduledCronJob) return writeStoreStatus(ctx, r.Client, types.NamespacedName{ Namespace: store.Namespace, diff --git a/internal/k8s/utils.go b/internal/k8s/utils.go index 1a8139e6..ddf4e1dd 100644 --- a/internal/k8s/utils.go +++ b/internal/k8s/utils.go @@ -398,7 +398,7 @@ func HasObjectChanged( } val := reflect.ValueOf(obj) - if val.Kind() == reflect.Ptr { + if val.Kind() == reflect.Pointer { val = reflect.Indirect(val) } oldObject := reflect.New(val.Type()).Interface().(client.Object) @@ -468,7 +468,7 @@ func EnsureObjectWithHash( obj.SetAnnotations(objAnnotations) val := reflect.ValueOf(obj) - if val.Kind() == reflect.Ptr { + if val.Kind() == reflect.Pointer { val = reflect.Indirect(val) } oldObject := reflect.New(val.Type()).Interface().(client.Object) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 00000000..b79ec847 --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,278 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + v1 "github.com/shopware/shopware-operator/api/v1" + batchv1 "k8s.io/api/batch/v1" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +var ( + storeState = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "shopware_store_state", + Help: "Current state of a Shopware store (1 for active state, 0 otherwise)", + }, []string{"store", "namespace", "state"}) + + storeCurrentImage = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "shopware_store_current_image", + Help: "Current image of a Shopware store", + }, []string{"store", "namespace", "image"}) + + storeDeploymentReplicasAvailable = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "shopware_store_deployment_replicas_available", + Help: "Available replica count per deployment type", + }, []string{"store", "namespace", "deployment_type"}) + + storeDeploymentReplicasDesired = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "shopware_store_deployment_replicas_desired", + Help: "Desired replica count per deployment type", + }, []string{"store", "namespace", "deployment_type"}) + + storeDeploymentState = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "shopware_store_deployment_state", + Help: "Current state of a store deployment (1 for active state, 0 otherwise)", + }, []string{"store", "namespace", "deployment_type", "state"}) + + storeUsageDataConsent = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "shopware_store_usage_data_consent", + Help: "Usage data consent status (1 for allowed, 0 for revoked)", + }, []string{"store", "namespace"}) + + storeHPAEnabled = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "shopware_store_hpa_enabled", + Help: "Whether the HorizontalPodAutoscaler is enabled (1) or disabled (0)", + }, []string{"store", "namespace"}) + + storeHPAMinReplicas = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "shopware_store_hpa_min_replicas", + Help: "HPA minimum replicas", + }, []string{"store", "namespace"}) + + storeHPAMaxReplicas = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "shopware_store_hpa_max_replicas", + Help: "HPA maximum replicas", + }, []string{"store", "namespace"}) + + storeScheduledTaskSuspended = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "shopware_store_scheduled_task_suspended", + Help: "Whether the scheduled task CronJob is suspended (1) or active/not found (0)", + }, []string{"store", "namespace"}) + + storeScheduledTaskLastRunStatus = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "shopware_store_scheduled_task_last_run_status", + Help: "Status of the latest scheduled task run (1 for success, -1 for failure, 0 for unknown/no runs)", + }, []string{"store", "namespace"}) + + storeScheduledTaskLastSuccessTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "shopware_store_scheduled_task_last_success_timestamp", + Help: "Unix timestamp of the last successful scheduled task run", + }, []string{"store", "namespace"}) + + allStates = []v1.StatefulAppState{ + v1.StateWait, + v1.StateSetup, + v1.StateSetupError, + v1.StateInitializing, + v1.StateMigration, + v1.StateMigrationError, + v1.StateReady, + } + + allDeploymentStates = []v1.DeploymentState{ + v1.DeploymentStateUnknown, + v1.DeploymentStateError, + v1.DeploymentStateNotFound, + v1.DeploymentStateRunning, + v1.DeploymentStateScaling, + } +) + +func init() { + metrics.Registry.MustRegister( + storeState, + storeCurrentImage, + storeDeploymentReplicasAvailable, + storeDeploymentReplicasDesired, + storeDeploymentState, + storeUsageDataConsent, + storeHPAEnabled, + storeHPAMinReplicas, + storeHPAMaxReplicas, + storeScheduledTaskSuspended, + storeScheduledTaskLastRunStatus, + storeScheduledTaskLastSuccessTime, + ) +} + +// UpdateStoreMetrics sets all gauge values from a Store's status. +func UpdateStoreMetrics(store *v1.Store) { + name := store.Name + ns := store.Namespace + + // Store state + for _, s := range allStates { + val := float64(0) + if store.Status.State == s { + val = 1 + } + storeState.WithLabelValues(name, ns, string(s)).Set(val) + } + + // Current image + if store.Status.CurrentImageTag != "" { + // Delete old image labels, then set new one + storeCurrentImage.DeletePartialMatch(prometheus.Labels{ + "store": name, + "namespace": ns, + }) + storeCurrentImage.WithLabelValues(name, ns, store.Status.CurrentImageTag).Set(1) + } + + // Usage data consent + val := float64(0) + if store.Spec.ShopConfiguration.UsageDataConsent == "allowed" { + val = 1 + } + storeUsageDataConsent.WithLabelValues(name, ns).Set(val) + + // HPA + hpa := store.Spec.HorizontalPodAutoscaler + if hpa.Enabled { + storeHPAEnabled.WithLabelValues(name, ns).Set(1) + storeHPAMaxReplicas.WithLabelValues(name, ns).Set(float64(hpa.MaxReplicas)) + if hpa.MinReplicas != nil { + storeHPAMinReplicas.WithLabelValues(name, ns).Set(float64(*hpa.MinReplicas)) + } else { + storeHPAMinReplicas.WithLabelValues(name, ns).Set(0) + } + } else { + storeHPAEnabled.WithLabelValues(name, ns).Set(0) + storeHPAMinReplicas.WithLabelValues(name, ns).Set(0) + storeHPAMaxReplicas.WithLabelValues(name, ns).Set(0) + } + + // Deployment metrics + setDeploymentMetrics(name, ns, "admin", store.Status.AdminState) + setDeploymentMetrics(name, ns, "storefront", store.Status.StorefrontState) + setDeploymentMetrics(name, ns, "worker", store.Status.WorkerState) +} + +func setDeploymentMetrics(name, ns, deploymentType string, cond v1.DeploymentCondition) { + // Parse available from the Ready field (format: "available/desired") + var available int + if cond.Ready != "" { + fmtScan(cond.Ready, &available, new(int)) + } + + storeDeploymentReplicasAvailable.WithLabelValues(name, ns, deploymentType).Set(float64(available)) + storeDeploymentReplicasDesired.WithLabelValues(name, ns, deploymentType).Set(float64(cond.StoreReplicas)) + + // Deployment state + for _, s := range allDeploymentStates { + val := float64(0) + if cond.State == s { + val = 1 + } + storeDeploymentState.WithLabelValues(name, ns, deploymentType, string(s)).Set(val) + } +} + +func fmtScan(ready string, available, desired *int) { + // Parse "X/Y" format + for i, c := range ready { + if c == '/' { + *available = atoi(ready[:i]) + *desired = atoi(ready[i+1:]) + return + } + } +} + +func atoi(s string) int { + n := 0 + for _, c := range s { + if c >= '0' && c <= '9' { + n = n*10 + int(c-'0') + } + } + return n +} + +// UpdateScheduledTaskMetrics sets metrics for the scheduled task CronJob. +func UpdateScheduledTaskMetrics(store *v1.Store, cronJob *batchv1.CronJob) { + name := store.Name + ns := store.Namespace + + if cronJob == nil { + storeScheduledTaskSuspended.WithLabelValues(name, ns).Set(0) + storeScheduledTaskLastRunStatus.WithLabelValues(name, ns).Set(0) + storeScheduledTaskLastSuccessTime.WithLabelValues(name, ns).Set(0) + return + } + + // Suspended + if cronJob.Spec.Suspend != nil && *cronJob.Spec.Suspend { + storeScheduledTaskSuspended.WithLabelValues(name, ns).Set(1) + } else { + storeScheduledTaskSuspended.WithLabelValues(name, ns).Set(0) + } + + // Last run status: compare LastSuccessfulTime vs LastScheduleTime. + // If last successful time >= last schedule time, the latest run succeeded. + // If last schedule time is after last successful time, the job either + // failed or is still running. Only report failure when no jobs are + // currently active; otherwise report 0 (in progress / unknown). + jobActive := len(cronJob.Status.Active) > 0 + switch { + case cronJob.Status.LastSuccessfulTime != nil && cronJob.Status.LastScheduleTime != nil: + if !cronJob.Status.LastSuccessfulTime.Before(cronJob.Status.LastScheduleTime) { + storeScheduledTaskLastRunStatus.WithLabelValues(name, ns).Set(1) + } else if jobActive { + storeScheduledTaskLastRunStatus.WithLabelValues(name, ns).Set(0) + } else { + storeScheduledTaskLastRunStatus.WithLabelValues(name, ns).Set(-1) + } + case cronJob.Status.LastSuccessfulTime != nil: + storeScheduledTaskLastRunStatus.WithLabelValues(name, ns).Set(1) + case cronJob.Status.LastScheduleTime != nil: + // Scheduled but never succeeded — could still be running + if jobActive { + storeScheduledTaskLastRunStatus.WithLabelValues(name, ns).Set(0) + } else { + storeScheduledTaskLastRunStatus.WithLabelValues(name, ns).Set(-1) + } + default: + storeScheduledTaskLastRunStatus.WithLabelValues(name, ns).Set(0) + } + + // Last success timestamp + if cronJob.Status.LastSuccessfulTime != nil { + storeScheduledTaskLastSuccessTime.WithLabelValues(name, ns).Set(float64(cronJob.Status.LastSuccessfulTime.Unix())) + } else { + storeScheduledTaskLastSuccessTime.WithLabelValues(name, ns).Set(0) + } +} + +// RemoveStoreMetrics removes all metrics for a deleted store. +func RemoveStoreMetrics(store *v1.Store) { + name := store.Name + ns := store.Namespace + + match := prometheus.Labels{ + "store": name, + "namespace": ns, + } + + storeState.DeletePartialMatch(match) + storeCurrentImage.DeletePartialMatch(match) + storeDeploymentReplicasAvailable.DeletePartialMatch(match) + storeDeploymentReplicasDesired.DeletePartialMatch(match) + storeDeploymentState.DeletePartialMatch(match) + storeUsageDataConsent.DeletePartialMatch(match) + storeHPAEnabled.DeletePartialMatch(match) + storeHPAMinReplicas.DeletePartialMatch(match) + storeHPAMaxReplicas.DeletePartialMatch(match) + storeScheduledTaskSuspended.DeletePartialMatch(match) + storeScheduledTaskLastRunStatus.DeletePartialMatch(match) + storeScheduledTaskLastSuccessTime.DeletePartialMatch(match) +} diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go new file mode 100644 index 00000000..a105e740 --- /dev/null +++ b/internal/metrics/metrics_test.go @@ -0,0 +1,163 @@ +package metrics + +import ( + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + v1 "github.com/shopware/shopware-operator/api/v1" + "github.com/stretchr/testify/assert" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func newStore() *v1.Store { + return &v1.Store{ + ObjectMeta: metav1.ObjectMeta{Name: "shop", Namespace: "default"}, + } +} + +func TestUpdateStoreMetricsState(t *testing.T) { + store := newStore() + store.Status.State = v1.StateReady + + UpdateStoreMetrics(store) + defer RemoveStoreMetrics(store) + + assert.Equal(t, float64(1), testutil.ToFloat64(storeState.WithLabelValues("shop", "default", string(v1.StateReady)))) + assert.Equal(t, float64(0), testutil.ToFloat64(storeState.WithLabelValues("shop", "default", string(v1.StateWait)))) +} + +func TestUpdateStoreMetricsUsageDataConsent(t *testing.T) { + store := newStore() + store.Spec.ShopConfiguration.UsageDataConsent = "allowed" + + UpdateStoreMetrics(store) + defer RemoveStoreMetrics(store) + + assert.Equal(t, float64(1), testutil.ToFloat64(storeUsageDataConsent.WithLabelValues("shop", "default"))) + + store.Spec.ShopConfiguration.UsageDataConsent = "revoked" + UpdateStoreMetrics(store) + + assert.Equal(t, float64(0), testutil.ToFloat64(storeUsageDataConsent.WithLabelValues("shop", "default"))) +} + +func TestUpdateStoreMetricsHPA(t *testing.T) { + store := newStore() + minReplicas := int32(2) + store.Spec.HorizontalPodAutoscaler = v1.HPASpec{ + Enabled: true, + MinReplicas: &minReplicas, + MaxReplicas: 5, + } + + UpdateStoreMetrics(store) + defer RemoveStoreMetrics(store) + + assert.Equal(t, float64(1), testutil.ToFloat64(storeHPAEnabled.WithLabelValues("shop", "default"))) + assert.Equal(t, float64(2), testutil.ToFloat64(storeHPAMinReplicas.WithLabelValues("shop", "default"))) + assert.Equal(t, float64(5), testutil.ToFloat64(storeHPAMaxReplicas.WithLabelValues("shop", "default"))) + + store.Spec.HorizontalPodAutoscaler = v1.HPASpec{Enabled: false} + UpdateStoreMetrics(store) + + assert.Equal(t, float64(0), testutil.ToFloat64(storeHPAEnabled.WithLabelValues("shop", "default"))) + assert.Equal(t, float64(0), testutil.ToFloat64(storeHPAMinReplicas.WithLabelValues("shop", "default"))) + assert.Equal(t, float64(0), testutil.ToFloat64(storeHPAMaxReplicas.WithLabelValues("shop", "default"))) +} + +func TestSetDeploymentMetricsParsesReady(t *testing.T) { + store := newStore() + store.Status.AdminState = v1.DeploymentCondition{ + State: v1.DeploymentStateRunning, + Ready: "2/3", + StoreReplicas: 3, + } + + UpdateStoreMetrics(store) + defer RemoveStoreMetrics(store) + + assert.Equal(t, float64(2), testutil.ToFloat64(storeDeploymentReplicasAvailable.WithLabelValues("shop", "default", "admin"))) + assert.Equal(t, float64(3), testutil.ToFloat64(storeDeploymentReplicasDesired.WithLabelValues("shop", "default", "admin"))) + assert.Equal(t, float64(1), testutil.ToFloat64(storeDeploymentState.WithLabelValues("shop", "default", "admin", string(v1.DeploymentStateRunning)))) + assert.Equal(t, float64(0), testutil.ToFloat64(storeDeploymentState.WithLabelValues("shop", "default", "admin", string(v1.DeploymentStateError)))) +} + +func TestUpdateScheduledTaskMetricsNilCronJob(t *testing.T) { + store := newStore() + + UpdateScheduledTaskMetrics(store, nil) + defer RemoveStoreMetrics(store) + + assert.Equal(t, float64(0), testutil.ToFloat64(storeScheduledTaskSuspended.WithLabelValues("shop", "default"))) + assert.Equal(t, float64(0), testutil.ToFloat64(storeScheduledTaskLastRunStatus.WithLabelValues("shop", "default"))) + assert.Equal(t, float64(0), testutil.ToFloat64(storeScheduledTaskLastSuccessTime.WithLabelValues("shop", "default"))) +} + +func TestUpdateScheduledTaskMetricsSuspended(t *testing.T) { + store := newStore() + suspend := true + cronJob := &batchv1.CronJob{Spec: batchv1.CronJobSpec{Suspend: &suspend}} + + UpdateScheduledTaskMetrics(store, cronJob) + defer RemoveStoreMetrics(store) + + assert.Equal(t, float64(1), testutil.ToFloat64(storeScheduledTaskSuspended.WithLabelValues("shop", "default"))) +} + +func TestUpdateScheduledTaskMetricsLastRunSuccess(t *testing.T) { + store := newStore() + scheduleTime := metav1.NewTime(time.Unix(100, 0)) + successTime := metav1.NewTime(time.Unix(200, 0)) + cronJob := &batchv1.CronJob{Status: batchv1.CronJobStatus{ + LastScheduleTime: &scheduleTime, + LastSuccessfulTime: &successTime, + }} + + UpdateScheduledTaskMetrics(store, cronJob) + defer RemoveStoreMetrics(store) + + assert.Equal(t, float64(1), testutil.ToFloat64(storeScheduledTaskLastRunStatus.WithLabelValues("shop", "default"))) + assert.Equal(t, float64(200), testutil.ToFloat64(storeScheduledTaskLastSuccessTime.WithLabelValues("shop", "default"))) +} + +func TestUpdateScheduledTaskMetricsLastRunFailed(t *testing.T) { + store := newStore() + successTime := metav1.NewTime(time.Unix(100, 0)) + scheduleTime := metav1.NewTime(time.Unix(200, 0)) + cronJob := &batchv1.CronJob{Status: batchv1.CronJobStatus{ + LastScheduleTime: &scheduleTime, + LastSuccessfulTime: &successTime, + }} + + UpdateScheduledTaskMetrics(store, cronJob) + defer RemoveStoreMetrics(store) + + assert.Equal(t, float64(-1), testutil.ToFloat64(storeScheduledTaskLastRunStatus.WithLabelValues("shop", "default"))) +} + +func TestUpdateScheduledTaskMetricsLastRunInProgress(t *testing.T) { + store := newStore() + successTime := metav1.NewTime(time.Unix(100, 0)) + scheduleTime := metav1.NewTime(time.Unix(200, 0)) + cronJob := &batchv1.CronJob{Status: batchv1.CronJobStatus{ + LastScheduleTime: &scheduleTime, + LastSuccessfulTime: &successTime, + Active: []corev1.ObjectReference{{Name: "shop-scheduled-task-1"}}, + }} + + UpdateScheduledTaskMetrics(store, cronJob) + defer RemoveStoreMetrics(store) + + assert.Equal(t, float64(0), testutil.ToFloat64(storeScheduledTaskLastRunStatus.WithLabelValues("shop", "default"))) +} + +func TestFmtScan(t *testing.T) { + var available, desired int + fmtScan("4/7", &available, &desired) + + assert.Equal(t, 4, available) + assert.Equal(t, 7, desired) +}