feat(aro-hcp): add periodic Grafana datasource cleanup job (AROSLSRE-1138)

ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main__periodic-cleanup.yaml

@@ -203,6 +203,18 @@ tests:
       CLEANUP_SWEEPER_WORKFLOW: shared-leftovers
     test:
     - ref: aro-hcp-deprovision-cleanup-sweeper
+- as: clean-grafana-datasources
+  cron: 0 6 1 * *
+  reporter_config:
+    channel: '#aro-hcp-failures-dev'
+    job_states_to_report:
+    - failure
+    - error
+    report_template: ':failed: Job *{{.Spec.Job}}* ended with *{{.Status.State}}*.
+      <{{.Status.URL}}|View logs>'
+  steps:
+    test:
+    - ref: aro-hcp-deprovision-grafana-datasources
 zz_generated_metadata:
   branch: main
   org: Azure

ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-periodics.yaml

@@ -178,6 +178,87 @@ periodics:
     - name: result-aggregator
       secret:
         secretName: result-aggregator
+- agent: kubernetes
+  cluster: build04
+  cron: 0 6 1 * *
+  decorate: true
+  decoration_config:
+    skip_cloning: true
+  extra_refs:
+  - base_ref: main
+    org: Azure
+    repo: ARO-HCP
+  labels:
+    ci-operator.openshift.io/variant: periodic-cleanup
+    ci.openshift.io/generator: prowgen
+    pj-rehearse.openshift.io/can-be-rehearsed: "true"
+  name: periodic-ci-Azure-ARO-HCP-main-periodic-cleanup-clean-grafana-datasources
+  reporter_config:
+    slack:
+      channel: '#aro-hcp-failures-dev'
+      job_states_to_report:
+      - failure
+      - error
+      report_template: ':failed: Job *{{.Spec.Job}}* ended with *{{.Status.State}}*.
+        <{{.Status.URL}}|View logs>'
+  spec:
+    containers:
+    - args:
+      - --gcs-upload-secret=/secrets/gcs/service-account.json
+      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+      - --lease-server-credentials-file=/etc/boskos/credentials
+      - --report-credentials-file=/etc/report/credentials
+      - --target=clean-grafana-datasources
+      - --variant=periodic-cleanup
+      command:
+      - ci-operator
+      env:
+      - name: HTTP_SERVER_IP
+        valueFrom:
+          fieldRef:
+            fieldPath: status.podIP
+      image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
+      imagePullPolicy: Always
+      name: ""
+      ports:
+      - containerPort: 8080
+        name: http
+      resources:
+        requests:
+          cpu: 10m
+      volumeMounts:
+      - mountPath: /etc/boskos
+        name: boskos
+        readOnly: true
+      - mountPath: /secrets/gcs
+        name: gcs-credentials
+        readOnly: true
+      - mountPath: /secrets/manifest-tool
+        name: manifest-tool-local-pusher
+        readOnly: true
+      - mountPath: /etc/pull-secret
+        name: pull-secret
+        readOnly: true
+      - mountPath: /etc/report
+        name: result-aggregator
+        readOnly: true
+    serviceAccountName: ci-operator
+    volumes:
+    - name: boskos
+      secret:
+        items:
+        - key: credentials
+          path: credentials
+        secretName: boskos-credentials
+    - name: manifest-tool-local-pusher
+      secret:
+        secretName: manifest-tool-local-pusher
+    - name: pull-secret
+      secret:
+        secretName: registry-pull-credentials
+    - name: result-aggregator
+      secret:
+        secretName: result-aggregator
 - agent: kubernetes
   cluster: build04
   cron: 35 * * * *

ci-operator/step-registry/aro-hcp/deprovision/grafana-datasources/OWNERS

@@ -0,0 +1,8 @@
+approvers:
+- aro-hcp-sl-approvers
+- geoberle
+- deads2k
+reviewers:
+- aro-hcp-sl-reviewers
+- geoberle
+- deads2k

ci-operator/step-registry/aro-hcp/deprovision/grafana-datasources/aro-hcp-deprovision-grafana-datasources-commands.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+export CLUSTER_PROFILE_DIR="/var/run/aro-hcp-${VAULT_SECRET_PROFILE}"
+
+export AZURE_CLIENT_ID; AZURE_CLIENT_ID=$(cat "${CLUSTER_PROFILE_DIR}/client-id")
+export AZURE_TENANT_ID; AZURE_TENANT_ID=$(cat "${CLUSTER_PROFILE_DIR}/tenant")
+export AZURE_CLIENT_SECRET; AZURE_CLIENT_SECRET=$(cat "${CLUSTER_PROFILE_DIR}/client-secret")
+export AZURE_TOKEN_CREDENTIALS=prod
+
+az login --service-principal -u "${AZURE_CLIENT_ID}" -p "${AZURE_CLIENT_SECRET}" --tenant "${AZURE_TENANT_ID}" --output none
+
+export GLOBAL_INFRA_SUBSCRIPTION_ID; GLOBAL_INFRA_SUBSCRIPTION_ID=$(cat "${CLUSTER_PROFILE_DIR}/infra-global-subscription-id")
+
+echo "Building grafanactl..."
+go build -o /tmp/grafanactl ./tooling/grafanactl
+
+echo "Running: grafanactl clean datasources"
+/tmp/grafanactl clean datasources \
+  --subscription "${GLOBAL_INFRA_SUBSCRIPTION_ID}" \
+  --resource-group "${GRAFANA_RESOURCE_GROUP}" \
+  --grafana-name "${GRAFANA_NAME}"
+
+echo "Running: grafanactl clean fixup-datasources"
+/tmp/grafanactl clean fixup-datasources \
+  --subscription "${GLOBAL_INFRA_SUBSCRIPTION_ID}" \
+  --resource-group "${GRAFANA_RESOURCE_GROUP}" \
+  --grafana-name "${GRAFANA_NAME}"

ci-operator/step-registry/aro-hcp/deprovision/grafana-datasources/aro-hcp-deprovision-grafana-datasources-ref.metadata.json

@@ -0,0 +1,15 @@
+{
+	"path": "aro-hcp/deprovision/grafana-datasources/aro-hcp-deprovision-grafana-datasources-ref.yaml",
+	"owners": {
+		"approvers": [
+			"aro-hcp-sl-approvers",
+			"geoberle",
+			"deads2k"
+		],
+		"reviewers": [
+			"aro-hcp-sl-reviewers",
+			"geoberle",
+			"deads2k"
+		]
+	}
+}

ci-operator/step-registry/aro-hcp/deprovision/grafana-datasources/aro-hcp-deprovision-grafana-datasources-ref.yaml

@@ -0,0 +1,30 @@
+ref:
+  as: aro-hcp-deprovision-grafana-datasources
+  from: aro-hcp-e2e-tests
+  commands: aro-hcp-deprovision-grafana-datasources-commands.sh
+  resources:
+    requests:
+      cpu: 100m
+      memory: 300Mi
+  credentials:
+    - namespace: test-credentials
+      name: cluster-secrets-aro-hcp-dev
+      mount_path: /var/run/aro-hcp-dev
+  env:
+  - name: VAULT_SECRET_PROFILE
+    default: "dev"
+    documentation: |-
+      Selects which environment's cluster secrets to use (dev, int, stg, prod).
+  - name: GRAFANA_RESOURCE_GROUP
+    default: "global"
+    documentation: |-
+      Azure resource group containing the Managed Grafana instance.
+  - name: GRAFANA_NAME
+    default: "arohcp-dev"
+    documentation: |-
+      Name of the Azure Managed Grafana instance to clean.
+  documentation: |-
+    Remove orphaned Prometheus datasources from an Azure Managed Grafana instance.
+    Runs grafanactl clean datasources (removes stale AMW integrations from the
+    Grafana resource) followed by grafanactl clean fixup-datasources (deletes
+    Managed_Prometheus_* datasources not backed by a live Azure Monitor Workspace).