From 65c5d1798b3abb5770da7c7f1fcc890f99766fda Mon Sep 17 00:00:00 2001 From: wmuldergov Date: Wed, 10 Jun 2026 08:09:58 -0700 Subject: [PATCH] DBC22-6678: Add webhook for backup container To allow for automated notifications to MS Teams --- .github/workflows/crunchy-dr-failover.yaml | 24 ++++++++++++------- .github/workflows/crunchy.yml | 6 +++-- infrastructure/crunchy-postgres/README.md | 8 +++++-- .../crunchy-postgres/values-dev.yaml | 5 ++++ .../crunchy-postgres/values-prod.yaml | 5 ++++ .../crunchy-postgres/values-test.yaml | 5 ++++ .../crunchy-postgres/values-uat.yaml | 5 ++++ 7 files changed, 46 insertions(+), 12 deletions(-) diff --git a/.github/workflows/crunchy-dr-failover.yaml b/.github/workflows/crunchy-dr-failover.yaml index 3540a63..8683f84 100644 --- a/.github/workflows/crunchy-dr-failover.yaml +++ b/.github/workflows/crunchy-dr-failover.yaml @@ -56,7 +56,8 @@ jobs: --set backup-storage.env.S3_BUCKET.value=${{ secrets.CRUNCHY_S3_BACKUP_STORAGE_BUCKET }} \ --set backup-storage.env.S3_ENDPOINT.value=${{ secrets.CRUNCHY_S3_ENDPOINT }} \ --set backup-storage.env.S3_USER.value=${{ secrets.CRUNCHY_S3_ACCESS_KEY }} \ - --set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }} + --set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }} \ + --set backup-storage.env.WEBHOOK_URL.value=${{ secrets.CRUNCHY_WEBHOOK_URL }} - name: Wait for Gold DB to shut down run: | @@ -99,7 +100,8 @@ jobs: --set backup-storage.env.S3_BUCKET.value=${{ secrets.CRUNCHY_S3_BACKUP_STORAGE_BUCKET }} \ --set backup-storage.env.S3_ENDPOINT.value=${{ secrets.CRUNCHY_S3_ENDPOINT }} \ --set backup-storage.env.S3_USER.value=${{ secrets.CRUNCHY_S3_ACCESS_KEY }} \ - --set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }} + --set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }} \ + --set backup-storage.env.WEBHOOK_URL.value=${{ secrets.CRUNCHY_WEBHOOK_URL }} - name: Confirm Gold DR is Primary run: | @@ -147,7 +149,8 @@ jobs: --set backup-storage.env.S3_BUCKET.value=${{ secrets.CRUNCHY_S3_BACKUP_STORAGE_BUCKET }} \ --set backup-storage.env.S3_ENDPOINT.value=${{ secrets.CRUNCHY_S3_ENDPOINT }} \ --set backup-storage.env.S3_USER.value=${{ secrets.CRUNCHY_S3_ACCESS_KEY }} \ - --set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }} + --set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }} \ + --set backup-storage.env.WEBHOOK_URL.value=${{ secrets.CRUNCHY_WEBHOOK_URL }} - name: Confirm Gold DB deleted run: | @@ -188,7 +191,8 @@ jobs: --set backup-storage.env.S3_BUCKET.value="${{ secrets.CRUNCHY_S3_BACKUP_STORAGE_BUCKET }}" \ --set backup-storage.env.S3_ENDPOINT.value="${{ secrets.CRUNCHY_S3_ENDPOINT }}" \ --set backup-storage.env.S3_USER.value="${{ secrets.CRUNCHY_S3_ACCESS_KEY }}" \ - --set backup-storage.env.S3_PASSWORD.value="${{ secrets.CRUNCHY_S3_SECRET_KEY }}" + --set backup-storage.env.S3_PASSWORD.value="${{ secrets.CRUNCHY_S3_SECRET_KEY }}" \ + --set backup-storage.env.WEBHOOK_URL.value=${{ secrets.CRUNCHY_WEBHOOK_URL }} - name: Confirm Cluster is Standby run: | @@ -239,7 +243,8 @@ jobs: --set backup-storage.env.S3_BUCKET.value=${{ secrets.CRUNCHY_S3_BACKUP_STORAGE_BUCKET }} \ --set backup-storage.env.S3_ENDPOINT.value=${{ secrets.CRUNCHY_S3_ENDPOINT }} \ --set backup-storage.env.S3_USER.value=${{ secrets.CRUNCHY_S3_ACCESS_KEY }} \ - --set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }} + --set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }} \ + --set backup-storage.env.WEBHOOK_URL.value=${{ secrets.CRUNCHY_WEBHOOK_URL }} - name: Wait for Gold DB DR to shut down run: | @@ -281,7 +286,8 @@ jobs: --set backup-storage.env.S3_BUCKET.value=${{ secrets.CRUNCHY_S3_BACKUP_STORAGE_BUCKET }} \ --set backup-storage.env.S3_ENDPOINT.value=${{ secrets.CRUNCHY_S3_ENDPOINT }} \ --set backup-storage.env.S3_USER.value=${{ secrets.CRUNCHY_S3_ACCESS_KEY }} \ - --set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }} + --set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }} \ + --set backup-storage.env.WEBHOOK_URL.value=${{ secrets.CRUNCHY_WEBHOOK_URL }} - name: Confirm Gold is Primary run: | @@ -330,7 +336,8 @@ jobs: --set backup-storage.env.S3_BUCKET.value=${{ secrets.CRUNCHY_S3_BACKUP_STORAGE_BUCKET }} \ --set backup-storage.env.S3_ENDPOINT.value=${{ secrets.CRUNCHY_S3_ENDPOINT }} \ --set backup-storage.env.S3_USER.value=${{ secrets.CRUNCHY_S3_ACCESS_KEY }} \ - --set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }} + --set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }} \ + --set backup-storage.env.WEBHOOK_URL.value=${{ secrets.CRUNCHY_WEBHOOK_URL }} - name: Verify Gold DR DB deleted run: | @@ -372,7 +379,8 @@ jobs: --set backup-storage.env.S3_BUCKET.value="${{ secrets.CRUNCHY_S3_BACKUP_STORAGE_BUCKET }}" \ --set backup-storage.env.S3_ENDPOINT.value="${{ secrets.CRUNCHY_S3_ENDPOINT }}" \ --set backup-storage.env.S3_USER.value="${{ secrets.CRUNCHY_S3_ACCESS_KEY }}" \ - --set backup-storage.env.S3_PASSWORD.value="${{ secrets.CRUNCHY_S3_SECRET_KEY }}" + --set backup-storage.env.S3_PASSWORD.value="${{ secrets.CRUNCHY_S3_SECRET_KEY }}" \ + --set backup-storage.env.WEBHOOK_URL.value=${{ secrets.CRUNCHY_WEBHOOK_URL }} - name: Confirm Gold DR is Standby run: | diff --git a/.github/workflows/crunchy.yml b/.github/workflows/crunchy.yml index ecde628..dab41a1 100644 --- a/.github/workflows/crunchy.yml +++ b/.github/workflows/crunchy.yml @@ -52,7 +52,8 @@ jobs: --set backup-storage.env.S3_BUCKET.value=${{ secrets.CRUNCHY_S3_BACKUP_STORAGE_BUCKET }} \ --set backup-storage.env.S3_ENDPOINT.value=${{ secrets.CRUNCHY_S3_ENDPOINT }} \ --set backup-storage.env.S3_USER.value=${{ secrets.CRUNCHY_S3_ACCESS_KEY }} \ - --set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }} + --set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }} \ + --set backup-storage.env.WEBHOOK_URL.value=${{ secrets.CRUNCHY_WEBHOOK_URL }} deploy-golddr: @@ -84,4 +85,5 @@ jobs: --set backup-storage.env.S3_BUCKET.value=${{ secrets.CRUNCHY_S3_BACKUP_STORAGE_BUCKET }} \ --set backup-storage.env.S3_ENDPOINT.value=${{ secrets.CRUNCHY_S3_ENDPOINT }} \ --set backup-storage.env.S3_USER.value=${{ secrets.CRUNCHY_S3_ACCESS_KEY }} \ - --set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }} + --set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }} \ + --set backup-storage.env.WEBHOOK_URL.value=${{ secrets.CRUNCHY_WEBHOOK_URL }} diff --git a/infrastructure/crunchy-postgres/README.md b/infrastructure/crunchy-postgres/README.md index 8f404ae..af1067c 100644 --- a/infrastructure/crunchy-postgres/README.md +++ b/infrastructure/crunchy-postgres/README.md @@ -34,7 +34,9 @@ Assumptions: --set backup-storage.env.S3_BUCKET.value="${{ secrets.CRUNCHY_S3_BACKUP_STORAGE_BUCKET }}" \ --set backup-storage.env.S3_ENDPOINT.value="${{ secrets.CRUNCHY_S3_ENDPOINT }}" \ --set backup-storage.env.S3_USER.value="${{ secrets.CRUNCHY_S3_ACCESS_KEY }}" \ - --set backup-storage.env.S3_PASSWORD.value="${{ secrets.CRUNCHY_S3_SECRET_KEY }}" + --set backup-storage.env.S3_PASSWORD.value="${{ secrets.CRUNCHY_S3_SECRET_KEY }}" \ + --set backup-storage.env.WEBHOOK_URL.value=${{ secrets.CRUNCHY_WEBHOOK_URL }} + 1. Confirm everything has installed as expected 1. Run these commands to set DB owner in the primary pod like this 1. `oc exec -it $(oc get pods -l "postgres-operator.crunchydata.com/role=master" -o jsonpath='{.items[0].metadata.name}') -- /bin/bash` @@ -61,7 +63,8 @@ If you are setting up Crunchy in Active-Standby configuration you will also need --set backup-storage.env.S3_BUCKET.value="${{ secrets.CRUNCHY_S3_BACKUP_STORAGE_BUCKET }}" \ --set backup-storage.env.S3_ENDPOINT.value="${{ secrets.CRUNCHY_S3_ENDPOINT }}" \ --set backup-storage.env.S3_USER.value="${{ secrets.CRUNCHY_S3_ACCESS_KEY }}" \ - --set backup-storage.env.S3_PASSWORD.value="${{ secrets.CRUNCHY_S3_SECRET_KEY }}" + --set backup-storage.env.S3_PASSWORD.value="${{ secrets.CRUNCHY_S3_SECRET_KEY }}" \ + --set backup-storage.env.WEBHOOK_URL.value=${{ secrets.CRUNCHY_WEBHOOK_URL }} Due to how the clusters are setup, you will need to update the password for `ccp_monitoring` for the Standby cluster based on this documentation: https://access.crunchydata.com/documentation/postgres-operator/latest/tutorials/backups-disaster-recovery/disaster-recovery#monitoring-a-standby-cluster Here are the steps: https://access.crunchydata.com/documentation/postgres-operator/latest/guides/exporter-configuration#setting-a-custom-ccp_monitoring-password @@ -86,6 +89,7 @@ To be able to run that workflow, ensure you have these secrets set for each envi - CRUNCHY_S3_ACCESS_KEY - CRUNCHY_S3_SECRET_KEY - CRUNCHY_USER_PASSWORD +- CRUNCHY_WEBHOOK_URL - OPENSHIFT_GOLD_TOKEN - OPENSHIFT_GOLDDR_TOKEN diff --git a/infrastructure/crunchy-postgres/values-dev.yaml b/infrastructure/crunchy-postgres/values-dev.yaml index 2a358a8..8acdf4c 100644 --- a/infrastructure/crunchy-postgres/values-dev.yaml +++ b/infrastructure/crunchy-postgres/values-dev.yaml @@ -207,6 +207,11 @@ backup-storage: env: ENVIRONMENT_FRIENDLY_NAME: value: "RIDE Dev Database" + ENVIRONMENT_NAME: + value: "dev-ride-db" + WEBHOOK_URL: + value: "" # DO NOT UPLOAD TO GITHUB. + secure: true BACKUP_STRATEGY: value: "rolling" DAILY_BACKUPS: diff --git a/infrastructure/crunchy-postgres/values-prod.yaml b/infrastructure/crunchy-postgres/values-prod.yaml index 3af2f3a..31ae3fe 100644 --- a/infrastructure/crunchy-postgres/values-prod.yaml +++ b/infrastructure/crunchy-postgres/values-prod.yaml @@ -207,6 +207,11 @@ backup-storage: env: ENVIRONMENT_FRIENDLY_NAME: value: "RIDE Prod Database" + ENVIRONMENT_NAME: + value: "prod-ride-db" + WEBHOOK_URL: + value: "" # DO NOT UPLOAD TO GITHUB. + secure: true BACKUP_STRATEGY: value: "rolling" DAILY_BACKUPS: diff --git a/infrastructure/crunchy-postgres/values-test.yaml b/infrastructure/crunchy-postgres/values-test.yaml index 0fc2729..63504e7 100644 --- a/infrastructure/crunchy-postgres/values-test.yaml +++ b/infrastructure/crunchy-postgres/values-test.yaml @@ -207,6 +207,11 @@ backup-storage: env: ENVIRONMENT_FRIENDLY_NAME: value: "RIDE Test Database" + ENVIRONMENT_NAME: + value: "test-ride-db" + WEBHOOK_URL: + value: "" # DO NOT UPLOAD TO GITHUB. + secure: true BACKUP_STRATEGY: value: "rolling" DAILY_BACKUPS: diff --git a/infrastructure/crunchy-postgres/values-uat.yaml b/infrastructure/crunchy-postgres/values-uat.yaml index 102f235..2b2520f 100644 --- a/infrastructure/crunchy-postgres/values-uat.yaml +++ b/infrastructure/crunchy-postgres/values-uat.yaml @@ -207,6 +207,11 @@ backup-storage: env: ENVIRONMENT_FRIENDLY_NAME: value: "RIDE UAT Database" + ENVIRONMENT_NAME: + value: "uat-ride-db" + WEBHOOK_URL: + value: "" # DO NOT UPLOAD TO GITHUB. + secure: true BACKUP_STRATEGY: value: "rolling" DAILY_BACKUPS: