From 8ec55cfa3a75d26369001658bf8bc5f32e90d8fd Mon Sep 17 00:00:00 2001 From: Blockchain Analytics Date: Wed, 27 May 2026 20:02:42 +0300 Subject: [PATCH 1/8] Move BQ update to kubernetes --- Dockerfile.bq-updater | 21 +++++ deployment/bq-updater/cronjob.yaml | 97 ++++++++++++++++++++++++ deployment/bq-updater/kustomization.yaml | 11 +++ scripts/run_bq_update.sh | 15 ++-- scripts/run_update_epoch.sh | 8 +- 5 files changed, 144 insertions(+), 8 deletions(-) create mode 100644 Dockerfile.bq-updater create mode 100644 deployment/bq-updater/cronjob.yaml create mode 100644 deployment/bq-updater/kustomization.yaml diff --git a/Dockerfile.bq-updater b/Dockerfile.bq-updater new file mode 100644 index 0000000..09e8eae --- /dev/null +++ b/Dockerfile.bq-updater @@ -0,0 +1,21 @@ +FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + bash \ + ca-certificates \ + coreutils \ + jq \ + postgresql-client \ + sed \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app/scripts + +COPY scripts/ /app/scripts/ +COPY schema/tables/ /app/schema/tables/ + +RUN find /app/scripts -maxdepth 1 -type f -name "*.sh" -exec chmod +x {} \; + +ENTRYPOINT ["bash", "-lc"] +CMD ["./run_bq_update.sh"] diff --git a/deployment/bq-updater/cronjob.yaml b/deployment/bq-updater/cronjob.yaml new file mode 100644 index 0000000..fbe4d85 --- /dev/null +++ b/deployment/bq-updater/cronjob.yaml @@ -0,0 +1,97 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: cardano-bq-slot-sync + labels: + app: cardano-bq-slot-sync +spec: + suspend: true + schedule: "3,33 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + startingDeadlineSeconds: 600 + jobTemplate: + spec: + backoffLimit: 2 + template: + metadata: + labels: + app: cardano-bq-slot-sync + spec: + imagePullSecrets: + - name: pullsecret + restartPolicy: OnFailure + containers: + - name: cardano-bq-slot-sync + image: code.blockchain-applied.com/bca/cardano-bq-updater:1.0.0 + imagePullPolicy: Always + command: ["bash", "-lc"] + args: + - gcloud config set project blockchain-analytics-392322 && ./run_bq_update.sh + env: + - name: BQ_DRYRUN + value: "false" + - name: PUBSUB_TOPIC_NAME + value: "bca-alerts" + envFrom: + - secretRef: + name: bq-config + - secretRef: + name: pg-config + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 1000m + memory: 2Gi +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: cardano-bq-epoch-sync + labels: + app: cardano-bq-epoch-sync +spec: + suspend: true + schedule: "16 5 * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + startingDeadlineSeconds: 1800 + jobTemplate: + spec: + backoffLimit: 2 + template: + metadata: + labels: + app: cardano-bq-epoch-sync + spec: + imagePullSecrets: + - name: pullsecret + restartPolicy: OnFailure + containers: + - name: cardano-bq-epoch-sync + image: code.blockchain-applied.com/bca/cardano-bq-updater:1.0.0 + imagePullPolicy: Always + command: ["bash", "-lc"] + args: + - gcloud config set project blockchain-analytics-392322 && ./run_update_epoch.sh + env: + - name: BQ_DRYRUN + value: "false" + - name: PUBSUB_TOPIC_NAME + value: "bca-alerts" + envFrom: + - secretRef: + name: bq-config + - secretRef: + name: pg-config + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 1000m + memory: 2Gi diff --git a/deployment/bq-updater/kustomization.yaml b/deployment/bq-updater/kustomization.yaml new file mode 100644 index 0000000..964ab79 --- /dev/null +++ b/deployment/bq-updater/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: cardano-bq + +resources: + - cronjob.yaml + +images: + - name: code.blockchain-applied.com/bca/cardano-bq-updater + newTag: "1.0.3" diff --git a/scripts/run_bq_update.sh b/scripts/run_bq_update.sh index 9b2caf0..b917b16 100755 --- a/scripts/run_bq_update.sh +++ b/scripts/run_bq_update.sh @@ -46,16 +46,19 @@ Q+="COMMIT TRANSACTION;" # Print the query for debugging (optional) echo -e "$Q" > query.txt -# DRYRUN="--dry_run" -DRYRUN= +# Set BQ_DRYRUN=true to skip all writes (useful for testing) +[ "${BQ_DRYRUN}" = "true" ] && DRYRUN="--dry_run" || DRYRUN="" # Execute the transaction ${BQ} query --bigqueryrc=$(pwd)/dot.bigqueryrc ${DRYRUN} --dataset_id=${DATASETID} --nouse_legacy_sql < query.txt 2> logs/transaction-query.err > logs/transaction-query.out - -echo "Updating db-sync slot_no to ${ENDING_SLOT} and epoch_no to ${PG_EPOCH} in BigQuery" -Q="UPDATE ${BQ_PROJECT}.db_sync.last_index set last_slot_no=${ENDING_SLOT}, last_epoch_no=${PG_EPOCH} WHERE tablename='db-sync';" -${BQ} query --nouse_legacy_sql "${Q}" +if [ "${BQ_DRYRUN}" = "true" ]; then + echo "DRY RUN: skipping last_index update and Pub/Sub publish" +else + echo "Updating db-sync slot_no to ${ENDING_SLOT} and epoch_no to ${PG_EPOCH} in BigQuery" + Q="UPDATE ${BQ_PROJECT}.db_sync.last_index set last_slot_no=${ENDING_SLOT}, last_epoch_no=${PG_EPOCH} WHERE tablename='db-sync';" + ${BQ} query --nouse_legacy_sql "${Q}" +fi rm ${TEMPDIR}/key.json rmdir ${TEMPDIR} diff --git a/scripts/run_update_epoch.sh b/scripts/run_update_epoch.sh index fdefcbd..aaf4831 100755 --- a/scripts/run_update_epoch.sh +++ b/scripts/run_update_epoch.sh @@ -43,14 +43,18 @@ do if [ "${PG_EPOCH_NO}" -gt "${BQ_EPOCH_NO}" ]; then SCRIPT="./update_epoch_${TABLE}.sh" echo "Updating ${TABLENAME}. BigQuery epoch: ${BQ_EPOCH_NO} - Postgres epoch: ${PG_EPOCH_NO}" - ${SCRIPT} ${BQ_EPOCH_NO} ${PG_EPOCH_NO} + [ "${BQ_DRYRUN}" = "true" ] && echo "DRY RUN: skipping ${SCRIPT}" || ${SCRIPT} ${BQ_EPOCH_NO} ${PG_EPOCH_NO} fi done rm ${TEMPDIR}/key.json rmdir ${TEMPDIR} -gcloud pubsub topics publish ${PUBSUB_TOPIC_NAME} --message "Updated BQ epoch tables to epoch_no ${PG_EPOCH_NO}" --project $BQ_PROJECT +if [ "${BQ_DRYRUN}" = "true" ]; then + echo "DRY RUN: skipping Pub/Sub publish" +else + gcloud pubsub topics publish ${PUBSUB_TOPIC_NAME} --message "Updated BQ epoch tables to epoch_no ${PG_EPOCH_NO}" --project $BQ_PROJECT +fi echo "All done." From 576b0f870a9b86e9eb60113a62df90f0da4d8081 Mon Sep 17 00:00:00 2001 From: Blockchain Analytics Date: Wed, 27 May 2026 20:19:26 +0300 Subject: [PATCH 2/8] Enable cronjobs --- deployment/bq-updater/cronjob.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/bq-updater/cronjob.yaml b/deployment/bq-updater/cronjob.yaml index fbe4d85..57e593d 100644 --- a/deployment/bq-updater/cronjob.yaml +++ b/deployment/bq-updater/cronjob.yaml @@ -5,7 +5,7 @@ metadata: labels: app: cardano-bq-slot-sync spec: - suspend: true + suspend: false schedule: "3,33 * * * *" concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 @@ -54,7 +54,7 @@ metadata: labels: app: cardano-bq-epoch-sync spec: - suspend: true + suspend: false schedule: "16 5 * * *" concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 From d44819f4f5b1534eec5857d5e19a6dd6ae91acf2 Mon Sep 17 00:00:00 2001 From: Blockchain Analytics Date: Wed, 27 May 2026 20:21:42 +0300 Subject: [PATCH 3/8] change restartPolicy --- deployment/bq-updater/cronjob.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/bq-updater/cronjob.yaml b/deployment/bq-updater/cronjob.yaml index 57e593d..afdf662 100644 --- a/deployment/bq-updater/cronjob.yaml +++ b/deployment/bq-updater/cronjob.yaml @@ -21,7 +21,7 @@ spec: spec: imagePullSecrets: - name: pullsecret - restartPolicy: OnFailure + restartPolicy: Never containers: - name: cardano-bq-slot-sync image: code.blockchain-applied.com/bca/cardano-bq-updater:1.0.0 @@ -70,7 +70,7 @@ spec: spec: imagePullSecrets: - name: pullsecret - restartPolicy: OnFailure + restartPolicy: Never containers: - name: cardano-bq-epoch-sync image: code.blockchain-applied.com/bca/cardano-bq-updater:1.0.0 From 3051766ed050a0b45dfa8e9359a24c4b94e138ad Mon Sep 17 00:00:00 2001 From: Blockchain Analytics Date: Tue, 2 Jun 2026 21:59:21 +0300 Subject: [PATCH 4/8] Suspend cronjobs --- deployment/bq-updater/cronjob.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/bq-updater/cronjob.yaml b/deployment/bq-updater/cronjob.yaml index afdf662..a3a81d8 100644 --- a/deployment/bq-updater/cronjob.yaml +++ b/deployment/bq-updater/cronjob.yaml @@ -5,7 +5,7 @@ metadata: labels: app: cardano-bq-slot-sync spec: - suspend: false + suspend: true schedule: "3,33 * * * *" concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 @@ -54,7 +54,7 @@ metadata: labels: app: cardano-bq-epoch-sync spec: - suspend: false + suspend: true schedule: "16 5 * * *" concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 From 297ee7c1a13c37df95666711c71c96d01085a607 Mon Sep 17 00:00:00 2001 From: monoid-is-a-monad <159135221+monoid-is-a-monad@users.noreply.github.com> Date: Tue, 9 Jun 2026 20:18:06 +0300 Subject: [PATCH 5/8] Update deployment/bq-updater/kustomization.yaml Co-authored-by: Alexander Diemand --- deployment/bq-updater/kustomization.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deployment/bq-updater/kustomization.yaml b/deployment/bq-updater/kustomization.yaml index 964ab79..e5246ce 100644 --- a/deployment/bq-updater/kustomization.yaml +++ b/deployment/bq-updater/kustomization.yaml @@ -7,5 +7,6 @@ resources: - cronjob.yaml images: - - name: code.blockchain-applied.com/bca/cardano-bq-updater + - name: cardano-bq-updater + newName: code.blockchain-applied.com/bca/cardano-bq-updater newTag: "1.0.3" From a5bbd1b2483001e515095db028c6738d4e9fc914 Mon Sep 17 00:00:00 2001 From: Blockchain Analytics Date: Tue, 9 Jun 2026 20:31:13 +0300 Subject: [PATCH 6/8] Amend PR comments --- deployment/bq-updater/cronjob.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/deployment/bq-updater/cronjob.yaml b/deployment/bq-updater/cronjob.yaml index a3a81d8..4dd92bd 100644 --- a/deployment/bq-updater/cronjob.yaml +++ b/deployment/bq-updater/cronjob.yaml @@ -5,8 +5,9 @@ metadata: labels: app: cardano-bq-slot-sync spec: - suspend: true - schedule: "3,33 * * * *" + suspend: false + schedule: "34 * * * *" + timeZone: "Etc/UTC" concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 3 @@ -24,7 +25,7 @@ spec: restartPolicy: Never containers: - name: cardano-bq-slot-sync - image: code.blockchain-applied.com/bca/cardano-bq-updater:1.0.0 + image: code.blockchain-applied.com/bca/cardano-bq-updater:1.0.3 imagePullPolicy: Always command: ["bash", "-lc"] args: @@ -54,8 +55,9 @@ metadata: labels: app: cardano-bq-epoch-sync spec: - suspend: true - schedule: "16 5 * * *" + suspend: false + schedule: "10 1 * * *" + timeZone: "Etc/UTC" concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 3 @@ -73,7 +75,7 @@ spec: restartPolicy: Never containers: - name: cardano-bq-epoch-sync - image: code.blockchain-applied.com/bca/cardano-bq-updater:1.0.0 + image: code.blockchain-applied.com/bca/cardano-bq-updater:1.0.3 imagePullPolicy: Always command: ["bash", "-lc"] args: From be0458d71d650cc09cc28705c4a92c01fb500fa9 Mon Sep 17 00:00:00 2001 From: monoid-is-a-monad <159135221+monoid-is-a-monad@users.noreply.github.com> Date: Tue, 9 Jun 2026 20:50:00 +0300 Subject: [PATCH 7/8] use kustomization name --- deployment/bq-updater/cronjob.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/bq-updater/cronjob.yaml b/deployment/bq-updater/cronjob.yaml index 4dd92bd..be2c191 100644 --- a/deployment/bq-updater/cronjob.yaml +++ b/deployment/bq-updater/cronjob.yaml @@ -25,7 +25,7 @@ spec: restartPolicy: Never containers: - name: cardano-bq-slot-sync - image: code.blockchain-applied.com/bca/cardano-bq-updater:1.0.3 + image: cardano-bq-updater imagePullPolicy: Always command: ["bash", "-lc"] args: From 84b363688495dbb971ae2aac12c9a37576aeccd3 Mon Sep 17 00:00:00 2001 From: monoid-is-a-monad <159135221+monoid-is-a-monad@users.noreply.github.com> Date: Tue, 9 Jun 2026 20:57:00 +0300 Subject: [PATCH 8/8] Update cronjob.yaml --- deployment/bq-updater/cronjob.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/bq-updater/cronjob.yaml b/deployment/bq-updater/cronjob.yaml index be2c191..71b8950 100644 --- a/deployment/bq-updater/cronjob.yaml +++ b/deployment/bq-updater/cronjob.yaml @@ -75,7 +75,7 @@ spec: restartPolicy: Never containers: - name: cardano-bq-epoch-sync - image: code.blockchain-applied.com/bca/cardano-bq-updater:1.0.3 + image: cardano-bq-updater imagePullPolicy: Always command: ["bash", "-lc"] args: