Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions .github/scripts/toolkit-config-v2.jq
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
(type == "object")
and (.version == 2)
and ((keys - ["invoke_max_body_bytes", "signed_http", "version"]) | length == 0)
and (.signed_http | type == "object")
and (.signed_http.mode == "required")
and ((.signed_http | keys - ["allowed_leaders", "allowed_leaders_path", "mode", "tools"]) | length == 0)
and ((.signed_http.allowed_leaders_path | type) == "string")
and ((.signed_http.allowed_leaders_path | length) > 0)
and (.signed_http.tools | type == "object")
and (($expected_fqns | length) > 0)
and ((.signed_http.tools | keys | sort) == $expected_fqns)
and ([
.signed_http.tools[] |
(type == "object")
and ((keys - ["replay_cache_ttl_ms", "tool_signing_key"]) | length == 0)
and ((.tool_signing_key | type) == "string")
and ((.tool_signing_key | length) > 0)
] | all)
31 changes: 20 additions & 11 deletions .github/workflows/offchain-tools.prepare.yml
Original file line number Diff line number Diff line change
Expand Up @@ -143,27 +143,36 @@ jobs:

- name: Generate signed HTTP keys (idempotent)
run: |
set -euo pipefail
set -Eeuo pipefail
VERSIONED="${{ steps.names.outputs.versioned }}"
PROJECT="${{ vars.GCP_PROJECT_ID }}"
META="${RUNNER_TEMP}/prepare/meta/${{ matrix.tool }}.json"

FQNS=$(jq -r '.[].fqn' "$META" | tr '\n' ' ' | xargs)
echo "FQNs for $VERSIONED: $FQNS"

FORCE_FLAG=""
FORCE_ARGS=()
EXISTING=$(gcloud secrets versions access latest \
--secret="nexus-tools-${VERSIONED}-signed-http-keys" \
--project="$PROJECT" 2>/dev/null || echo "")
if [ -n "$EXISTING" ]; then
EXISTING_FQNS=$(echo "$EXISTING" | jq -r '[.keys_registry[].fqn] | sort | join(" ")')
WANTED_FQNS=$(echo "$FQNS" | tr ' ' '\n' | sort | tr '\n' ' ' | xargs)
if [ "$EXISTING_FQNS" != "$WANTED_FQNS" ]; then
echo "FQNs changed: '$EXISTING_FQNS' -> '$WANTED_FQNS' — forcing new keys"
FORCE_FLAG="--force"
else
echo "FQNs unchanged — skipping keygen"
if [[ -n "$EXISTING" ]]; then
EXPECTED_FQNS=$(jq -c '[.[].fqn] | sort' "$META")
if ! EXISTING_FQNS=$(jq -ce '[.keys_registry[].fqn] | sort' <<<"$EXISTING") \
|| ! EXISTING_TOOLKIT_CONFIG=$(jq -ce '.toolkit_config' <<<"$EXISTING"); then
echo "Malformed signing-key secret found — forcing Toolkit-v2 keys"
FORCE_ARGS=(--force)
elif [[ "$EXISTING_FQNS" != "$EXPECTED_FQNS" ]]; then
echo "FQNs changed: '$EXISTING_FQNS' -> '$EXPECTED_FQNS' — forcing new keys"
FORCE_ARGS=(--force)
elif jq -e \
--argjson expected_fqns "$EXPECTED_FQNS" \
-f .github/scripts/toolkit-config-v2.jq \
<<<"$EXISTING_TOOLKIT_CONFIG" >/dev/null; then
echo "FQNs and Toolkit-v2 config unchanged — skipping keygen"
exit 0
else
echo "Legacy or invalid Toolkit config found — forcing Toolkit-v2 keys"
FORCE_ARGS=(--force)
fi
fi

Expand All @@ -173,7 +182,7 @@ jobs:
gcr.io/${{ vars.GCP_INFRA_PROJECT_ID }}/nexus-next/generate-signed-http-keys:latest \
python /app/bin/generate_signed_http_keys.py "nexus-tools" "$VERSIONED" \
--fqns $FQNS \
--project "$PROJECT" $FORCE_FLAG
--project "$PROJECT" "${FORCE_ARGS[@]}"

- name: Upload tool config to GCS
run: |
Expand Down
31 changes: 15 additions & 16 deletions .github/workflows/offchain-tools.register.yml
Original file line number Diff line number Diff line change
Expand Up @@ -372,15 +372,13 @@ jobs:
done
done

- name: Reconcile toolkit-config secret
- name: Reconcile Toolkit-v2 config secret
shell: bash
run: |
set -euo pipefail
set -Eeuo pipefail
MANIFEST="${{ steps.artifacts.outputs.manifest }}"
META_DIR="${{ steps.artifacts.outputs.meta_dir }}"
PROJECT="${{ vars.GCP_PROJECT_ID }}"
BUCKET="${{ vars.GCP_PROJECT_ID }}-nexus-tools"
NETWORK="${{ vars.TOOLS_NAMESPACE || vars.SUI_NETWORK }}"

for TOOL in $(jq -r 'keys[]' "$MANIFEST"); do
VERSION=$(jq -r --arg t "$TOOL" '.[$t].version' "$MANIFEST")
Expand All @@ -393,22 +391,23 @@ jobs:
SKELETON=$(gcloud secrets versions access latest \
--secret="nexus-tools-${VERSIONED}-signed-http-keys" \
--project="$PROJECT" \
| jq -c '.toolkit_config')
| jq -cS '.toolkit_config')
EXPECTED_FQNS=$(jq -c '[.[].fqn] | sort' "${META_DIR}/${TOOL}.json")

# Runtime config v2 embeds only each Tool's signing key. `tool_kid`
# remains on-chain registration metadata and is forbidden here.
if ! jq -e \
--argjson expected_fqns "$EXPECTED_FQNS" \
-f .github/scripts/toolkit-config-v2.jq \
<<<"$SKELETON" >/dev/null; then
echo "::error::Generated Toolkit config for $VERSIONED is not strict schema version 2"
exit 1
fi

DESIRED="$SKELETON"
for FQN in $(jq -r '.[].fqn' "${META_DIR}/${TOOL}.json"); do
REG=$(gsutil cat "gs://${BUCKET}/${NETWORK}/offchain/registration/${TOOL}/${FQN}.json")
KID=$(echo "$REG" | jq -r '.tool_kid // "MISSING"')
if [ "$KID" = "MISSING" ] || [ "$KID" = "null" ]; then
echo "::error::No tool_kid for $FQN — refusing to write toolkit-config"
exit 1
fi
DESIRED=$(echo "$DESIRED" | jq -c --arg fqn "$FQN" --argjson kid "$KID" \
'.signed_http.tools[$fqn].tool_kid = $kid')
done

CURRENT=$(gcloud secrets versions access latest \
--secret="$SECRET" --project="$PROJECT" 2>/dev/null | jq -c . 2>/dev/null || echo "")
--secret="$SECRET" --project="$PROJECT" 2>/dev/null | jq -cS . 2>/dev/null || echo "")
if [ "$DESIRED" = "$CURRENT" ]; then
echo " ✓ toolkit-config for $VERSIONED unchanged"
else
Expand Down
42 changes: 16 additions & 26 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ conventions the CI pipeline relies on:
}
~~~

The `signed_http.enabled` value belongs to the infrastructure deployment descriptor; it is not the Toolkit runtime JSON. CI generates and validates a strict Toolkit runtime config with `"version": 2` before publishing the mounted `toolkit-config` secret.

1. **`build.rs`** — must compile-time validate that the crate name
matches the binary and emit `TOOL_FQN_VERSION` from the Docker
build-arg (defaults to `"1"` for local builds). Copy from an
Expand Down Expand Up @@ -99,14 +101,14 @@ Two roles for branches, one for tags:
deployment is attributable to an exact ref — no merge-strategy
gotchas, no long-lived deploy branches to keep clean.

| Trigger | Matrix | Build | Push images | Chain ops | TF apply |
| --- | --- | --- | --- | --- | --- |
| PR (any base) | changed | ✓ | dry-run | — | — |
| Push to `main` | changed | ✓ | ✓ | — | — |
| Push to `iterate/testnet/**` / `iterate/mainnet/**` | all | ✓ | ✓ | ✓ (env) | ✓ |
| Push tag `testnet-*` / `mainnet-*` | all | ✓ | ✓ | ✓ (tag prefix) | ✓ |
| `workflow_dispatch` `target-env=testnet\|mainnet` | all | ✓ | ✓ | ✓ (chosen) | ✓ |
| `workflow_dispatch` `pr-number=<N>` | all | ✓ | ✓ | ✓ (PR's base) | ✓ |
| Trigger | Matrix | Build | Push images | Chain ops | TF apply |
| --------------------------------------------------- | ------- | ----- | ----------- | -------------- | -------- |
| PR (any base) | changed | ✓ | dry-run | — | — |
| Push to `main` | changed | ✓ | ✓ | — | — |
| Push to `iterate/testnet/**` / `iterate/mainnet/**` | all | ✓ | ✓ | ✓ (env) | ✓ |
| Push tag `testnet-*` / `mainnet-*` | all | ✓ | ✓ | ✓ (tag prefix) | ✓ |
| `workflow_dispatch` `target-env=testnet\|mainnet` | all | ✓ | ✓ | ✓ (chosen) | ✓ |
| `workflow_dispatch` `pr-number=<N>` | all | ✓ | ✓ | ✓ (PR's base) | ✓ |

## Lifecycle: PR → main → deploy

Expand Down Expand Up @@ -153,28 +155,16 @@ FQN URLs on chain point at the right endpoints.

## What "full pipeline" does

1. **discover** — globs `offchain/tools/*/tools.json` and computes a
per-tool content version from `git rev-parse HEAD:offchain/tools/<name>/`
(first 8 hex of sha256 → u32).
1. **deploy** — builds per-tool Docker images and pushes to
`gcr.io/<infra-project>/nexus-tools/<tool>:sha-<7>`. Build and auth
are split so token expiry can't kill a slow build mid-push.
1. **prepare** — runs each container's `--meta` to extract the
ToolMeta JSON, renders a Cloud Run config blob to
`gs://<bucket>/<network>/offchain/tools/<tool>-v<version>.json`,
and generates a signed-HTTP keys secret in Secret Manager. The
`nexus_contracts_tag` is baked from `vars.NEXUS_TAG` into each blob
so it stays pinned to whichever Nexus version was current at
prepare time.
1. **discover** — globs `offchain/tools/*/tools.json` and computes a per-tool content version from `git rev-parse HEAD:offchain/tools/<name>/` (first 8 hex of sha256 → u32).
1. **deploy** — builds per-tool Docker images and pushes to `gcr.io/<infra-project>/nexus-tools/<tool>:sha-<7>`. Build and auth are split so token expiry can't kill a slow build mid-push.
1. **prepare** — runs each container's `--meta` to extract the ToolMeta JSON, renders a Cloud Run config blob to `gs://<bucket>/<network>/offchain/tools/<tool>-v<version>.json`, and generates a signed-HTTP keys secret containing strict Toolkit runtime config version 2 in Secret Manager. The `nexus_contracts_tag` is baked from `vars.NEXUS_TAG` into each blob so it stays pinned to whichever Nexus version was current at prepare time.
1. **register** — for each FQN:
- skips the CLI call if already on chain (snapshot from `nexus tool list --json`),
- otherwise pipes the ToolMeta to `nexus tool register offchain --from-meta -`,
- persists `owner_cap_over_tool` + `owner_cap_over_gas` to
`gs://<bucket>/<network>/offchain/registration/<tool>/<fqn>.json`,
- persists `owner_cap_over_tool` + `owner_cap_over_gas` to `gs://<bucket>/<network>/offchain/registration/<tool>/<fqn>.json`,
- registers signing keys (`nexus tool auth register-key`),
- reconciles the per-tool `toolkit-config` Secret Manager secret.
Pre-step: consolidates the deployer wallet's coins so a single coin
can cover the 1 SUI per-tx budget that heavy schemas require.
- validates strict Toolkit runtime config version 2 and reconciles the per-tool `toolkit-config` Secret Manager secret.
Pre-step: consolidates the deployer wallet's coins so a single coin can cover the 1 SUI per-tx budget that heavy schemas require.
1. **trigger-tf-apply** — dispatches `Talus-Network/tf-nexus-tools`'s
`terraform.yml` (via `the-actions-org/workflow-dispatch`, which
works with fine-grained PATs). Terraform materializes the Cloud
Expand Down
Loading
Loading